From 98c4d1681f76382a1b809750ded71ebeb90e880a Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Mon, 11 Oct 2021 17:56:41 -0500 Subject: [PATCH 01/16] Http2 to origin This implements the HTTP/2 to origin feature. --- doc/admin-guide/files/records.yaml.en.rst | 19 +- .../statistics/core/http-connection.en.rst | 15 + .../statistics/core/http-transaction.en.rst | 10 + iocore/eventsystem/I_EThread.h | 2 + iocore/eventsystem/I_Thread.h | 1 + iocore/net/UnixNetVConnection.cc | 4 + proxy/PoolableSession.h | 7 + proxy/ProxyTransaction.cc | 28 + proxy/ProxyTransaction.h | 12 + proxy/hdrs/HdrToken.cc | 3 + proxy/hdrs/VersionConverter.cc | 42 +- proxy/http/ConnectingEntry.cc | 157 ++++ proxy/http/ConnectingEntry.h | 79 ++ proxy/http/HttpProxyServerMain.cc | 2 + proxy/http/HttpSM.cc | 735 +++++++++++++----- proxy/http/HttpSM.h | 25 +- proxy/http/HttpSessionManager.cc | 13 +- proxy/http/HttpSessionManager.h | 5 +- proxy/http/HttpTransact.cc | 3 +- proxy/http/HttpTunnel.cc | 33 +- proxy/http/Makefile.am | 2 + proxy/http2/HTTP2.cc | 72 +- proxy/http2/HTTP2.h | 20 +- proxy/http2/Http2ClientSession.cc | 53 +- proxy/http2/Http2ClientSession.h | 5 +- proxy/http2/Http2CommonSession.cc | 26 +- proxy/http2/Http2CommonSession.h | 7 +- proxy/http2/Http2ConnectionState.cc | 460 ++++++++--- proxy/http2/Http2ConnectionState.h | 8 + proxy/http2/Http2ServerSession.cc | 418 ++++++++++ proxy/http2/Http2ServerSession.h | 94 +++ proxy/http2/Http2Stream.cc | 464 +++++++---- proxy/http2/Http2Stream.h | 120 ++- proxy/http2/Makefile.am | 2 + proxy/http2/unit_tests/test_HTTP2.cc | 2 +- src/records/RecHttp.cc | 14 +- src/records/RecordsConfig.cc | 2 + src/records/unit_tests/test_RecHttp.cc | 30 +- src/traffic_server/InkAPI.cc | 12 +- .../chunked_encoding/chunked_encoding.test.py | 2 +- tests/gold_tests/h2/gold/nghttp_0_stdout.gold | 2 - tests/gold_tests/h2/h2origin.test.py | 94 +++ .../h2/h2origin_single_thread.test.py | 90 +++ tests/gold_tests/h2/h2spec.test.py | 2 +- tests/gold_tests/h2/http2.test.py | 6 +- tests/gold_tests/h2/httpbin.test.py | 4 +- .../h2/replay/h1-client-h2-origin.yaml | 596 ++++++++++++++ tests/gold_tests/h2/replay/h2-origin.yaml | 624 +++++++++++++++ ..._slow_server_max_requests_in_0_stderr.gold | 2 +- .../gold_tests/redirect/redirect_post.test.py | 3 +- .../timeout/tls_conn_timeout.test.py | 4 +- .../tls_client_alpn_configuration.replay.yaml | 45 +- .../tls/tls_client_alpn_configuration.test.py | 27 +- 53 files changed, 3871 insertions(+), 636 deletions(-) create mode 100644 proxy/http/ConnectingEntry.cc create mode 100644 proxy/http/ConnectingEntry.h create mode 100644 proxy/http2/Http2ServerSession.cc create mode 100644 proxy/http2/Http2ServerSession.h create mode 100644 tests/gold_tests/h2/h2origin.test.py create mode 100644 tests/gold_tests/h2/h2origin_single_thread.test.py create mode 100644 tests/gold_tests/h2/replay/h1-client-h2-origin.yaml create mode 100644 tests/gold_tests/h2/replay/h2-origin.yaml diff --git a/doc/admin-guide/files/records.yaml.en.rst b/doc/admin-guide/files/records.yaml.en.rst index ea1f7d60a7a..ff2bcce9347 100644 --- a/doc/admin-guide/files/records.yaml.en.rst +++ b/doc/admin-guide/files/records.yaml.en.rst @@ -4026,10 +4026,14 @@ Client-Related Configuration Sets the ALPN string that |TS| will send to the origin in the ClientHello of TLS handshakes. Configuring this to an empty string (the default configuration) means that the ALPN extension - will not be sent as a part of the TLS ClientHello. + will not be sent as a part of the TLS ClientHello, resulting in HTTP/1.x being negotiated for all + origin-side connections. Configuring the ALPN string provides a mechanism to control origin-side HTTP protocol - negotiation. Configuring this requires an understanding of the ALPN TLS protocol extension. See + negotiation. Including ``h2`` in the ALPN list is required for negotiatnge origin-side HTTP/2 + connections. + + Configuring this requires an understanding of the ALPN TLS protocol extension. See `RFC 7301 `_ for details about the ALPN protocol. See the official `IANA ALPN protocol registration `_ @@ -4044,6 +4048,7 @@ Client-Related Configuration - ``http/1.0`` - ``http/1.1`` + - ``h2`` Here are some example configurations and the consequences of each: @@ -4066,6 +4071,9 @@ Client-Related Configuration is currently not supported by |TS|.) ================================ ====================================================================== + Note that this is an overridable configuration, so the ALPN can be configured on a per-origin + basis via the :ref:`admin-plugins-conf-remap` plugin. + .. ts:cv:: CONFIG proxy.config.ssl.async.handshake.enabled INT 0 Enables the use of OpenSSL async job during the TLS handshake. Traffic @@ -4301,6 +4309,13 @@ HTTP/2 Configuration misconfigured or misbehaving clients are opening a large number of connections without submitting requests. +.. ts:cv:: CONFIG proxy.config.http2.no_activity_timeout_out INT 120 + :reloadable: + :units: seconds + + Specifies how long |TS| keeps connections to origins open if a + transaction stalls. + .. ts:cv:: CONFIG proxy.config.http2.zombie_debug_timeout_in INT 0 :reloadable: diff --git a/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst b/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst index a2d95c4e089..667dcf9de15 100644 --- a/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst @@ -183,6 +183,21 @@ HTTP/2 Represents the current number of HTTP/2 active connections from client to the |TS|. +.. ts:stat:: global proxy.process.http2.total_server_connections integer + :type: counter + + Represents the total number of HTTP/2 connections from |TS| to the origin. + +.. ts:stat:: global proxy.process.http2.current_server_connections integer + :type: gauge + + Represents the current number of HTTP/2 connections from |TS| to the origin. + +.. ts:stat:: global proxy.process.http2.current_active_server_connections integer + :type: gauge + + Represents the current number of HTTP/2 active connections from |TS| to the origin. + .. ts:stat:: global proxy.process.http2.connection_errors integer :type: counter diff --git a/doc/admin-guide/monitoring/statistics/core/http-transaction.en.rst b/doc/admin-guide/monitoring/statistics/core/http-transaction.en.rst index 07a6e60a0d8..9dd730c2be4 100644 --- a/doc/admin-guide/monitoring/statistics/core/http-transaction.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/http-transaction.en.rst @@ -165,6 +165,16 @@ HTTP/2 Represents the current number of HTTP/2 streams from client to the |TS|. +.. ts:stat:: global proxy.process.http2.total_server_streams integer + :type: counter + + Represents the total number of HTTP/2 streams from |TS| to the origin. + +.. ts:stat:: global proxy.process.http2.current_server_streams integer + :type: gauge + + Represents the current number of HTTP/2 streams from |TS| to the origin. + .. ts:stat:: global proxy.process.http2.total_transactions_time integer :type: counter :units: seconds diff --git a/iocore/eventsystem/I_EThread.h b/iocore/eventsystem/I_EThread.h index eaff00118dc..eb3d282ef74 100644 --- a/iocore/eventsystem/I_EThread.h +++ b/iocore/eventsystem/I_EThread.h @@ -48,6 +48,7 @@ class PreWarmQueue; class Event; class Continuation; +class ConnectingPool; enum ThreadType { REGULAR = 0, @@ -354,6 +355,7 @@ class EThread : public Thread ServerSessionPool *server_session_pool = nullptr; PreWarmQueue *prewarm_queue = nullptr; + ConnectingPool *connecting_pool = nullptr; /** Default handler used until it is overridden. diff --git a/iocore/eventsystem/I_Thread.h b/iocore/eventsystem/I_Thread.h index b458f59a9eb..43acbde76b5 100644 --- a/iocore/eventsystem/I_Thread.h +++ b/iocore/eventsystem/I_Thread.h @@ -121,6 +121,7 @@ class Thread ProxyAllocator quicNetVCAllocator; ProxyAllocator http1ClientSessionAllocator; ProxyAllocator http2ClientSessionAllocator; + ProxyAllocator http2ServerSessionAllocator; ProxyAllocator http2StreamAllocator; ProxyAllocator httpSMAllocator; ProxyAllocator quicClientSessionAllocator; diff --git a/iocore/net/UnixNetVConnection.cc b/iocore/net/UnixNetVConnection.cc index f8a7d78d224..f1ffb55c2b0 100644 --- a/iocore/net/UnixNetVConnection.cc +++ b/iocore/net/UnixNetVConnection.cc @@ -359,6 +359,7 @@ write_to_net_io(NetHandler *nh, UnixNetVConnection *vc, EThread *thread) { NetState *s = &vc->write; ProxyMutex *mutex = thread->mutex.get(); + Continuation *c = vc->write.vio.cont; MUTEX_TRY_LOCK(lock, s->vio.mutex, thread); @@ -443,6 +444,9 @@ write_to_net_io(NetHandler *nh, UnixNetVConnection *vc, EThread *thread) if (towrite != ntodo && !buf.writer()->high_water()) { if (write_signal_and_update(VC_EVENT_WRITE_READY, vc) != EVENT_CONT) { return; + } else if (c != s->vio.cont) { /* The write vio was updated in the handler */ + write_reschedule(nh, vc); + return; } ntodo = s->vio.ntodo(); diff --git a/proxy/PoolableSession.h b/proxy/PoolableSession.h index 7f6b31ac72d..88cce063402 100644 --- a/proxy/PoolableSession.h +++ b/proxy/PoolableSession.h @@ -85,6 +85,7 @@ class PoolableSession : public ProxySession bool is_private() const; virtual void set_netvc(NetVConnection *newvc); + virtual bool is_multiplexing() const; // Keep track of connection limiting and a pointer to the // singleton that keeps track of the connection counts. @@ -237,3 +238,9 @@ PoolableSession::attach_hostname(const char *hostname) CryptoContext().hash_immediate(hostname_hash, (unsigned char *)hostname, strlen(hostname)); } } + +inline bool +PoolableSession::is_multiplexing() const +{ + return false; +} diff --git a/proxy/ProxyTransaction.cc b/proxy/ProxyTransaction.cc index cb80b3cc2cc..a4d5cb6d83a 100644 --- a/proxy/ProxyTransaction.cc +++ b/proxy/ProxyTransaction.cc @@ -235,6 +235,34 @@ ProxyTransaction::get_version(HTTPHdr &hdr) const return hdr.version_get(); } +bool +ProxyTransaction::is_read_closed() const +{ + return false; +} + +bool +ProxyTransaction::expect_send_trailer() const +{ + return false; +} + +void +ProxyTransaction::set_expect_send_trailer() +{ +} + +bool +ProxyTransaction::expect_receive_trailer() const +{ + return false; +} + +void +ProxyTransaction::set_expect_receive_trailer() +{ +} + bool ProxyTransaction::allow_half_open() const { diff --git a/proxy/ProxyTransaction.h b/proxy/ProxyTransaction.h index c6d1ba79cf8..43fd760114d 100644 --- a/proxy/ProxyTransaction.h +++ b/proxy/ProxyTransaction.h @@ -50,6 +50,11 @@ class ProxyTransaction : public VConnection virtual void set_default_inactivity_timeout(ink_hrtime timeout_in); virtual void cancel_inactivity_timeout(); virtual void cancel_active_timeout(); + virtual bool is_read_closed() const; + virtual bool expect_send_trailer() const; + virtual void set_expect_send_trailer(); + virtual bool expect_receive_trailer() const; + virtual void set_expect_receive_trailer(); // Implement VConnection interface. VIO *do_io_read(Continuation *c, int64_t nbytes = INT64_MAX, MIOBuffer *buf = nullptr) override; @@ -119,6 +124,7 @@ class ProxyTransaction : public VConnection const IpAllow::ACL &get_acl() const; ProxySession *get_proxy_ssn(); + ProxySession const *get_proxy_ssn() const; PoolableSession *get_server_session() const; HttpSM *get_sm() const; @@ -203,6 +209,12 @@ ProxyTransaction::get_proxy_ssn() return _proxy_ssn; } +inline ProxySession const * +ProxyTransaction::get_proxy_ssn() const +{ + return _proxy_ssn; +} + inline PoolableSession * ProxyTransaction::get_server_session() const { diff --git a/proxy/hdrs/HdrToken.cc b/proxy/hdrs/HdrToken.cc index f613fca3e8c..1c8682a2552 100644 --- a/proxy/hdrs/HdrToken.cc +++ b/proxy/hdrs/HdrToken.cc @@ -230,6 +230,9 @@ static HdrTokenFieldInfo _hdrtoken_strs_field_initializers[] = { {"Strict-Transport-Security", MIME_SLOTID_NONE, MIME_PRESENCE_NONE, (HTIF_MULTVALS) }, {"Subject", MIME_SLOTID_NONE, MIME_PRESENCE_SUBJECT, HTIF_NONE }, {"Summary", MIME_SLOTID_NONE, MIME_PRESENCE_SUMMARY, HTIF_NONE }, + // TODO: In the past we have observed issues with having hop-by-hop in here + // for gRPC. We plan to work on gRPC in a future. We should experiment with + // this and verify that it works as expected. {"TE", MIME_SLOTID_TE, MIME_PRESENCE_TE, (HTIF_COMMAS | HTIF_MULTVALS | HTIF_HOPBYHOP)}, {"Transfer-Encoding", MIME_SLOTID_TRANSFER_ENCODING, MIME_PRESENCE_TRANSFER_ENCODING, (HTIF_COMMAS | HTIF_MULTVALS | HTIF_HOPBYHOP) }, diff --git a/proxy/hdrs/VersionConverter.cc b/proxy/hdrs/VersionConverter.cc index bbf61b9c989..cd5efb98cac 100644 --- a/proxy/hdrs/VersionConverter.cc +++ b/proxy/hdrs/VersionConverter.cc @@ -76,7 +76,7 @@ VersionConverter::_convert_req_from_1_to_2(HTTPHdr &header) const field->value_set(header.m_heap, header.m_mime, value, value_len); } else { - ink_abort("initialize HTTP/2 pseudo-headers"); + ink_abort("initialize HTTP/2 pseudo-headers, no :method"); return PARSE_RESULT_ERROR; } @@ -91,7 +91,7 @@ VersionConverter::_convert_req_from_1_to_2(HTTPHdr &header) const field->value_set(header.m_heap, header.m_mime, URL_SCHEME_HTTPS, URL_LEN_HTTPS); } } else { - ink_abort("initialize HTTP/2 pseudo-headers"); + ink_abort("initialize HTTP/2 pseudo-headers, no :scheme"); return PARSE_RESULT_ERROR; } @@ -110,8 +110,11 @@ VersionConverter::_convert_req_from_1_to_2(HTTPHdr &header) const } else { field->value_set(header.m_heap, header.m_mime, value, value_len); } + // Remove the host header field, redundant to the authority field + // For istio/envoy, having both was causing 404 responses + header.field_delete(MIME_FIELD_HOST, MIME_LEN_HOST); } else { - ink_abort("initialize HTTP/2 pseudo-headers"); + ink_abort("initialize HTTP/2 pseudo-headers, no :authority"); return PARSE_RESULT_ERROR; } @@ -119,15 +122,29 @@ VersionConverter::_convert_req_from_1_to_2(HTTPHdr &header) const if (MIMEField *field = header.field_find(PSEUDO_HEADER_PATH.data(), PSEUDO_HEADER_PATH.size()); field != nullptr) { int value_len = 0; const char *value = header.path_get(&value_len); + int param_len = 0; + const char *param = header.params_get(¶m_len); + int query_len = 0; + const char *query = header.query_get(&query_len); + int path_len = value_len + 1; - ts::LocalBuffer buf(value_len + 1); + ts::LocalBuffer buf(value_len + 1 + 1 + 1 + query_len + param_len); char *path = buf.data(); path[0] = '/'; memcpy(path + 1, value, value_len); - - field->value_set(header.m_heap, header.m_mime, path, value_len + 1); + if (param_len > 0) { + path[path_len] = ';'; + memcpy(path + path_len + 1, param, param_len); + path_len += 1 + param_len; + } + if (query_len > 0) { + path[path_len] = '?'; + memcpy(path + path_len + 1, query, query_len); + path_len += 1 + query_len; + } + field->value_set(header.m_heap, header.m_mime, path, path_len); } else { - ink_abort("initialize HTTP/2 pseudo-headers"); + ink_abort("initialize HTTP/2 pseudo-headers, no :path"); return PARSE_RESULT_ERROR; } @@ -173,10 +190,15 @@ VersionConverter::_convert_req_from_2_to_1(HTTPHdr &header) const if (MIMEField *field = header.field_find(PSEUDO_HEADER_AUTHORITY.data(), PSEUDO_HEADER_AUTHORITY.size()); field != nullptr && field->value_is_valid(is_control_BIT | is_ws_BIT)) { int authority_len; + // Set the host header field + MIMEField *host = header.field_find(MIME_FIELD_HOST, MIME_LEN_HOST); + if (host == nullptr) { + host = header.field_create(MIME_FIELD_HOST, MIME_LEN_HOST); + header.field_attach(host); + } const char *authority = field->value_get(&authority_len); - header.m_http->u.req.m_url_impl->set_host(header.m_heap, authority, authority_len, true); - + host->value_set(header.m_heap, header.m_mime, authority, authority_len); header.field_delete(field); } else { return PARSE_RESULT_ERROR; @@ -234,7 +256,7 @@ VersionConverter::_convert_res_from_1_to_2(HTTPHdr &header) const field->value_set(header.m_heap, header.m_mime, status_str, STATUS_VALUE_LEN); } else { - ink_abort("initialize HTTP/2 pseudo-headers"); + ink_abort("initialize HTTP/2 pseudo-headers, no :status"); return PARSE_RESULT_ERROR; } diff --git a/proxy/http/ConnectingEntry.cc b/proxy/http/ConnectingEntry.cc new file mode 100644 index 00000000000..2bd0bd2ada0 --- /dev/null +++ b/proxy/http/ConnectingEntry.cc @@ -0,0 +1,157 @@ +/** @file + + Server side connection management. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + */ + +#include "ConnectingEntry.h" +#include "HttpSM.h" + +ConnectingEntry::~ConnectingEntry() +{ + if (_netvc_read_buffer != nullptr) { + free_MIOBuffer(_netvc_read_buffer); + _netvc_read_buffer = nullptr; + } +} + +int +ConnectingEntry::state_http_server_open(int event, void *data) +{ + Debug("http_connect", "entered inside ConnectingEntry::state_http_server_open"); + + switch (event) { + case NET_EVENT_OPEN: { + _netvc = static_cast(data); + UnixNetVConnection *vc = static_cast(_netvc); + ink_release_assert(_pending_action == nullptr || _pending_action->continuation == vc->get_action()->continuation); + _pending_action = nullptr; + Debug("http_connect", "ConnectingEntrysetting handler for connection handshake"); + // Just want to get a write-ready event so we know that the connection handshake is complete. + // The buffer we create will be handed over to the eventually created server session + _netvc_read_buffer = new_MIOBuffer(HTTP_SERVER_RESP_HDR_BUFFER_INDEX); + _netvc_reader = _netvc_read_buffer->alloc_reader(); + _netvc->do_io_write(this, 1, _netvc_reader); + ink_release_assert(!_connect_sms.empty()); + if (!_connect_sms.empty()) { + HttpSM *prime_connect_sm = *(_connect_sms.begin()); + _netvc->set_inactivity_timeout(prime_connect_sm->get_server_connect_timeout()); + } + ink_release_assert(_pending_action == nullptr); + return 0; + } + case VC_EVENT_READ_COMPLETE: + case VC_EVENT_WRITE_READY: + case VC_EVENT_WRITE_COMPLETE: { + Debug("http_connect", "Kick off %zd state machines waiting for origin", _connect_sms.size()); + this->remove_entry(); + _netvc->do_io_write(nullptr, 0, nullptr); + if (!_connect_sms.empty()) { + auto prime_iter = _connect_sms.rbegin(); + ink_release_assert(prime_iter != _connect_sms.rend()); + PoolableSession *new_session = (*prime_iter)->create_server_session(_netvc, _netvc_read_buffer, _netvc_reader); + _netvc = nullptr; + _netvc_read_buffer = nullptr; + + // Did we end up with a multiplexing session? + int count = 0; + if (new_session->is_multiplexing()) { + // Hand off to all queued up ConnectSM's. + while (!_connect_sms.empty()) { + Debug("http_connect", "ConnectingEntry Pass along CONNECT_EVENT_TXN %d", count++); + auto entry = _connect_sms.begin(); + + SCOPED_MUTEX_LOCK(lock, (*entry)->mutex, this_ethread()); + (*entry)->handleEvent(CONNECT_EVENT_TXN, new_session); + _connect_sms.erase(entry); + } + } else { + // Hand off to one and tell all of the others to connect directly + Debug("http_connect", "ConnectingEntry send CONNECT_EVENT_TXN to first %d", count++); + { + SCOPED_MUTEX_LOCK(lock, (*prime_iter)->mutex, this_ethread()); + (*prime_iter)->handleEvent(CONNECT_EVENT_TXN, new_session); + _connect_sms.erase((++prime_iter).base()); + } + while (!_connect_sms.empty()) { + auto entry = _connect_sms.begin(); + Debug("http_connect", "ConnectingEntry Pass along CONNECT_EVENT_DIRECT %d", count++); + SCOPED_MUTEX_LOCK(lock, (*entry)->mutex, this_ethread()); + (*entry)->handleEvent(CONNECT_EVENT_DIRECT, nullptr); + _connect_sms.erase(entry); + } + } + } else { + ink_release_assert(!"There should be some sms on the connect_entry"); + } + delete this; + + // ConnectingEntry should remove itself from the tables and delete itself + return 0; + } + case VC_EVENT_INACTIVITY_TIMEOUT: + case VC_EVENT_ACTIVE_TIMEOUT: + case VC_EVENT_ERROR: + case NET_EVENT_OPEN_FAILED: { + Debug("http_connect", "Stop %zd state machines waiting for failed origin", _connect_sms.size()); + this->remove_entry(); + int vc_provided_cert = 0; + int lerrno = EIO; + if (_netvc != nullptr) { + vc_provided_cert = _netvc->provided_cert(); + lerrno = _netvc->lerrno; + _netvc->do_io_close(); + } + while (!_connect_sms.empty()) { + auto entry = _connect_sms.begin(); + SCOPED_MUTEX_LOCK(lock, (*entry)->mutex, this_ethread()); + (*entry)->t_state.set_connect_fail(lerrno); + (*entry)->server_connection_provided_cert = vc_provided_cert; + (*entry)->handleEvent(event, data); + _connect_sms.erase(entry); + } + // ConnectingEntry should remove itself from the tables and delete itself + delete this; + + return 0; + } + default: + Error("[ConnectingEntry::state_http_server_open] Unknown event: %d", event); + ink_release_assert(0); + return 0; + } + + return 0; +} + +void +ConnectingEntry::remove_entry() +{ + EThread *ethread = this_ethread(); + auto ip_iter = ethread->connecting_pool->m_ip_pool.find(this->_ipaddr); + while (ip_iter != ethread->connecting_pool->m_ip_pool.end() && this->_ipaddr == ip_iter->first) { + if (ip_iter->second == this) { + ethread->connecting_pool->m_ip_pool.erase(ip_iter); + break; + } + ++ip_iter; + } +} diff --git a/proxy/http/ConnectingEntry.h b/proxy/http/ConnectingEntry.h new file mode 100644 index 00000000000..3427295a4b8 --- /dev/null +++ b/proxy/http/ConnectingEntry.h @@ -0,0 +1,79 @@ +/** @file + + Server side connection management. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + */ + +#include "PoolableSession.h" + +#include +#include + +class HttpSM; + +/** Represents a server side session entry in a ConnectionPool to an origin. */ +class ConnectingEntry : public Continuation +{ +public: + ConnectingEntry() = default; + ~ConnectingEntry() override; + void remove_entry(); + int state_http_server_open(int event, void *data); + static PoolableSession *create_server_session(HttpSM *root_sm, NetVConnection *netvc, MIOBuffer *netvc_read_buffer, + IOBufferReader *netvc_reader); + +public: + std::string sni; + std::string cert_name; + IpEndpoint _ipaddr; + std::string hostname; + std::set _connect_sms; + NetVConnection *_netvc = nullptr; + +private: + MIOBuffer *_netvc_read_buffer = nullptr; + IOBufferReader *_netvc_reader = nullptr; + Action *_pending_action = nullptr; + NetVCOptions opt; +}; + +struct IpHelper { + size_t + operator()(IpEndpoint const &arg) const + { + return IpAddr{&arg.sa}.hash(); + } + bool + operator()(IpEndpoint const &arg1, IpEndpoint const &arg2) const + { + return ats_ip_addr_port_eq(&arg1.sa, &arg2.sa); + } +}; + +using ConnectingIpPool = std::unordered_multimap; + +/** Represents the set of connections to an origin. */ +class ConnectingPool +{ +public: + ConnectingPool() = default; + ConnectingIpPool m_ip_pool; +}; diff --git a/proxy/http/HttpProxyServerMain.cc b/proxy/http/HttpProxyServerMain.cc index 6b5a1fa9ebe..d092c60a369 100644 --- a/proxy/http/HttpProxyServerMain.cc +++ b/proxy/http/HttpProxyServerMain.cc @@ -51,6 +51,7 @@ HttpSessionAccept *plugin_http_accept = nullptr; HttpSessionAccept *plugin_http_transparent_accept = nullptr; extern std::function create_h1_server_session; +extern std::function create_h2_server_session; extern std::map> ProtocolSessionCreateMap; static SLL ssl_plugin_acceptors; @@ -225,6 +226,7 @@ MakeHttpProxyAcceptor(HttpProxyAcceptor &acceptor, HttpProxyPort &port, unsigned } ProtocolSessionCreateMap.insert({TS_ALPN_PROTOCOL_INDEX_HTTP_1_0, create_h1_server_session}); ProtocolSessionCreateMap.insert({TS_ALPN_PROTOCOL_INDEX_HTTP_1_1, create_h1_server_session}); + ProtocolSessionCreateMap.insert({TS_ALPN_PROTOCOL_INDEX_HTTP_2_0, create_h2_server_session}); if (port.isSSL()) { SSLNextProtocolAccept *ssl = new SSLNextProtocolAccept(probe, port.m_transparent_passthrough); diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 47f8249b57f..ba92d11d085 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -24,11 +24,13 @@ #include "../ProxyTransaction.h" #include "HttpSM.h" +#include "ConnectingEntry.h" #include "HttpTransact.h" #include "HttpBodyFactory.h" #include "HttpTransactHeaders.h" #include "ConfigProcessor.h" #include "Http1ServerSession.h" +#include "Http2ServerSession.h" #include "HttpDebugNames.h" #include "HttpSessionManager.h" #include "P_Cache.h" @@ -204,7 +206,6 @@ HttpVCTable::find_entry(VIO *vio) void HttpVCTable::remove_entry(HttpVCTableEntry *e) { - ink_assert(e->vc == nullptr || e->in_tunnel); e->vc = nullptr; e->eos = false; if (e->read_buffer) { @@ -237,18 +238,6 @@ HttpVCTable::cleanup_entry(HttpVCTableEntry *e) { ink_assert(e->vc); if (e->in_tunnel == false) { - // Update stats - switch (e->vc_type) { - case HTTP_UA_VC: - // proxy.process.http.current_client_transactions is decremented in HttpSM::destroy - break; - default: - // This covers: - // HTTP_UNKNOWN, HTTP_SERVER_VC, HTTP_TRANSFORM_VC, HTTP_CACHE_READ_VC, - // HTTP_CACHE_WRITE_VC, HTTP_RAW_SERVER_VC - break; - } - if (e->vc_type == HTTP_SERVER_VC) { HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_cleanup_entry); } @@ -268,6 +257,14 @@ HttpVCTable::cleanup_all() } } +void +initialize_thread_for_connecting_pools(EThread *thread) +{ + if (thread->connecting_pool == nullptr) { + thread->connecting_pool = new ConnectingPool(); + } +} + #define SMDebug(tag, fmt, ...) SpecificDebug(debug_on, tag, "[%" PRId64 "] " fmt, sm_id, ##__VA_ARGS__) #define REMEMBER(e, r) \ @@ -372,6 +369,8 @@ HttpSM::init(bool from_early_data) magic = HTTP_SM_MAGIC_ALIVE; + server_txn = nullptr; + // Unique state machine identifier sm_id = next_sm_id++; t_state.state_machine = this; @@ -601,8 +600,7 @@ HttpSM::attach_client_session(ProxyTransaction *client_vc) // this hook maybe asynchronous, we need to disable IO on // client but set the continuation to be the state machine // so if we get an timeout events the sm handles them - // hold onto enabling read until setup_client_read_request_header - ua_entry->read_vio = client_vc->do_io_read(this, 0, nullptr); + ua_entry->read_vio = client_vc->do_io_read(this, 0, ua_txn->get_remote_reader()->mbuf); ua_entry->write_vio = client_vc->do_io_write(this, 0, nullptr); ///////////////////////// @@ -795,8 +793,9 @@ HttpSM::state_read_client_request_header(int event, void *data) ua_raw_buffer_reader = nullptr; } http_parser_clear(&http_parser); - ua_entry->vc_read_handler = &HttpSM::state_watch_for_client_abort; - ua_entry->vc_write_handler = &HttpSM::state_watch_for_client_abort; + ua_entry->vc_read_handler = &HttpSM::state_watch_for_client_abort; + ua_entry->vc_write_handler = &HttpSM::state_watch_for_client_abort; + ua_txn->cancel_inactivity_timeout(); milestones[TS_MILESTONE_UA_READ_HEADER_DONE] = Thread::get_hrtime(); } @@ -998,19 +997,23 @@ HttpSM::state_watch_for_client_abort(int event, void *data) */ case VC_EVENT_EOS: { // We got an early EOS. If the tunnal has cache writer, don't kill it for background fill. - NetVConnection *netvc = ua_txn->get_netvc(); - if (ua_txn->allow_half_open() || tunnel.has_consumer_besides_client()) { - if (netvc) { - netvc->do_io_shutdown(IO_SHUTDOWN_READ); + if (!terminate_sm) { // Not done already + NetVConnection *netvc = ua_txn->get_netvc(); + if (ua_txn->allow_half_open() || tunnel.has_consumer_besides_client()) { + if (netvc) { + netvc->do_io_shutdown(IO_SHUTDOWN_READ); + } + } else { + ua_txn->do_io_close(); + vc_table.cleanup_entry(ua_entry); + ua_entry = nullptr; + tunnel.kill_tunnel(); + terminate_sm = true; // Just die already, the requester is gone + set_ua_abort(HttpTransact::ABORTED, event); + } + if (ua_entry) { + ua_entry->eos = true; } - ua_entry->eos = true; - } else { - ua_txn->do_io_close(); - vc_table.cleanup_entry(ua_entry); - ua_entry = nullptr; - tunnel.kill_tunnel(); - terminate_sm = true; // Just die already, the requester is gone - set_ua_abort(HttpTransact::ABORTED, event); } break; } @@ -1222,14 +1225,6 @@ HttpSM::state_raw_http_server_open(int event, void *data) pending_action = nullptr; switch (event) { - case EVENT_INTERVAL: - // If we get EVENT_INTERNAL it means that we moved the transaction - // to a different thread in do_http_server_open. Since we didn't - // do any of the actual work in do_http_server_open, we have to - // go back and do it now. - do_http_server_open(true); - return 0; - case NET_EVENT_OPEN: { // Record the VC in our table server_entry = vc_table.new_entry(); @@ -1550,7 +1545,7 @@ plugins required to work with sni_routing. api_timer = -Thread::get_hrtime_updated(); HTTP_SM_SET_DEFAULT_HANDLER(&HttpSM::state_api_callout); ink_release_assert(pending_action.empty()); - pending_action = mutex->thread_holding->schedule_in(this, HRTIME_MSECONDS(10)); + pending_action = this_ethread()->schedule_in(this, HRTIME_MSECONDS(10)); return -1; } @@ -1625,6 +1620,10 @@ plugins required to work with sni_routing. } break; + // Eat the EOS while we are waiting for any locks to complete the transaction + case VC_EVENT_EOS: + return 0; + default: ink_assert(false); terminate_sm = true; @@ -1818,7 +1817,7 @@ HttpSM::handle_api_return() } PoolableSession * -HttpSM::create_server_session(NetVConnection *netvc) +HttpSM::create_server_session(NetVConnection *netvc, MIOBuffer *netvc_read_buffer, IOBufferReader *netvc_reader) { // Figure out what protocol was negotiated int proto_index = SessionProtocolNameRegistry::INVALID; @@ -1833,28 +1832,24 @@ HttpSM::create_server_session(NetVConnection *netvc) PoolableSession *retval = ProxySession::create_outbound_session(proto_index); - HttpTransact::State &s = this->t_state; - retval->sharing_pool = static_cast(s.http_config_param->server_session_sharing_pool); - retval->sharing_match = static_cast(s.txn_conf->server_session_sharing_match); - MIOBuffer *netvc_read_buffer = new_MIOBuffer(HTTP_SERVER_RESP_HDR_BUFFER_INDEX); - IOBufferReader *netvc_reader = netvc_read_buffer->alloc_reader(); + retval->sharing_pool = static_cast(t_state.http_config_param->server_session_sharing_pool); + retval->sharing_match = static_cast(t_state.txn_conf->server_session_sharing_match); + retval->attach_hostname(t_state.current.server->name); retval->new_connection(netvc, netvc_read_buffer, netvc_reader); - retval->attach_hostname(s.current.server->name); - - ATS_PROBE1(new_origin_server_connection, s.current.server->name); + ATS_PROBE1(new_origin_server_connection, t_state.current.server->name); retval->set_active(); if (netvc) { - ats_ip_copy(&s.server_info.src_addr, netvc->get_local_addr()); + ats_ip_copy(&t_state.server_info.src_addr, netvc->get_local_addr()); } // If origin_max_connections or origin_min_keep_alive_connections is set then we are metering // the max and or min number of connections per host. Transfer responsibility for this to the // session object. - if (s.outbound_conn_track_state.is_active()) { - SMDebug("http_connect", "max number of outbound connections: %d", s.txn_conf->outbound_conntrack.max); - retval->enable_outbound_connection_tracking(s.outbound_conn_track_state.drop()); + if (t_state.outbound_conn_track_state.is_active()) { + SMDebug("http_connect", "max number of outbound connections: %d", t_state.txn_conf->outbound_conntrack.max); + retval->enable_outbound_connection_tracking(t_state.outbound_conn_track_state.drop()); } return retval; } @@ -1862,14 +1857,26 @@ HttpSM::create_server_session(NetVConnection *netvc) bool HttpSM::create_server_txn(PoolableSession *new_session) { + ink_assert(new_session != nullptr); bool retval = false; - server_txn = new_session->new_transaction(); - if (server_txn != nullptr) { + + server_txn = new_session->new_transaction(); + if (server_txn) { + retval = true; server_txn->attach_transaction(this); + if (t_state.current.request_to == ResolveInfo::PARENT_PROXY) { + new_session->to_parent_proxy = true; + HTTP_INCREMENT_DYN_STAT(http_current_parent_proxy_connections_stat); + HTTP_INCREMENT_DYN_STAT(http_total_parent_proxy_connections_stat); + } else { + new_session->to_parent_proxy = false; + } server_txn->do_io_write(this, 0, nullptr); attach_server_session(); - retval = true; } + _netvc = nullptr; + _netvc_read_buffer = nullptr; + _netvc_reader = nullptr; return retval; } @@ -1892,78 +1899,72 @@ HttpSM::state_http_server_open(int event, void *data) switch (event) { case NET_EVENT_OPEN: { - NetVConnection *netvc = static_cast(data); - UnixNetVConnection *vc = static_cast(data); - PoolableSession *new_session = this->create_server_session(netvc); - if (t_state.current.request_to == ResolveInfo::PARENT_PROXY) { - new_session->to_parent_proxy = true; - HTTP_INCREMENT_DYN_STAT(http_current_parent_proxy_connections_stat); - HTTP_INCREMENT_DYN_STAT(http_total_parent_proxy_connections_stat); - } else { - new_session->to_parent_proxy = false; - } - this->create_server_txn(new_session); - // Since the UnixNetVConnection::action_ or SocksEntry::action_ may be returned from netProcessor.connect_re, and the - // SocksEntry::action_ will be copied into UnixNetVConnection::action_ before call back NET_EVENT_OPEN from SocksEntry::free(), - // so we just compare the Continuation between pending_action and VC's action_. + // SocksEntry::action_ will be copied into UnixNetVConnection::action_ before call back NET_EVENT_OPEN from + // SocksEntry::free(), so we just compare the Continuation between pending_action and VC's action_. + _netvc = static_cast(data); + _netvc_read_buffer = new_MIOBuffer(HTTP_SERVER_RESP_HDR_BUFFER_INDEX); + _netvc_reader = _netvc_read_buffer->alloc_reader(); + UnixNetVConnection *vc = static_cast(_netvc); ink_release_assert(pending_action.empty() || pending_action.get_continuation() == vc->get_action()->continuation); pending_action = nullptr; if (this->plugin_tunnel_type == HTTP_NO_PLUGIN_TUNNEL) { - SMDebug("http", "setting handler for TCP handshake"); - // Just want to get a write-ready event so we know that the TCP handshake is complete. - server_entry->vc_write_handler = &HttpSM::state_http_server_open; - server_entry->vc_read_handler = &HttpSM::state_http_server_open; - - int64_t nbytes = 1; - if (t_state.txn_conf->proxy_protocol_out >= 0) { - nbytes = do_outbound_proxy_protocol(server_txn->get_remote_reader()->mbuf, vc, ua_txn->get_netvc(), - t_state.txn_conf->proxy_protocol_out); - } - - server_entry->write_vio = server_txn->do_io_write(this, nbytes, server_txn->get_remote_reader()); + SMDebug("http_connect", "setting handler for connection handshake timeout %" PRId64, this->get_server_connect_timeout()); + // Just want to get a write-ready event so we know that the connection handshake is complete. + // The buffer we create will be handed over to the eventually created server session + _netvc->do_io_write(this, 1, _netvc_reader); + _netvc->set_inactivity_timeout(this->get_server_connect_timeout()); } else { // in the case of an intercept plugin don't to the connect timeout change - SMDebug("http", "not setting handler for TCP handshake"); + SMDebug("http_connect", "not setting handler for connection handshake"); + this->create_server_txn(this->create_server_session(_netvc, _netvc_read_buffer, _netvc_reader)); handle_http_server_open(); } - + ink_assert(pending_action.empty()); return 0; } + case CONNECT_EVENT_DIRECT: + // Try it again, but direct this time + do_http_server_open(false, true); + break; + case CONNECT_EVENT_TXN: + SMDebug("http", "Connection handshake complete via CONNECT_EVENT_TXN"); + if (this->create_server_txn(static_cast(data))) { + write_outbound_proxy_protocol(); + handle_http_server_open(); + } else { // Failed to create transaction. Maybe too many active transactions already + // Try again (probably need a bounding counter here) + do_http_server_open(false); + } + return 0; case VC_EVENT_READ_COMPLETE: case VC_EVENT_WRITE_READY: case VC_EVENT_WRITE_COMPLETE: // Update the time out to the regular connection timeout. - SMDebug("http_ss", "TCP Handshake complete"); - server_entry->vc_write_handler = &HttpSM::state_send_server_request_header; - - // Reset the timeout to the non-connect timeout - server_txn->set_inactivity_timeout(get_server_inactivity_timeout()); + SMDebug("http_ss", "Connection handshake complete"); + this->create_server_txn(this->create_server_session(_netvc, _netvc_read_buffer, _netvc_reader)); + write_outbound_proxy_protocol(); t_state.current.server->clear_connect_fail(); handle_http_server_open(); return 0; - case EVENT_INTERVAL: // Delayed call from another thread - if (server_txn == nullptr) { - do_http_server_open(); - } - break; case VC_EVENT_INACTIVITY_TIMEOUT: case VC_EVENT_ACTIVE_TIMEOUT: t_state.set_connect_fail(ETIMEDOUT); /* fallthrough */ case VC_EVENT_ERROR: case NET_EVENT_OPEN_FAILED: { - if (server_txn) { - NetVConnection *vc = server_txn->get_netvc(); - if (vc) { - t_state.set_connect_fail(vc->lerrno); - server_connection_provided_cert = vc->provided_cert(); - } - } - t_state.current.state = HttpTransact::CONNECTION_ERROR; t_state.outbound_conn_track_state.clear(); + if (_netvc != nullptr) { + if (event == VC_EVENT_ERROR || event == NET_EVENT_OPEN_FAILED) { + t_state.set_connect_fail(_netvc->lerrno); + } + this->server_connection_provided_cert = _netvc->provided_cert(); + _netvc->do_io_write(nullptr, 0, nullptr); + _netvc->do_io_close(); + _netvc = nullptr; + } /* If we get this error in transparent mode, then we simply can't bind to the 4-tuple to make the connection. There's no hope of retries succeeding in the near future. The best option is to just shut down the connection without further comment. The @@ -2026,6 +2027,8 @@ HttpSM::state_read_server_response_header(int event, void *data) case VC_EVENT_READ_READY: case VC_EVENT_READ_COMPLETE: // More data to parse + // Got some data, won't retry origin connection on error + t_state.current.attempts.maximize(t_state.configured_connect_attempts_max_retries()); break; case VC_EVENT_ERROR: @@ -2077,6 +2080,12 @@ HttpSM::state_read_server_response_header(int event, void *data) http_parser_clear(&http_parser); milestones[TS_MILESTONE_SERVER_READ_HEADER_DONE] = Thread::get_hrtime(); + // Any other events to the end + if (server_entry->vc_type == HTTP_SERVER_VC) { + server_entry->vc_read_handler = &HttpSM::tunnel_handler; + server_entry->vc_write_handler = &HttpSM::tunnel_handler; + } + // If there is a post body in transit, give up on it if (tunnel.is_tunnel_alive()) { tunnel.abort_tunnel(); @@ -2105,6 +2114,9 @@ HttpSM::state_read_server_response_header(int event, void *data) if (allow_error == false) { SMDebug("http_seq", "Error parsing server response header"); t_state.current.state = HttpTransact::PARSE_ERROR; + // We set this to 0 because otherwise HttpTransact::retry_server_connection_not_open + // will raise an assertion if the value is the default UNKNOWN_INTERNAL_ERROR. + t_state.cause_of_death_errno = 0; // If the server closed prematurely on us, use the // server setup error routine since it will forward @@ -2186,9 +2198,9 @@ HttpSM::state_send_server_request_header(int event, void *data) break; case VC_EVENT_WRITE_COMPLETE: - if (server_entry->write_vio != nullptr) { - // We are done sending the request header, deallocate - // our buffer and then decide what to do next + // We are done sending the request header, deallocate + // our buffer and then decide what to do next + if (server_entry->write_buffer) { free_MIOBuffer(server_entry->write_buffer); server_entry->write_buffer = nullptr; method = t_state.hdr_info.server_request.method_get_wksidx(); @@ -2204,6 +2216,10 @@ HttpSM::state_send_server_request_header(int event, void *data) } } } + // Any other events to these read response + if (server_entry->vc_type == HTTP_SERVER_VC) { + server_entry->vc_read_handler = &HttpSM::state_read_server_response_header; + } } break; @@ -2254,6 +2270,91 @@ HttpSM::state_send_server_request_header(int event, void *data) return 0; } +bool +HttpSM::origin_multiplexed() const +{ + return (t_state.dns_info.http_version == HTTP_2_0 || t_state.dns_info.http_version == HTTP_INVALID); +} + +void +HttpSM::cancel_pending_server_connection() +{ + EThread *ethread = this_ethread(); + if (nullptr == ethread->connecting_pool) { + return; // No pending requests + } + if (t_state.current.server) { + IpEndpoint ip; + ip.assign(&this->t_state.current.server->dst_addr.sa); + auto ip_iter = ethread->connecting_pool->m_ip_pool.find(ip); + while (ip_iter != ethread->connecting_pool->m_ip_pool.end() && ip_iter->first == ip) { + ConnectingEntry *connecting_entry = ip_iter->second; + // Found a match + // Look for our sm in the queue + auto entry = connecting_entry->_connect_sms.find(this); + if (entry != connecting_entry->_connect_sms.end()) { + connecting_entry->_connect_sms.erase(entry); + if (connecting_entry->_connect_sms.empty()) { + if (connecting_entry->_netvc) { + connecting_entry->_netvc->do_io_write(nullptr, 0, nullptr); + connecting_entry->_netvc->do_io_close(); + } + ethread->connecting_pool->m_ip_pool.erase(ip_iter); + delete connecting_entry; + break; + } else { + // Leave the shared entry remaining alone + } + } + ++ip_iter; + } + } +} + +// Returns true if there was a matching entry that we +// queued this request on +bool +HttpSM::add_to_existing_request() +{ + HttpTransact::State &s = this->t_state; + bool retval = false; + EThread *ethread = this_ethread(); + + if (this->plugin_tunnel_type != HTTP_NO_PLUGIN_TUNNEL) { + return false; + } + + if (nullptr == ethread->connecting_pool) { + initialize_thread_for_connecting_pools(ethread); + } + auto my_nh = ((UnixNetVConnection *)(this)->ua_txn->get_netvc())->nh; + ink_release_assert(my_nh == nullptr /* PluginVC */ || my_nh == get_NetHandler(this_ethread())); + + HTTP_SM_SET_DEFAULT_HANDLER(&HttpSM::state_http_server_open); + + IpEndpoint ip; + ip.assign(&s.current.server->dst_addr.sa); + auto ip_iter = ethread->connecting_pool->m_ip_pool.find(ip); + std::string_view proposed_sni = this->get_outbound_sni(); + std::string_view proposed_cert = this->get_outbound_cert(); + std::string_view proposed_hostname = this->t_state.current.server->name; + while (!retval && ip_iter != ethread->connecting_pool->m_ip_pool.end() && ip_iter->first == ip) { + // Check that entry matches sni, hostname, and cert + if (proposed_hostname == ip_iter->second->hostname && proposed_sni == ip_iter->second->sni && + proposed_cert == ip_iter->second->cert_name && ip_iter->second->_connect_sms.size() < 50) { + // Pre-emptively set a server connect failure that will be cleared once a WRITE_READY is received from origin or + // bytes are received back + this->t_state.set_connect_fail(EIO); + ip_iter->second->_connect_sms.insert(this); + Debug("http_connect", "Add entry to connection queue. size=%" PRId64, ip_iter->second->_connect_sms.size()); + retval = true; + break; + } + ++ip_iter; + } + return retval; +} + void HttpSM::process_srv_info(HostDBRecord *record) { @@ -2350,11 +2451,6 @@ int HttpSM::state_hostdb_lookup(int event, void *data) { STATE_ENTER(&HttpSM::state_hostdb_lookup, event); - // ink_assert (m_origin_server_vc == 0); - // REQ_FLAVOR_SCHEDULED_UPDATE can be transformed into - // REQ_FLAVOR_REVPROXY - ink_assert(t_state.req_flavor == HttpTransact::REQ_FLAVOR_SCHEDULED_UPDATE || - t_state.req_flavor == HttpTransact::REQ_FLAVOR_REVPROXY || ua_entry->vc != nullptr); switch (event) { case EVENT_HOST_DB_LOOKUP: @@ -2385,7 +2481,6 @@ HttpSM::state_hostdb_lookup(int event, void *data) default: ink_assert(!"Unexpected event"); } - return 0; } @@ -2700,7 +2795,7 @@ HttpSM::main_handler(int event, void *data) } if (vc_entry) { - jump_point = static_cast(data) == vc_entry->read_vio ? vc_entry->vc_read_handler : vc_entry->vc_write_handler; + jump_point = (static_cast(data) == vc_entry->read_vio) ? vc_entry->vc_read_handler : vc_entry->vc_write_handler; ink_assert(jump_point != (HttpSMHandler) nullptr); ink_assert(vc_entry->vc != (VConnection *)nullptr); (this->*jump_point)(event, data); @@ -2865,7 +2960,6 @@ HttpSM::tunnel_handler_post(int event, void *data) // Is the response header ready and waiting? // If so, go ahead and do the hook processing if (milestones[TS_MILESTONE_SERVER_READ_HEADER_DONE] != 0) { - Warning("Process waiting response id=[%" PRId64, sm_id); t_state.current.state = HttpTransact::CONNECTION_ALIVE; t_state.transact_return_point = HttpTransact::HandleResponse; t_state.api_next_action = HttpTransact::SM_ACTION_API_READ_RESPONSE_HDR; @@ -2879,6 +2973,50 @@ HttpSM::tunnel_handler_post(int event, void *data) return 0; } +int +HttpSM::tunnel_handler_trailer(int event, void *data) +{ + STATE_ENTER(&HttpSM::tunnel_handler_trailer, event); + + switch (event) { + case HTTP_TUNNEL_EVENT_DONE: // Response tunnel done. + break; + + default: + // If the response tunnel did not succeed, just clean up as in the default case + return tunnel_handler(event, data); + } + + ink_assert(event == HTTP_TUNNEL_EVENT_DONE); + + // Set up a new tunnel to transport the trailing header to the UA + HTTP_SM_SET_DEFAULT_HANDLER(&HttpSM::tunnel_handler); + + MIOBuffer *trailer_buffer = new_MIOBuffer(HTTP_HEADER_BUFFER_SIZE_INDEX); + IOBufferReader *buf_start = trailer_buffer->alloc_reader(); + + size_t nbytes = INT64_MAX; + int start_bytes = trailer_buffer->write(server_txn->get_remote_reader(), server_txn->get_remote_reader()->read_avail()); + server_txn->get_remote_reader()->consume(start_bytes); + // The server has already sent all it has + if (server_txn->is_read_closed()) { + nbytes = start_bytes; + } + // Signal the ua_txn to get ready for a trailer + ua_txn->set_expect_send_trailer(); + tunnel.reset(); + HttpTunnelProducer *p = tunnel.add_producer(server_entry->vc, nbytes, buf_start, &HttpSM::tunnel_handler_trailer_server, + HT_HTTP_SERVER, "http server trailer"); + tunnel.add_consumer(ua_entry->vc, server_entry->vc, &HttpSM::tunnel_handler_trailer_ua, HT_HTTP_CLIENT, "user agent trailer"); + + ua_entry->in_tunnel = true; + server_entry->in_tunnel = true; + + tunnel.tunnel_run(p); + + return 0; +} + int HttpSM::tunnel_handler_cache_fill(int event, void *data) { @@ -2889,12 +3027,31 @@ HttpSM::tunnel_handler_cache_fill(int event, void *data) ink_release_assert(cache_sm.cache_write_vc); - tunnel.deallocate_buffers(); - this->postbuf_clear(); - tunnel.reset(); + int64_t alloc_index = find_server_buffer_size(); + MIOBuffer *buf = new_MIOBuffer(alloc_index); + IOBufferReader *buf_start = buf->alloc_reader(); - setup_server_transfer_to_cache_only(); - tunnel.tunnel_run(); + TunnelChunkingAction_t action = + (t_state.current.server && t_state.current.server->transfer_encoding == HttpTransact::CHUNKED_ENCODING) ? + TCA_DECHUNK_CONTENT : + TCA_PASSTHRU_DECHUNKED_CONTENT; + + int64_t nbytes = server_transfer_init(buf, 0); + + HTTP_SM_SET_DEFAULT_HANDLER(&HttpSM::tunnel_handler); + + server_entry->vc = server_txn; + HttpTunnelProducer *p = + tunnel.add_producer(server_entry->vc, nbytes, buf_start, &HttpSM::tunnel_handler_server, HT_HTTP_SERVER, "http server"); + + tunnel.set_producer_chunking_action(p, 0, action); + tunnel.set_producer_chunking_size(p, t_state.txn_conf->http_chunking_size); + + setup_cache_write_transfer(&cache_sm, server_entry->vc, &t_state.cache_info.object_store, 0, "cache write"); + + server_entry->in_tunnel = true; + // Kick off the new producer + tunnel.tunnel_run(p); return 0; } @@ -3082,7 +3239,6 @@ HttpSM::tunnel_handler_server(int event, HttpTunnelProducer *p) t_state.current.server->state = HttpTransact::TRANSACTION_COMPLETE; break; } - HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_server); close_connection = true; @@ -3146,6 +3302,13 @@ HttpSM::tunnel_handler_server(int event, HttpTunnelProducer *p) tunnel.local_finish_all(p); } } + if (server_txn->expect_receive_trailer()) { + SMDebug("http", "wait for that trailing header"); + // Swap out the default hander to set up the new tunnel for the trailer exchange. + HTTP_SM_SET_DEFAULT_HANDLER(&HttpSM::tunnel_handler_trailer); + tunnel.local_finish_all(p); + return 0; + } break; case HTTP_TUNNEL_EVENT_CONSUMER_DETACH: @@ -3221,6 +3384,84 @@ HttpSM::tunnel_handler_server(int event, HttpTunnelProducer *p) return 0; } +int +HttpSM::tunnel_handler_trailer_server(int event, HttpTunnelProducer *p) +{ + STATE_ENTER(&HttpSM::tunnel_handler_trailer_server, event); + + switch (event) { + case VC_EVENT_INACTIVITY_TIMEOUT: + case VC_EVENT_ACTIVE_TIMEOUT: + case VC_EVENT_ERROR: + t_state.squid_codes.log_code = SQUID_LOG_ERR_READ_TIMEOUT; + t_state.squid_codes.hier_code = SQUID_HIER_TIMEOUT_DIRECT; + /* fallthru */ + + case VC_EVENT_EOS: + + switch (event) { + case VC_EVENT_INACTIVITY_TIMEOUT: + t_state.current.server->state = HttpTransact::INACTIVE_TIMEOUT; + break; + case VC_EVENT_ACTIVE_TIMEOUT: + t_state.current.server->state = HttpTransact::ACTIVE_TIMEOUT; + break; + case VC_EVENT_ERROR: + t_state.current.server->state = HttpTransact::CONNECTION_ERROR; + break; + case VC_EVENT_EOS: + t_state.current.server->state = HttpTransact::TRANSACTION_COMPLETE; + break; + } + + ink_assert(p->vc_type == HT_HTTP_SERVER); + + SMDebug("http", "aborting HTTP tunnel due to server truncation"); + tunnel.chain_abort_all(p); + + t_state.current.server->abort = HttpTransact::ABORTED; + t_state.client_info.keep_alive = HTTP_NO_KEEPALIVE; + t_state.current.server->keep_alive = HTTP_NO_KEEPALIVE; + t_state.squid_codes.log_code = SQUID_LOG_ERR_READ_ERROR; + break; + + case HTTP_TUNNEL_EVENT_PRECOMPLETE: + case VC_EVENT_READ_COMPLETE: + // + // The transfer completed successfully + p->read_success = true; + t_state.current.server->state = HttpTransact::TRANSACTION_COMPLETE; + t_state.current.server->abort = HttpTransact::DIDNOT_ABORT; + break; + + case HTTP_TUNNEL_EVENT_CONSUMER_DETACH: + case VC_EVENT_READ_READY: + case VC_EVENT_WRITE_READY: + case VC_EVENT_WRITE_COMPLETE: + default: + // None of these events should ever come our way + ink_assert(0); + break; + } + + // We handled the event. Now either shutdown server transaction + ink_assert(server_entry->vc == p->vc); + ink_assert(p->vc_type == HT_HTTP_SERVER); + ink_assert(p->vc == server_txn); + + // The server session has been released. Clean all pointer + // Calling remove_entry instead of server_entry because we don't + // want to close the server VC at this point + vc_table.remove_entry(server_entry); + + p->vc->do_io_close(); + p->read_vio = nullptr; + + server_entry = nullptr; + + return 0; +} + // int HttpSM::tunnel_handler_100_continue_ua(int event, HttpTunnelConsumer* c) // // Used for tunneling the 100 continue response. The tunnel @@ -3242,6 +3483,7 @@ HttpSM::tunnel_handler_100_continue_ua(int event, HttpTunnelConsumer *c) case VC_EVENT_ACTIVE_TIMEOUT: case VC_EVENT_ERROR: set_ua_abort(HttpTransact::ABORTED, event); + vc_table.remove_entry(ua_entry); c->vc->do_io_close(); break; case VC_EVENT_WRITE_COMPLETE: @@ -3340,13 +3582,13 @@ HttpSM::tunnel_handler_ua(int event, HttpTunnelConsumer *c) HTTP_INCREMENT_DYN_STAT(http_background_fill_current_count_stat); HTTP_INCREMENT_DYN_STAT(http_background_fill_total_count_stat); - ink_assert(server_entry->vc == server_txn); ink_assert(c->is_downstream_from(server_txn)); server_txn->set_active_timeout(HRTIME_SECONDS(t_state.txn_conf->background_fill_active_timeout)); } // Even with the background fill, the client side should go down c->write_vio = nullptr; + vc_table.remove_entry(ua_entry); c->vc->do_io_close(EHTTP_ERROR); c->alive = false; @@ -3415,8 +3657,9 @@ HttpSM::tunnel_handler_ua(int event, HttpTunnelConsumer *c) break; } - ink_assert(ua_entry->vc == c->vc); - if (close_connection) { + if (event == VC_EVENT_WRITE_COMPLETE && server_txn && server_txn->expect_receive_trailer()) { + // Don't shutdown if we are still expecting a trailer + } else if (close_connection) { // If the client could be pipelining or is doing a POST, we need to // set the ua_txn into half close mode @@ -3428,6 +3671,7 @@ HttpSM::tunnel_handler_ua(int event, HttpTunnelConsumer *c) } vc_table.remove_entry(this->ua_entry); + ink_release_assert(vc_table.find_entry(ua_txn) == nullptr); ua_txn->do_io_close(); } else { ink_assert(ua_txn->get_remote_reader() != nullptr); @@ -3438,6 +3682,66 @@ HttpSM::tunnel_handler_ua(int event, HttpTunnelConsumer *c) return 0; } +int +HttpSM::tunnel_handler_trailer_ua(int event, HttpTunnelConsumer *c) +{ + HttpTunnelProducer *p = nullptr; + HttpTunnelConsumer *selfc = nullptr; + + STATE_ENTER(&HttpSM::tunnel_handler_trailer_ua, event); + ink_assert(c->vc == ua_txn); + milestones[TS_MILESTONE_UA_CLOSE] = Thread::get_hrtime(); + + switch (event) { + case VC_EVENT_EOS: + ua_entry->eos = true; + + // FALL-THROUGH + case VC_EVENT_INACTIVITY_TIMEOUT: + case VC_EVENT_ACTIVE_TIMEOUT: + case VC_EVENT_ERROR: + + // The user agent died or aborted. Check to + // see if we should setup a background fill + set_ua_abort(HttpTransact::ABORTED, event); + + // Should not be processing trailer headers in the background fill case + ink_assert(!is_bg_fill_necessary(c)); + p = c->producer; + tunnel.chain_abort_all(c->producer); + selfc = p->self_consumer; + if (selfc) { + // This is the case where there is a transformation between ua and os + p = selfc->producer; + // if producer is the cache or OS, close the producer. + // Otherwise in case of large docs, producer iobuffer gets filled up, + // waiting for a consumer to consume data and the connection is never closed. + if (p->alive && ((p->vc_type == HT_CACHE_READ) || (p->vc_type == HT_HTTP_SERVER))) { + tunnel.chain_abort_all(p); + } + } + break; + + case VC_EVENT_WRITE_COMPLETE: + c->write_success = true; + t_state.client_info.abort = HttpTransact::DIDNOT_ABORT; + break; + case VC_EVENT_WRITE_READY: + case VC_EVENT_READ_READY: + case VC_EVENT_READ_COMPLETE: + default: + // None of these events should ever come our way + ink_assert(0); + break; + } + + ink_assert(ua_entry->vc == c->vc); + vc_table.remove_entry(this->ua_entry); + ua_txn->do_io_close(); + ink_release_assert(vc_table.find_entry(ua_txn) == nullptr); + return 0; +} + int HttpSM::tunnel_handler_ua_push(int event, HttpTunnelProducer *p) { @@ -4944,7 +5248,7 @@ HttpSM::get_outbound_sni() const // ////////////////////////////////////////////////////////////////////////// void -HttpSM::do_http_server_open(bool raw) +HttpSM::do_http_server_open(bool raw, bool only_direct) { int ip_family = t_state.current.server->dst_addr.sa.sa_family; auto fam_name = ats_ip_family_name(ip_family); @@ -5089,6 +5393,7 @@ HttpSM::do_http_server_open(bool raw) (t_state.txn_conf->keep_alive_post_out == 1 || t_state.hdr_info.request_content_length <= 0) && !is_private() && ua_txn != nullptr) { HSMresult_t shared_result; + SMDebug("http_ss", "Try to acquire_session for %s", t_state.current.server->name); shared_result = httpSessionManager.acquire_session(this, // state machine &t_state.current.server->dst_addr.sa, // ip + port t_state.current.server->name, // hostname @@ -5168,6 +5473,18 @@ HttpSM::do_http_server_open(bool raw) ink_release_assert(ua_txn == nullptr); } } + + bool multiplexed_origin = !only_direct && !raw && this->origin_multiplexed() && !is_private(); + if (multiplexed_origin) { + SMDebug("http_ss", "Check for existing connect request"); + if (this->add_to_existing_request()) { + SMDebug("http_ss", "Queue behind existing request"); + // We are queued up behind an existing connect request + // Go away and wait. + return; + } + } + // Check to see if we have reached the max number of connections. // Atomically read the current number of connections and check to see // if we have gone above the max allowed. @@ -5331,7 +5648,7 @@ HttpSM::do_http_server_open(bool raw) opt.ssl_client_private_key_name = t_state.txn_conf->ssl_client_private_key_filename; opt.ssl_client_ca_cert_name = t_state.txn_conf->ssl_client_ca_cert_filename; if (is_private()) { - // If the connection to origin is private, don't try to negotiate higher overhead protocols. + // If the connection to origin is private, don't try to negotiate the higher overhead H2 opt.alpn_protocols_array_size = -1; SMDebug("ssl_alpn", "Clear ALPN for private session"); } else if (t_state.txn_conf->ssl_client_alpn_protocols != nullptr) { @@ -5341,6 +5658,28 @@ HttpSM::do_http_server_open(bool raw) opt.alpn_protocols_array_size); } + ConnectingEntry *new_entry = nullptr; + if (multiplexed_origin) { + EThread *ethread = this_ethread(); + if (nullptr != ethread->connecting_pool) { + SMDebug("http_ss", "Queue multiplexed request"); + new_entry = new ConnectingEntry(); + new_entry->mutex = this->mutex; + new_entry->handler = (ContinuationHandler)&ConnectingEntry::state_http_server_open; + new_entry->_ipaddr.assign(&t_state.current.server->dst_addr.sa); + new_entry->hostname = t_state.current.server->name; + new_entry->sni = this->get_outbound_sni(); + new_entry->cert_name = this->get_outbound_cert(); + this->t_state.set_connect_fail(EIO); + new_entry->_connect_sms.insert(this); + ethread->connecting_pool->m_ip_pool.insert(std::make_pair(new_entry->_ipaddr, new_entry)); + } + } + + Continuation *cont = new_entry; + if (!cont) { + cont = this; + } if (tls_upstream) { SMDebug("http", "calling sslNetProcessor.connect_re"); @@ -5361,12 +5700,12 @@ HttpSM::do_http_server_open(bool raw) opt.set_ssl_servername(t_state.server_info.name); } - pending_action = sslNetProcessor.connect_re(this, // state machine + pending_action = sslNetProcessor.connect_re(cont, // state machine or ConnectingEntry &t_state.current.server->dst_addr.sa, // addr + port &opt); } else { SMDebug("http", "calling netProcessor.connect_re"); - pending_action = netProcessor.connect_re(this, // state machine + pending_action = netProcessor.connect_re(cont, // state machine or ConnectingEntry &t_state.current.server->dst_addr.sa, // addr + port &opt); } @@ -5659,7 +5998,6 @@ HttpSM::handle_post_failure() t_state.client_info.keep_alive = HTTP_NO_KEEPALIVE; t_state.current.server->keep_alive = HTTP_NO_KEEPALIVE; - ink_assert(server_txn->get_remote_reader()->read_avail() == 0); tunnel.deallocate_buffers(); tunnel.reset(); // Server died @@ -5702,15 +6040,18 @@ HttpSM::handle_http_server_open() } } server_txn->set_inactivity_timeout(get_server_inactivity_timeout()); - } - int method = t_state.hdr_info.server_request.method_get_wksidx(); - if (method != HTTP_WKSIDX_TRACE && - (t_state.hdr_info.request_content_length > 0 || t_state.client_info.transfer_encoding == HttpTransact::CHUNKED_ENCODING) && - do_post_transform_open()) { - do_setup_post_tunnel(HTTP_TRANSFORM_VC); // Seems like we should be sending the request along this way too - } else if (server_txn != nullptr) { - setup_server_send_request_api(); + int method = t_state.hdr_info.server_request.method_get_wksidx(); + if (method != HTTP_WKSIDX_TRACE && + server_txn->has_request_body(t_state.hdr_info.response_content_length, + t_state.server_info.transfer_encoding == HttpTransact::CHUNKED_ENCODING) && + do_post_transform_open()) { + do_setup_post_tunnel(HTTP_TRANSFORM_VC); /* This doesn't seem quite right. Should be sending the request header */ + } else { + setup_server_send_request_api(); + } + } else { + ink_release_assert(!"No server_txn"); } } @@ -5969,6 +6310,10 @@ HttpSM::do_setup_post_tunnel(HttpVC_t to_vc_type) client_request_body_bytes = num_body_bytes; } ua_txn->get_remote_reader()->consume(num_body_bytes); + // The user agent has already sent all it has + if (ua_txn->is_read_closed()) { + post_bytes = num_body_bytes; + } p = tunnel.add_producer(ua_entry->vc, post_bytes - transfered_bytes, buf_start, &HttpSM::tunnel_handler_post_ua, HT_HTTP_CLIENT, "user agent post"); } @@ -6005,14 +6350,22 @@ HttpSM::do_setup_post_tunnel(HttpVC_t to_vc_type) this->setup_client_request_plugin_agents(p); - // The user agent may support chunked (HTTP/1.1) or not (HTTP/2) - // In either case, the server will support chunked (HTTP/1.1) + // The user agent and origin may support chunked (HTTP/1.1) or not (HTTP/2) if (chunked) { if (ua_txn->is_chunked_encoding_supported()) { - tunnel.set_producer_chunking_action(p, 0, TCA_PASSTHRU_CHUNKED_CONTENT); + if (server_txn->is_chunked_encoding_supported()) { + tunnel.set_producer_chunking_action(p, 0, TCA_PASSTHRU_CHUNKED_CONTENT); + } else { + tunnel.set_producer_chunking_action(p, 0, TCA_DECHUNK_CONTENT); + tunnel.set_producer_chunking_size(p, 0); + } } else { - tunnel.set_producer_chunking_action(p, 0, TCA_CHUNK_CONTENT); - tunnel.set_producer_chunking_size(p, 0); + if (server_txn->is_chunked_encoding_supported()) { + tunnel.set_producer_chunking_action(p, 0, TCA_CHUNK_CONTENT); + tunnel.set_producer_chunking_size(p, 0); + } else { + tunnel.set_producer_chunking_action(p, 0, TCA_PASSTHRU_DECHUNKED_CONTENT); + } } } @@ -6180,6 +6533,17 @@ HttpSM::write_header_into_buffer(HTTPHdr *h, MIOBuffer *b) return dumpoffset; } +void +HttpSM::write_outbound_proxy_protocol() +{ + int64_t nbytes = 1; + if (t_state.txn_conf->proxy_protocol_out >= 0) { + nbytes = do_outbound_proxy_protocol(server_txn->get_remote_reader()->mbuf, server_txn->get_netvc(), ua_txn->get_netvc(), + t_state.txn_conf->proxy_protocol_out); + } + server_entry->write_vio = server_txn->do_io_write(this, nbytes, server_txn->get_remote_reader()); +} + void HttpSM::attach_server_session() { @@ -6266,13 +6630,15 @@ HttpSM::attach_server_session() // Do we need Transfer_Encoding? if (ua_txn->has_request_body(t_state.hdr_info.request_content_length, t_state.client_info.transfer_encoding == HttpTransact::CHUNKED_ENCODING)) { - // See if we need to insert a chunked header - if (!t_state.hdr_info.server_request.presence(MIME_PRESENCE_CONTENT_LENGTH) && - !t_state.hdr_info.server_request.presence(MIME_PRESENCE_TRANSFER_ENCODING)) { - // Stuff in a TE setting so we treat this as chunked, sort of. - t_state.server_info.transfer_encoding = HttpTransact::CHUNKED_ENCODING; - t_state.hdr_info.server_request.value_append(MIME_FIELD_TRANSFER_ENCODING, MIME_LEN_TRANSFER_ENCODING, HTTP_VALUE_CHUNKED, - HTTP_LEN_CHUNKED, true); + if (server_txn->is_chunked_encoding_supported()) { + // See if we need to insert a chunked header + if (!t_state.hdr_info.server_request.presence(MIME_PRESENCE_CONTENT_LENGTH) && + !t_state.hdr_info.server_request.presence(MIME_PRESENCE_TRANSFER_ENCODING)) { + // Stuff in a TE setting so we treat this as chunked, sort of. + t_state.server_info.transfer_encoding = HttpTransact::CHUNKED_ENCODING; + t_state.hdr_info.server_request.value_append(MIME_FIELD_TRANSFER_ENCODING, MIME_LEN_TRANSFER_ENCODING, HTTP_VALUE_CHUNKED, + HTTP_LEN_CHUNKED, true); + } } } @@ -6362,10 +6728,6 @@ HttpSM::setup_server_read_response_header() server_response_hdr_bytes = 0; milestones[TS_MILESTONE_SERVER_READ_HEADER_DONE] = 0; - // We already done the READ when we setup the connection to - // read the request header - ink_assert(server_entry->read_vio); - // The tunnel from OS to UA is now setup. Ready to read the response server_entry->read_vio = server_txn->do_io_read(this, INT64_MAX, server_txn->get_remote_reader()->mbuf); @@ -6376,6 +6738,7 @@ HttpSM::setup_server_read_response_header() if (server_txn->get_remote_reader()->read_avail() > 0) { state_read_server_response_header((server_entry->eos) ? VC_EVENT_EOS : VC_EVENT_READ_READY, server_entry->read_vio); } + ink_assert(server_entry->vc != nullptr); } HttpTunnelProducer * @@ -6809,36 +7172,6 @@ HttpSM::setup_transfer_from_transform_to_cache_only() return p; } -void -HttpSM::setup_server_transfer_to_cache_only() -{ - TunnelChunkingAction_t action; - int64_t alloc_index; - int64_t nbytes; - - alloc_index = find_server_buffer_size(); - MIOBuffer *buf = new_MIOBuffer(alloc_index); - IOBufferReader *buf_start = buf->alloc_reader(); - - action = (t_state.current.server && t_state.current.server->transfer_encoding == HttpTransact::CHUNKED_ENCODING) ? - TCA_DECHUNK_CONTENT : - TCA_PASSTHRU_DECHUNKED_CONTENT; - - nbytes = server_transfer_init(buf, 0); - - HTTP_SM_SET_DEFAULT_HANDLER(&HttpSM::tunnel_handler); - - HttpTunnelProducer *p = - tunnel.add_producer(server_entry->vc, nbytes, buf_start, &HttpSM::tunnel_handler_server, HT_HTTP_SERVER, "http server"); - - tunnel.set_producer_chunking_action(p, 0, action); - tunnel.set_producer_chunking_size(p, t_state.txn_conf->http_chunking_size); - - setup_cache_write_transfer(&cache_sm, server_entry->vc, &t_state.cache_info.object_store, 0, "cache write"); - - server_entry->in_tunnel = true; -} - HttpTunnelProducer * HttpSM::setup_server_transfer() { @@ -6897,28 +7230,6 @@ HttpSM::setup_server_transfer() this->setup_client_response_plugin_agents(p, client_response_hdr_bytes); - // If the incoming server response is chunked and the client does not - // expect a chunked response, then dechunk it. Otherwise, if the - // incoming response is not chunked and the client expects a chunked - // response, then chunk it. - /* - // this block is moved up so that we know if we need to remove - // Content-Length field from response header before writing the - // response header into buffer bz50730 - TunnelChunkingAction_t action; - if (t_state.client_info.receive_chunked_response == false) { - if (t_state.current.server->transfer_encoding == - HttpTransact::CHUNKED_ENCODING) - action = TCA_DECHUNK_CONTENT; - else action = TCA_PASSTHRU_DECHUNKED_CONTENT; - } - else { - if (t_state.current.server->transfer_encoding != - HttpTransact::CHUNKED_ENCODING) - action = TCA_CHUNK_CONTENT; - else action = TCA_PASSTHRU_CHUNKED_CONTENT; - } - */ tunnel.set_producer_chunking_action(p, client_response_hdr_bytes, action); tunnel.set_producer_chunking_size(p, t_state.txn_conf->http_chunking_size); return p; @@ -7151,12 +7462,17 @@ HttpSM::kill_this() transform_cache_sm.end_both(); vc_table.cleanup_all(); - // tunnel.deallocate_buffers(); - // Why don't we just kill the tunnel? Might still be - // active if the state machine is going down hard, - // and we should clean it up. + // Clean up the tunnel resources. Take + // it down if it is still active tunnel.kill_tunnel(); + if (_netvc) { + _netvc->do_io_close(); + free_MIOBuffer(_netvc_read_buffer); + } else if (server_txn == nullptr) { + this->cancel_pending_server_connection(); + } + // It possible that a plugin added transform hook // but the hook never executed due to a client abort // In that case, we need to manually close all the @@ -7638,7 +7954,7 @@ HttpSM::set_next_state() if (ua_txn && !ua_txn->has_request_body(t_state.hdr_info.request_content_length, t_state.client_info.transfer_encoding == HttpTransact::CHUNKED_ENCODING)) { ua_txn->cancel_inactivity_timeout(); - } else if (!ua_txn) { + } else if (!ua_txn || ua_txn->get_netvc() == nullptr) { terminate_sm = true; return; // Give up if there is no session } @@ -8252,6 +8568,9 @@ HttpSM::get_http_schedule(int event, void * /* data ATS_UNUSED */) return 0; } +/* + * Used from an InkAPI + */ bool HttpSM::set_server_session_private(bool private_session) { @@ -8262,8 +8581,8 @@ HttpSM::set_server_session_private(bool private_session) return false; } -inline bool -HttpSM::is_private() +bool +HttpSM::is_private() const { bool res = false; if (will_be_private_ss) { diff --git a/proxy/http/HttpSM.h b/proxy/http/HttpSM.h index 3239c19d645..0bacac38a38 100644 --- a/proxy/http/HttpSM.h +++ b/proxy/http/HttpSM.h @@ -49,6 +49,9 @@ #define HTTP_API_CONTINUE (INK_API_EVENT_EVENTS_START + 0) #define HTTP_API_ERROR (INK_API_EVENT_EVENTS_START + 1) +#define CONNECT_EVENT_TXN (HTTP_NET_CONNECTION_EVENT_EVENTS_START) + 0 +#define CONNECT_EVENT_DIRECT (HTTP_NET_CONNECTION_EVENT_EVENTS_START) + 1 + // The default size for http header buffers when we don't // need to include extra space for the document static size_t const HTTP_HEADER_BUFFER_SIZE_INDEX = CLIENT_CONNECTION_FIRST_READ_BUFFER_SIZE_INDEX; @@ -60,7 +63,7 @@ static size_t const HTTP_HEADER_BUFFER_SIZE_INDEX = CLIENT_CONNECTION_FIRST_READ // the larger buffer size static size_t const HTTP_SERVER_RESP_HDR_BUFFER_INDEX = BUFFER_SIZE_INDEX_8K; -class Http1ServerSession; +class PoolableSession; class AuthHttpAdapter; class PreWarmSM; @@ -225,13 +228,15 @@ class HttpSM : public Continuation, public PluginUserArgs // holding the lock for the server session void attach_server_session(); - PoolableSession *create_server_session(NetVConnection *netvc); + PoolableSession *create_server_session(NetVConnection *netvc, MIOBuffer *netvc_read_buffer, IOBufferReader *netvc_reader); bool create_server_txn(PoolableSession *new_session); HTTPVersion get_server_version(HTTPHdr &hdr) const; ProxyTransaction *get_ua_txn(); ProxyTransaction *get_server_txn(); + // Write out the proxy_protocol information on a new outbound connection + void write_outbound_proxy_protocol(); // Called by transact. Updates are fire and forget // so there are no callbacks and are safe to do @@ -263,6 +268,8 @@ class HttpSM : public Continuation, public PluginUserArgs // A NULL 'r' argument indicates the hostdb lookup failed void process_hostdb_info(HostDBRecord *record); void process_srv_info(HostDBRecord *record); + bool origin_multiplexed() const; + bool add_to_existing_request(); // Called by transact. Synchronous. VConnection *do_transform_open(); @@ -288,7 +295,7 @@ class HttpSM : public Continuation, public PluginUserArgs void txn_hook_add(TSHttpHookID id, INKContInternal *cont); APIHook *txn_hook_get(TSHttpHookID id); - bool is_private(); + bool is_private() const; bool is_redirect_required(); /// Get the protocol stack for the inbound (client, user agent) connection. @@ -358,6 +365,7 @@ class HttpSM : public Continuation, public PluginUserArgs int tunnel_handler(int event, void *data); int tunnel_handler_push(int event, void *data); int tunnel_handler_post(int event, void *data); + int tunnel_handler_trailer(int event, void *data); // YTS Team, yamsat Plugin int tunnel_handler_for_partial_post(int event, void *data); @@ -407,6 +415,8 @@ class HttpSM : public Continuation, public PluginUserArgs int tunnel_handler_cache_read(int event, HttpTunnelProducer *p); int tunnel_handler_post_ua(int event, HttpTunnelProducer *c); int tunnel_handler_post_server(int event, HttpTunnelConsumer *c); + int tunnel_handler_trailer_ua(int event, HttpTunnelConsumer *c); + int tunnel_handler_trailer_server(int event, HttpTunnelProducer *c); int tunnel_handler_ssl_producer(int event, HttpTunnelProducer *p); int tunnel_handler_ssl_consumer(int event, HttpTunnelConsumer *p); int tunnel_handler_transform_write(int event, HttpTunnelConsumer *c); @@ -416,7 +426,7 @@ class HttpSM : public Continuation, public PluginUserArgs void do_hostdb_lookup(); void do_hostdb_reverse_lookup(); void do_cache_lookup_and_read(); - void do_http_server_open(bool raw = false); + void do_http_server_open(bool raw = false, bool only_direct = false); void send_origin_throttled_response(); void do_setup_post_tunnel(HttpVC_t to_vc_type); void do_cache_prepare_write(); @@ -450,7 +460,6 @@ class HttpSM : public Continuation, public PluginUserArgs void setup_server_send_request(); void setup_server_send_request_api(); HttpTunnelProducer *setup_server_transfer(); - void setup_server_transfer_to_cache_only(); HttpTunnelProducer *setup_cache_read_transfer(); void setup_internal_transfer(HttpSMHandler handler); void setup_error_transfer(); @@ -617,6 +626,9 @@ class HttpSM : public Continuation, public PluginUserArgs SNIRoutingType _tunnel_type = SNIRoutingType::NONE; PreWarmSM *_prewarm_sm = nullptr; PostDataBuffers _postbuf; + NetVConnection *_netvc = nullptr; + IOBufferReader *_netvc_reader = nullptr; + MIOBuffer *_netvc_read_buffer = nullptr; void kill_this(); void update_stats(); @@ -638,6 +650,9 @@ class HttpSM : public Continuation, public PluginUserArgs ink_hrtime get_server_active_timeout(); ink_hrtime get_server_connect_timeout(); void rewind_state_machine(); + +private: + void cancel_pending_server_connection(); }; //// diff --git a/proxy/http/HttpSessionManager.cc b/proxy/http/HttpSessionManager.cc index d20a1e9e7fe..09aeb8d73e0 100644 --- a/proxy/http/HttpSessionManager.cc +++ b/proxy/http/HttpSessionManager.cc @@ -165,7 +165,9 @@ ServerSessionPool::acquireSession(sockaddr const *addr, CryptoHash const &hostna } if (zret == HSM_DONE) { to_return = first; - this->removeSession(to_return); + if (!to_return->is_multiplexing()) { + this->removeSession(to_return); + } } else if (first != m_fqdn_pool.end()) { Debug("http_ss", "Failed find entry due to name mismatch %s", sm->t_state.current.server->name); } @@ -190,7 +192,9 @@ ServerSessionPool::acquireSession(sockaddr const *addr, CryptoHash const &hostna } if (zret == HSM_DONE) { to_return = first; - this->removeSession(to_return); + if (!to_return->is_multiplexing()) { + this->removeSession(to_return); + } } } return zret; @@ -447,7 +451,10 @@ HttpSessionManager::_acquire_session(sockaddr const *ip, CryptoHash const &hostn } else { Debug("http_ss", "[%" PRId64 "] [acquire session] failed to get transaction on session from shared pool", to_return->connection_id()); - to_return->do_io_close(); + // Don't close the H2 origin. Otherwise you get use-after free with the activity timeout cop + if (!to_return->is_multiplexing()) { + to_return->do_io_close(); + } retval = HSM_RETRY; } } diff --git a/proxy/http/HttpSessionManager.h b/proxy/http/HttpSessionManager.h index 0f10b5d5998..9d7266e191f 100644 --- a/proxy/http/HttpSessionManager.h +++ b/proxy/http/HttpSessionManager.h @@ -67,6 +67,8 @@ class ServerSessionPool : public Continuation static bool validate_host_sni(HttpSM *sm, NetVConnection *netvc); static bool validate_sni(HttpSM *sm, NetVConnection *netvc); static bool validate_cert(HttpSM *sm, NetVConnection *netvc); + void removeSession(PoolableSession *ssn); + void addSession(PoolableSession *ssn); int count() const { @@ -74,9 +76,6 @@ class ServerSessionPool : public Continuation } private: - void removeSession(PoolableSession *ssn); - void addSession(PoolableSession *ssn); - using IPTable = IntrusiveHashMap; using FQDNTable = IntrusiveHashMap; diff --git a/proxy/http/HttpTransact.cc b/proxy/http/HttpTransact.cc index 3c8e933eddd..8be3635c6a4 100644 --- a/proxy/http/HttpTransact.cc +++ b/proxy/http/HttpTransact.cc @@ -3764,7 +3764,8 @@ HttpTransact::handle_response_from_server(State *s) TxnDebug("http_trans", "max_connect_retries: %d s->current.attempts: %d", max_connect_retries, s->current.attempts.get()); - if (is_request_retryable(s) && s->current.attempts.get() < max_connect_retries) { + if (is_request_retryable(s) && s->current.attempts.get() < max_connect_retries && + !HttpTransact::is_response_valid(s, &s->hdr_info.server_response)) { // If this is a round robin DNS entry & we're tried configured // number of times, we should try another node if (ResolveInfo::OS_Addr::TRY_CLIENT == s->dns_info.os_addr_style) { diff --git a/proxy/http/HttpTunnel.cc b/proxy/http/HttpTunnel.cc index 19c65c8427f..1c1c515670b 100644 --- a/proxy/http/HttpTunnel.cc +++ b/proxy/http/HttpTunnel.cc @@ -725,6 +725,7 @@ HttpTunnel::chain(HttpTunnelConsumer *c, HttpTunnelProducer *p) void HttpTunnel::tunnel_run(HttpTunnelProducer *p_arg) { + ++reentrancy_count; Debug("http_tunnel", "tunnel_run started, p_arg is %s", p_arg ? "provided" : "NULL"); if (p_arg) { producer_run(p_arg); @@ -740,6 +741,7 @@ HttpTunnel::tunnel_run(HttpTunnelProducer *p_arg) } } } + --reentrancy_count; // It is possible that there was nothing to do // due to a all transfers being zero length @@ -984,11 +986,14 @@ HttpTunnel::producer_run(HttpTunnelProducer *p) p->handler_state = HTTP_SM_POST_SUCCESS; } } + Debug("http_tunnel", "Start write vio %ld bytes", c_write); // Start the writes now that we know we will consume all the initial data c->write_vio = c->vc->do_io_write(this, c_write, c->buffer_reader); ink_assert(c_write > 0); if (c->write_vio == nullptr) { consumer_handler(VC_EVENT_ERROR, c); + } else if (c->write_vio->ntodo() == 0 && c->alive) { + consumer_handler(VC_EVENT_WRITE_COMPLETE, c); } } } @@ -1008,9 +1013,17 @@ HttpTunnel::producer_run(HttpTunnelProducer *p) if (read_start_pos > 0) { p->read_vio = ((CacheVC *)p->vc)->do_io_pread(this, producer_n, p->read_buffer, read_start_pos); } else { + Debug("http_tunnel", "Start read vio %ld bytes", producer_n); p->read_vio = p->vc->do_io_read(this, producer_n, p->read_buffer); } } + } else { + // If the producer is not alive (precomplete) make sure to kick the consumers + for (c = p->consumer_list.head; c; c = c->link.next) { + if (c->alive && c->write_vio) { + c->write_vio->reenable(); + } + } } // Now that the tunnel has started, we must remove producer's reader so @@ -1136,14 +1149,6 @@ HttpTunnel::producer_handler(int event, HttpTunnelProducer *p) // Handle chunking/dechunking/chunked-passthrough if necessary. if (p->do_chunking) { event = producer_handler_dechunked(event, p); - - // If we were in PRECOMPLETE when this function was called - // and we are doing chunking, then we just wrote the last - // chunk in the function call above. We are done with the - // tunnel. - if (event == HTTP_TUNNEL_EVENT_PRECOMPLETE) { - event = VC_EVENT_EOS; - } } else if (p->do_dechunking || p->do_chunked_passthru) { event = producer_handler_chunked(event, p); } else { @@ -1186,6 +1191,7 @@ HttpTunnel::producer_handler(int event, HttpTunnelProducer *p) // Data read from producer, reenable consumers for (c = p->consumer_list.head; c; c = c->link.next) { if (c->alive && c->write_vio) { + Debug("http_redirect", "Read ready alive"); c->write_vio->reenable(); } } @@ -1195,6 +1201,8 @@ HttpTunnel::producer_handler(int event, HttpTunnelProducer *p) // If the write completes on the stack (as it can for http2), then // consumer could have called back by this point. Must treat this as // a regular read complete (falling through to the following cases). + p->bytes_read = p->init_bytes_done; + [[fallthrough]]; case VC_EVENT_READ_COMPLETE: case VC_EVENT_EOS: @@ -1208,7 +1216,6 @@ HttpTunnel::producer_handler(int event, HttpTunnelProducer *p) // the message length being a property of the encoding) // In that case, we won't have done a do_io so there // will not be vio - p->bytes_read = 0; } // callback the SM to notify of completion @@ -1223,9 +1230,12 @@ HttpTunnel::producer_handler(int event, HttpTunnelProducer *p) sm_callback = true; p->update_state_if_not_set(HTTP_SM_POST_SUCCESS); - // Data read from producer, reenable consumers + // Kick off the consumers if appropriate for (c = p->consumer_list.head; c; c = c->link.next) { if (c->alive && c->write_vio) { + if (c->write_vio->nbytes == INT64_MAX) { + c->write_vio->nbytes = p->bytes_read + p->init_bytes_done - c->skip_bytes; + } c->write_vio->reenable(); } } @@ -1361,6 +1371,9 @@ HttpTunnel::consumer_handler(int event, HttpTunnelConsumer *c) case VC_EVENT_INACTIVITY_TIMEOUT: ink_assert(c->alive); ink_assert(c->buffer_reader); + if (c->write_vio) { + c->write_vio->reenable(); + } c->alive = false; c->bytes_written = c->write_vio ? c->write_vio->ndone : 0; diff --git a/proxy/http/Makefile.am b/proxy/http/Makefile.am index df6d8468a8d..5bbe5b15e2f 100644 --- a/proxy/http/Makefile.am +++ b/proxy/http/Makefile.am @@ -41,6 +41,8 @@ noinst_HEADERS = HttpProxyServerMain.h noinst_LIBRARIES = libhttp.a libhttp_a_SOURCES = \ + ConnectingEntry.cc \ + ConnectingEntry.h \ HttpSessionAccept.cc \ HttpSessionAccept.h \ HttpBodyFactory.cc \ diff --git a/proxy/http2/HTTP2.cc b/proxy/http2/HTTP2.cc index ab81a0484cc..6877d1abcef 100644 --- a/proxy/http2/HTTP2.cc +++ b/proxy/http2/HTTP2.cc @@ -49,11 +49,16 @@ static VersionConverter hvc; // Statistics RecRawStatBlock *http2_rsb; static const char *const HTTP2_STAT_CURRENT_CLIENT_CONNECTION_NAME = "proxy.process.http2.current_client_connections"; +static const char *const HTTP2_STAT_CURRENT_SERVER_CONNECTION_NAME = "proxy.process.http2.current_server_connections"; static const char *const HTTP2_STAT_CURRENT_ACTIVE_CLIENT_CONNECTION_NAME = "proxy.process.http2.current_active_client_connections"; +static const char *const HTTP2_STAT_CURRENT_ACTIVE_SERVER_CONNECTION_NAME = "proxy.process.http2.current_active_server_connections"; static const char *const HTTP2_STAT_CURRENT_CLIENT_STREAM_NAME = "proxy.process.http2.current_client_streams"; +static const char *const HTTP2_STAT_CURRENT_SERVER_STREAM_NAME = "proxy.process.http2.current_server_streams"; static const char *const HTTP2_STAT_TOTAL_CLIENT_STREAM_NAME = "proxy.process.http2.total_client_streams"; +static const char *const HTTP2_STAT_TOTAL_SERVER_STREAM_NAME = "proxy.process.http2.total_server_streams"; static const char *const HTTP2_STAT_TOTAL_TRANSACTIONS_TIME_NAME = "proxy.process.http2.total_transactions_time"; static const char *const HTTP2_STAT_TOTAL_CLIENT_CONNECTION_NAME = "proxy.process.http2.total_client_connections"; +static const char *const HTTP2_STAT_TOTAL_SERVER_CONNECTION_NAME = "proxy.process.http2.total_server_connections"; static const char *const HTTP2_STAT_CONNECTION_ERRORS_NAME = "proxy.process.http2.connection_errors"; static const char *const HTTP2_STAT_STREAM_ERRORS_NAME = "proxy.process.http2.stream_errors"; static const char *const HTTP2_STAT_SESSION_DIE_DEFAULT_NAME = "proxy.process.http2.session_die_default"; @@ -463,14 +468,13 @@ http2_encode_header_blocks(HTTPHdr *in, uint8_t *out, uint32_t out_len, uint32_t */ Http2ErrorCode http2_decode_header_blocks(HTTPHdr *hdr, const uint8_t *buf_start, const uint32_t buf_len, uint32_t *len_read, HpackHandle &handle, - bool &trailing_header, uint32_t maximum_table_size) + bool is_trailing_header, uint32_t maximum_table_size, bool is_outbound) { - const MIMEField *field = nullptr; - const char *name = nullptr; - int name_len = 0; - const char *value = nullptr; - int value_len = 0; - bool is_trailing_header = trailing_header; + const MIMEField *field = nullptr; + const char *name = nullptr; + int name_len = 0; + const char *value = nullptr; + int value_len = 0; int64_t result = hpack_decode_header_block(handle, hdr, buf_start, buf_len, Http2::max_header_list_size, maximum_table_size); if (result < 0) { @@ -487,7 +491,7 @@ http2_decode_header_blocks(HTTPHdr *hdr, const uint8_t *buf_start, const uint32_ } MIMEFieldIter iter; - unsigned int expected_pseudo_header_count = 4; + unsigned int expected_pseudo_header_count = is_outbound ? 1 : 4; unsigned int pseudo_header_count = 0; if (is_trailing_header) { @@ -515,7 +519,6 @@ http2_decode_header_blocks(HTTPHdr *hdr, const uint8_t *buf_start, const uint32_ if (hdr->field_find(MIME_FIELD_CONNECTION, MIME_LEN_CONNECTION) != nullptr || hdr->field_find(MIME_FIELD_KEEP_ALIVE, MIME_LEN_KEEP_ALIVE) != nullptr || hdr->field_find(MIME_FIELD_PROXY_CONNECTION, MIME_LEN_PROXY_CONNECTION) != nullptr || - hdr->field_find(MIME_FIELD_TRANSFER_ENCODING, MIME_LEN_TRANSFER_ENCODING) != nullptr || hdr->field_find(MIME_FIELD_UPGRADE, MIME_LEN_UPGRADE) != nullptr) { return Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR; } @@ -529,13 +532,6 @@ http2_decode_header_blocks(HTTPHdr *hdr, const uint8_t *buf_start, const uint32_ } } - // turn on that we have a trailer header - const char trailer_name[] = "trailer"; - field = hdr->field_find(trailer_name, sizeof(trailer_name) - 1); - if (field) { - trailing_header = true; - } - // when The TE header field is received, it MUST NOT contain any // value other than "trailers". field = hdr->field_find(MIME_FIELD_TE, MIME_LEN_TE); @@ -548,18 +544,29 @@ http2_decode_header_blocks(HTTPHdr *hdr, const uint8_t *buf_start, const uint32_ if (!is_trailing_header) { // Check pseudo headers - if (hdr->fields_count() >= 4) { - if (hdr->field_find(PSEUDO_HEADER_SCHEME.data(), PSEUDO_HEADER_SCHEME.size()) == nullptr || - hdr->field_find(PSEUDO_HEADER_METHOD.data(), PSEUDO_HEADER_METHOD.size()) == nullptr || - hdr->field_find(PSEUDO_HEADER_PATH.data(), PSEUDO_HEADER_PATH.size()) == nullptr || - hdr->field_find(PSEUDO_HEADER_AUTHORITY.data(), PSEUDO_HEADER_AUTHORITY.size()) == nullptr || - hdr->field_find(PSEUDO_HEADER_STATUS.data(), PSEUDO_HEADER_STATUS.size()) != nullptr) { - // Decoded header field is invalid + if (is_outbound) { + if (hdr->fields_count() >= 1) { + if (hdr->field_find(PSEUDO_HEADER_STATUS.data(), PSEUDO_HEADER_STATUS.size()) == nullptr) { + return Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR; + } + } else { + // There should at least be :status pseudo header. return Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR; } } else { - // Pseudo headers is insufficient - return Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR; + if (hdr->fields_count() >= 4) { + if (hdr->field_find(PSEUDO_HEADER_SCHEME.data(), PSEUDO_HEADER_SCHEME.size()) == nullptr || + hdr->field_find(PSEUDO_HEADER_METHOD.data(), PSEUDO_HEADER_METHOD.size()) == nullptr || + hdr->field_find(PSEUDO_HEADER_PATH.data(), PSEUDO_HEADER_PATH.size()) == nullptr || + hdr->field_find(PSEUDO_HEADER_AUTHORITY.data(), PSEUDO_HEADER_AUTHORITY.size()) == nullptr || + hdr->field_find(PSEUDO_HEADER_STATUS.data(), PSEUDO_HEADER_STATUS.size()) != nullptr) { + // Decoded header field is invalid + return Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR; + } + } else { + // Pseudo headers is insufficient + return Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR; + } } } @@ -579,6 +586,7 @@ uint32_t Http2::header_table_size = 4096; uint32_t Http2::max_header_list_size = 4294967295; uint32_t Http2::accept_no_activity_timeout = 120; uint32_t Http2::no_activity_timeout_in = 120; +uint32_t Http2::no_activity_timeout_out = 120; uint32_t Http2::active_timeout_in = 0; uint32_t Http2::push_diary_size = 256; uint32_t Http2::zombie_timeout_in = 0; @@ -620,6 +628,7 @@ Http2::init() REC_EstablishStaticConfigInt32U(max_header_list_size, "proxy.config.http2.max_header_list_size"); REC_EstablishStaticConfigInt32U(accept_no_activity_timeout, "proxy.config.http2.accept_no_activity_timeout"); REC_EstablishStaticConfigInt32U(no_activity_timeout_in, "proxy.config.http2.no_activity_timeout_in"); + REC_EstablishStaticConfigInt32U(no_activity_timeout_out, "proxy.config.http2.no_activity_timeout_out"); REC_EstablishStaticConfigInt32U(active_timeout_in, "proxy.config.http2.active_timeout_in"); REC_EstablishStaticConfigInt32U(push_diary_size, "proxy.config.http2.push_diary_size"); REC_EstablishStaticConfigInt32U(zombie_timeout_in, "proxy.config.http2.zombie_debug_timeout_in"); @@ -658,18 +667,31 @@ Http2::init() RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_CURRENT_CLIENT_CONNECTION_NAME, RECD_INT, RECP_NON_PERSISTENT, static_cast(HTTP2_STAT_CURRENT_CLIENT_SESSION_COUNT), RecRawStatSyncSum); HTTP2_CLEAR_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_SESSION_COUNT); + RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_CURRENT_SERVER_CONNECTION_NAME, RECD_INT, RECP_NON_PERSISTENT, + static_cast(HTTP2_STAT_CURRENT_SERVER_SESSION_COUNT), RecRawStatSyncSum); + HTTP2_CLEAR_DYN_STAT(HTTP2_STAT_CURRENT_SERVER_SESSION_COUNT); RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_CURRENT_ACTIVE_CLIENT_CONNECTION_NAME, RECD_INT, RECP_NON_PERSISTENT, static_cast(HTTP2_STAT_CURRENT_ACTIVE_CLIENT_CONNECTION_COUNT), RecRawStatSyncSum); HTTP2_CLEAR_DYN_STAT(HTTP2_STAT_CURRENT_ACTIVE_CLIENT_CONNECTION_COUNT); + RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_CURRENT_ACTIVE_SERVER_CONNECTION_NAME, RECD_INT, RECP_NON_PERSISTENT, + static_cast(HTTP2_STAT_CURRENT_ACTIVE_SERVER_CONNECTION_COUNT), RecRawStatSyncSum); + HTTP2_CLEAR_DYN_STAT(HTTP2_STAT_CURRENT_ACTIVE_SERVER_CONNECTION_COUNT); RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_CURRENT_CLIENT_STREAM_NAME, RECD_INT, RECP_NON_PERSISTENT, static_cast(HTTP2_STAT_CURRENT_CLIENT_STREAM_COUNT), RecRawStatSyncSum); HTTP2_CLEAR_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_STREAM_COUNT); + RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_CURRENT_SERVER_STREAM_NAME, RECD_INT, RECP_NON_PERSISTENT, + static_cast(HTTP2_STAT_CURRENT_SERVER_STREAM_COUNT), RecRawStatSyncSum); + HTTP2_CLEAR_DYN_STAT(HTTP2_STAT_CURRENT_SERVER_STREAM_COUNT); RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_TOTAL_CLIENT_STREAM_NAME, RECD_INT, RECP_PERSISTENT, static_cast(HTTP2_STAT_TOTAL_CLIENT_STREAM_COUNT), RecRawStatSyncCount); + RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_TOTAL_SERVER_STREAM_NAME, RECD_INT, RECP_PERSISTENT, + static_cast(HTTP2_STAT_TOTAL_SERVER_STREAM_COUNT), RecRawStatSyncCount); RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_TOTAL_TRANSACTIONS_TIME_NAME, RECD_INT, RECP_PERSISTENT, static_cast(HTTP2_STAT_TOTAL_TRANSACTIONS_TIME), RecRawStatSyncSum); RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_TOTAL_CLIENT_CONNECTION_NAME, RECD_INT, RECP_PERSISTENT, static_cast(HTTP2_STAT_TOTAL_CLIENT_CONNECTION_COUNT), RecRawStatSyncSum); + RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_TOTAL_SERVER_CONNECTION_NAME, RECD_INT, RECP_PERSISTENT, + static_cast(HTTP2_STAT_TOTAL_SERVER_CONNECTION_COUNT), RecRawStatSyncSum); RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_CONNECTION_ERRORS_NAME, RECD_INT, RECP_PERSISTENT, static_cast(HTTP2_STAT_CONNECTION_ERRORS_COUNT), RecRawStatSyncSum); RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_STREAM_ERRORS_NAME, RECD_INT, RECP_PERSISTENT, diff --git a/proxy/http2/HTTP2.h b/proxy/http2/HTTP2.h index fdb2d6f1d41..814b10ff38f 100644 --- a/proxy/http2/HTTP2.h +++ b/proxy/http2/HTTP2.h @@ -73,12 +73,17 @@ const uint8_t HTTP2_PRIORITY_DEFAULT_WEIGHT = 15; // Statistics enum { - HTTP2_STAT_CURRENT_CLIENT_SESSION_COUNT, // Current # of HTTP2 connections - HTTP2_STAT_CURRENT_ACTIVE_CLIENT_CONNECTION_COUNT, // Current # of active HTTP2 connections - HTTP2_STAT_CURRENT_CLIENT_STREAM_COUNT, // Current # of active HTTP2 streams + HTTP2_STAT_CURRENT_CLIENT_SESSION_COUNT, // Current # of inbound HTTP2 connections + HTTP2_STAT_CURRENT_SERVER_SESSION_COUNT, // Current # of outbound HTTP2 connections + HTTP2_STAT_CURRENT_ACTIVE_CLIENT_CONNECTION_COUNT, // Current # of active inbound HTTP2 connections + HTTP2_STAT_CURRENT_ACTIVE_SERVER_CONNECTION_COUNT, // Current # of active outbound HTTP2 connections + HTTP2_STAT_CURRENT_CLIENT_STREAM_COUNT, // Current # of active inbound HTTP2 streams + HTTP2_STAT_CURRENT_SERVER_STREAM_COUNT, // Current # of active outboundHTTP2 streams HTTP2_STAT_TOTAL_CLIENT_STREAM_COUNT, + HTTP2_STAT_TOTAL_SERVER_STREAM_COUNT, HTTP2_STAT_TOTAL_TRANSACTIONS_TIME, // Total stream time and streams - HTTP2_STAT_TOTAL_CLIENT_CONNECTION_COUNT, // Total connections running http2 + HTTP2_STAT_TOTAL_CLIENT_CONNECTION_COUNT, // Total inbound connections running http2 + HTTP2_STAT_TOTAL_SERVER_CONNECTION_COUNT, // Total outbound connections running http2 HTTP2_STAT_STREAM_ERRORS_COUNT, HTTP2_STAT_CONNECTION_ERRORS_COUNT, HTTP2_STAT_SESSION_DIE_DEFAULT, @@ -236,8 +241,7 @@ enum Http2SettingsIdentifier { HTTP2_SETTINGS_INITIAL_WINDOW_SIZE = 4, HTTP2_SETTINGS_MAX_FRAME_SIZE = 5, HTTP2_SETTINGS_MAX_HEADER_LIST_SIZE = 6, - - HTTP2_SETTINGS_MAX + HTTP2_SETTINGS_MAX, // Really just the max of the "densely numbered" core id's }; // [RFC 7540] 4.1. Frame Format @@ -353,7 +357,8 @@ bool http2_parse_goaway(IOVec, Http2Goaway &); bool http2_parse_window_update(IOVec, uint32_t &); -Http2ErrorCode http2_decode_header_blocks(HTTPHdr *, const uint8_t *, const uint32_t, uint32_t *, HpackHandle &, bool &, uint32_t); +Http2ErrorCode http2_decode_header_blocks(HTTPHdr *, const uint8_t *, const uint32_t, uint32_t *, HpackHandle &, bool, uint32_t, + bool is_outbound = false); Http2ErrorCode http2_encode_header_blocks(HTTPHdr *, uint8_t *, uint32_t, uint32_t *, HpackHandle &, int32_t); @@ -390,6 +395,7 @@ class Http2 static uint32_t max_header_list_size; static uint32_t accept_no_activity_timeout; static uint32_t no_activity_timeout_in; + static uint32_t no_activity_timeout_out; static uint32_t active_timeout_in; static uint32_t push_diary_size; static uint32_t zombie_timeout_in; diff --git a/proxy/http2/Http2ClientSession.cc b/proxy/http2/Http2ClientSession.cc index 897e7a545db..42300aa2a68 100644 --- a/proxy/http2/Http2ClientSession.cc +++ b/proxy/http2/Http2ClientSession.cc @@ -46,6 +46,10 @@ Http2ClientSession::destroy() in_destroy = true; REMEMBER(NO_EVENT, this->recursion) Http2SsnDebug("session destroy"); + if (_vc) { + _vc->do_io_close(); + _vc = nullptr; + } // Let everyone know we are going down do_api_callout(TS_HTTP_SSN_CLOSE_HOOK); } @@ -54,14 +58,10 @@ Http2ClientSession::destroy() void Http2ClientSession::free() { - if (_vc) { - _vc->do_io_close(); - _vc = nullptr; - } auto mutex_thread = this->mutex->thread_holding; if (Http2CommonSession::common_free(this)) { HTTP2_DECREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_SESSION_COUNT, mutex_thread); - THREAD_FREE(this, http2ClientSessionAllocator, this_ethread()); + THREAD_FREE(this, http2ClientSessionAllocator, mutex_thread); } } @@ -98,7 +98,6 @@ Http2ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOB _vc->set_inactivity_timeout(HRTIME_SECONDS(Http2::accept_no_activity_timeout)); this->schedule_event = nullptr; this->mutex = new_vc->mutex; - this->in_destroy = false; this->connection_state.mutex = this->mutex; @@ -145,17 +144,20 @@ void Http2ClientSession::do_io_close(int alerrno) { REMEMBER(NO_EVENT, this->recursion) - Http2SsnDebug("session closed"); - ink_assert(this->mutex->thread_holding == this_ethread()); - send_connection_event(&this->connection_state, HTTP2_SESSION_EVENT_FINI, this); + if (!this->connection_state.is_state_closed()) { + Http2SsnDebug("session closed"); - this->connection_state.release_stream(); + ink_assert(this->mutex->thread_holding == this_ethread()); + send_connection_event(&this->connection_state, HTTP2_SESSION_EVENT_FINI, this); - this->clear_session_active(); + this->connection_state.release_stream(); - // Clean up the write VIO in case of inactivity timeout - this->do_io_write(this, 0, nullptr); + this->clear_session_active(); + + // Clean up the write VIO in case of inactivity timeout + this->do_io_write(this, 0, nullptr); + } } int @@ -163,6 +165,7 @@ Http2ClientSession::main_event_handler(int event, void *edata) { ink_assert(this->mutex->thread_holding == this_ethread()); int retval; + bool set_closed = false; recursion++; @@ -196,7 +199,8 @@ Http2ClientSession::main_event_handler(int event, void *edata) Http2SsnDebug("Closing event %d", event); this->set_dying_event(event); this->do_io_close(); - retval = 0; + retval = 0; + set_closed = true; break; case VC_EVENT_WRITE_READY: @@ -238,7 +242,7 @@ Http2ClientSession::main_event_handler(int event, void *edata) } } - if (this->connection_state.get_shutdown_state() == HTTP2_SHUTDOWN_NOT_INITIATED) { + if (!set_closed && this->connection_state.get_shutdown_state() == HTTP2_SHUTDOWN_NOT_INITIATED) { send_connection_event(&this->connection_state, HTTP2_SESSION_EVENT_SHUTDOWN_INIT, this); } @@ -279,17 +283,17 @@ Http2ClientSession::get_transact_count() const return connection_state.get_stream_requests(); } -void -Http2ClientSession::release(ProxyTransaction *trans) -{ -} - const char * Http2ClientSession::get_protocol_string() const { return "http/2"; } +void +Http2ClientSession::release(ProxyTransaction *trans) +{ +} + int Http2ClientSession::populate_protocol(std::string_view *result, int size) const { @@ -322,6 +326,15 @@ Http2ClientSession::get_proxy_session() return this; } +void +Http2ClientSession::set_no_activity_timeout() +{ + // Only set if not previously set + if (this->_vc->get_inactivity_timeout() == 0) { + this->set_inactivity_timeout(HRTIME_SECONDS(Http2::no_activity_timeout_in)); + } +} + HTTPVersion Http2ClientSession::get_version(HTTPHdr &hdr) const { diff --git a/proxy/http2/Http2ClientSession.h b/proxy/http2/Http2ClientSession.h index b839d78542c..893feb753a8 100644 --- a/proxy/http2/Http2ClientSession.h +++ b/proxy/http2/Http2ClientSession.h @@ -33,8 +33,7 @@ class Http2ClientSession : public ProxySession, public Http2CommonSession { public: - using super = ProxySession; ///< Parent type. - using SessionHandler = int (Http2ClientSession::*)(int, void *); + using super = ProxySession; ///< Parent type. Http2ClientSession(); @@ -63,6 +62,8 @@ class Http2ClientSession : public ProxySession, public Http2CommonSession void increment_current_active_connections_stat() override; void decrement_current_active_connections_stat() override; + void set_no_activity_timeout() override; + ProxySession *get_proxy_session() override; // noncopyable diff --git a/proxy/http2/Http2CommonSession.cc b/proxy/http2/Http2CommonSession.cc index 11464420553..0484134d203 100644 --- a/proxy/http2/Http2CommonSession.cc +++ b/proxy/http2/Http2CommonSession.cc @@ -91,6 +91,7 @@ Http2CommonSession::common_free(ProxySession *ssn) ink_hrtime_to_msec(this->_milestones[Http2SsnMilestone::OPEN]), this->_milestones.difference_sec(Http2SsnMilestone::OPEN, Http2SsnMilestone::CLOSE)); } + // Update stats on how we died. May want to eliminate this. Was useful for // tracking down which cases we were having problems cleaning up. But for general // use probably not worth the effort @@ -152,7 +153,6 @@ Http2CommonSession::xmit(const Http2TxFrame &frame, bool flush) { int64_t len = frame.write_to(this->write_buffer); this->_pending_sending_data_size += len; - // Force flush for some cases if (!flush) { // Flush if we already use half of the buffer to avoid adding a new block to the chain. // A frame size can be 16MB at maximum so blocks can be added, but that's fine. @@ -160,7 +160,6 @@ Http2CommonSession::xmit(const Http2TxFrame &frame, bool flush) flush = true; } } - if (flush) { this->flush(); } @@ -341,6 +340,8 @@ Http2CommonSession::do_complete_frame_read() int Http2CommonSession::do_process_frame_read(int event, VIO *vio, bool inside_frame) { + Http2SsnDebug("do_process_frame_read %" PRId64 " bytes ready", this->_read_buffer_reader->read_avail()); + if (inside_frame) { do_complete_frame_read(); } @@ -354,7 +355,8 @@ Http2CommonSession::do_process_frame_read(int event, VIO *vio, bool inside_frame } Http2ErrorCode err = Http2ErrorCode::HTTP2_ERROR_NO_ERROR; - if (this->connection_state.get_stream_error_rate() > std::min(1.0, Http2::stream_error_rate_threshold * 2.0)) { + if (this->connection_state.get_stream_error_rate() > std::min(1.0, Http2::stream_error_rate_threshold * 2.0) && + !this->is_outbound()) { ip_port_text_buffer ipb; const char *peer_ip = ats_ip_ntop(this->get_proxy_session()->get_remote_addr(), ipb, sizeof(ipb)); SiteThrottledWarning("HTTP/2 session error peer_ip=%s session_id=%" PRId64 @@ -422,7 +424,12 @@ Http2CommonSession::is_write_high_water() const void Http2CommonSession::write_reenable() { - write_vio->reenable(); + if (write_vio) { + // Grab the lock for the write_vio. Holding the lock is + // checked eventually via the reenable logic + SCOPED_MUTEX_LOCK(lock, write_vio->mutex, this_ethread()); + write_vio->reenable(); + } } void @@ -438,3 +445,14 @@ Http2CommonSession::add_url_to_pushed_table(const char *url, int url_len) _h2_pushed_urls->emplace(url); } } + +void +Http2CommonSession::add_session() +{ +} + +bool +Http2CommonSession::is_outbound() const +{ + return false; +} diff --git a/proxy/http2/Http2CommonSession.h b/proxy/http2/Http2CommonSession.h index 6e046469f47..6eb42c3dd02 100644 --- a/proxy/http2/Http2CommonSession.h +++ b/proxy/http2/Http2CommonSession.h @@ -109,6 +109,11 @@ class Http2CommonSession virtual ProxySession *get_proxy_session() = 0; + virtual void add_session(); + virtual bool is_outbound() const; + + virtual void set_no_activity_timeout() = 0; + /////////////////// // Variables Http2ConnectionState connection_state; @@ -203,7 +208,7 @@ Http2CommonSession::is_url_pushed(const char *url, int url_len) return false; } - return _h2_pushed_urls->find(url) != _h2_pushed_urls->end(); + return _h2_pushed_urls->find(std::string{url, static_cast(url_len)}) != _h2_pushed_urls->end(); } inline int64_t diff --git a/proxy/http2/Http2ConnectionState.cc b/proxy/http2/Http2ConnectionState.cc index 95b3d08fddc..818358dd184 100644 --- a/proxy/http2/Http2ConnectionState.cc +++ b/proxy/http2/Http2ConnectionState.cc @@ -25,6 +25,7 @@ #include "HTTP2.h" #include "Http2ConnectionState.h" #include "Http2ClientSession.h" +#include "Http2ServerSession.h" #include "Http2Stream.h" #include "Http2Frame.h" #include "Http2DebugNames.h" @@ -44,12 +45,10 @@ } \ } -#define Http2ConDebug(session, fmt, ...) \ - SsnDebug(session->get_proxy_session(), "http2_con", "[%" PRId64 "] " fmt, session->get_connection_id(), ##__VA_ARGS__); +#define Http2ConDebug(session, fmt, ...) Debug("http2_con", "[%" PRId64 "] " fmt, session->get_connection_id(), ##__VA_ARGS__); -#define Http2StreamDebug(session, stream_id, fmt, ...) \ - SsnDebug(session->get_proxy_session(), "http2_con", "[%" PRId64 "] [%u] " fmt, session->get_connection_id(), stream_id, \ - ##__VA_ARGS__); +#define Http2StreamDebug(session, stream_id, fmt, ...) \ + Debug("http2_con", "[%" PRId64 "] [%u] " fmt, session->get_connection_id(), stream_id, ##__VA_ARGS__); static const int buffer_size_index[HTTP2_FRAME_TYPE_MAX] = { BUFFER_SIZE_INDEX_16K, // HTTP2_FRAME_TYPE_DATA @@ -105,13 +104,24 @@ Http2ConnectionState::rcv_data_frame(const Http2Frame &frame) if (stream == nullptr) { if (this->is_valid_streamid(id)) { // This error occurs fairly often, and is probably innocuous (SM initiates the shutdown) - return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_STREAM_CLOSED, nullptr); + if (this->session->is_outbound()) { + this->send_rst_stream_frame(id, Http2ErrorCode::HTTP2_ERROR_NO_ERROR); + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); + } else { + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_STREAM_CLOSED, nullptr); + } } else { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR, "recv data stream freed with invalid id"); } } + if (stream->get_state() == Http2StreamState::HTTP2_STREAM_STATE_CLOSED || + stream->get_state() == Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_REMOTE) { + this->send_rst_stream_frame(id, Http2ErrorCode::HTTP2_ERROR_STREAM_CLOSED); + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); + } + // If a DATA frame is received whose stream is not in "open" or "half closed // (local)" state, // the recipient MUST respond with a stream error of type STREAM_CLOSED. @@ -147,24 +157,29 @@ Http2ConnectionState::rcv_data_frame(const Http2Frame &frame) // Pure END_STREAM if (payload_length == 0) { - stream->signal_read_event(VC_EVENT_READ_COMPLETE); + if (stream->read_enabled()) { + stream->signal_read_event(VC_EVENT_READ_COMPLETE); + } return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); } } else { - // If payload length is 0 without END_STREAM flag, do nothing - if (payload_length == 0) { - return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); - } + // Any headers that show up after we received data are by definition trailing headers + stream->set_trailing_header_is_possible(); + } + + // If payload length is 0 without END_STREAM flag, do nothing + if (payload_length == 0 && !stream->receive_end_stream) { + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); } // Check whether Window Size is acceptable if (!this->_local_rwnd_is_shrinking_in && this->get_local_rwnd_in() < payload_length) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_FLOW_CONTROL_ERROR, - "recv data cstate.server_rwnd < payload_length"); + "recv data this->local_rwnd < payload_length"); } if (stream->get_local_rwnd() < payload_length) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_FLOW_CONTROL_ERROR, - "recv data stream->server_rwnd < payload_length"); + "recv data stream->local_rwnd < payload_length"); } // Update Window size @@ -181,7 +196,7 @@ Http2ConnectionState::rcv_data_frame(const Http2Frame &frame) const uint32_t unpadded_length = payload_length - pad_length; MIOBuffer *writer = stream->read_vio_writer(); if (writer == nullptr) { - return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_INTERNAL_ERROR); + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_INTERNAL_ERROR, "no writer"); } // If we call write() multiple times, we must keep the same reader, so we can @@ -201,17 +216,24 @@ Http2ConnectionState::rcv_data_frame(const Http2Frame &frame) unsigned int num_written = writer->write(myreader, read_len); if (num_written != read_len) { myreader->writer()->dealloc_reader(myreader); - return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_INTERNAL_ERROR); + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_INTERNAL_ERROR, "Write mismatch"); } myreader->consume(num_written); + stream->read_update(num_written); } myreader->writer()->dealloc_reader(myreader); if (frame.header().flags & HTTP2_FLAGS_DATA_END_STREAM) { // TODO: set total written size to read_vio.nbytes - stream->signal_read_event(VC_EVENT_READ_COMPLETE); - } else { - stream->signal_read_event(VC_EVENT_READ_READY); + stream->read_done(); + } + + if (stream->read_enabled()) { + if (frame.header().flags & HTTP2_FLAGS_DATA_END_STREAM) { + stream->signal_read_event(VC_EVENT_READ_COMPLETE); + } else { + stream->signal_read_event(VC_EVENT_READ_READY); + } } return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); @@ -239,28 +261,50 @@ Http2ConnectionState::rcv_headers_frame(const Http2Frame &frame) "recv headers bad client id"); } - Http2Stream *stream = nullptr; - bool new_stream = false; + Http2Stream *stream = nullptr; + bool new_stream = false; + bool reset_header_after_decoding = false; + bool free_stream_after_decoding = false; if (this->is_valid_streamid(stream_id)) { stream = this->find_stream(stream_id); - if (stream == nullptr) { - return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_STREAM_CLOSED, - "recv headers cannot find existing stream_id"); - } else if (stream->get_state() == Http2StreamState::HTTP2_STREAM_STATE_CLOSED) { + if (!this->session->is_outbound() && (stream == nullptr || !stream->trailing_header_is_possible())) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_STREAM_CLOSED, - "recv_header to closed stream"); - } else if (!stream->has_trailing_header()) { - return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR, "stream not expecting trailer header"); + } else if (stream == nullptr || stream->get_state() == Http2StreamState::HTTP2_STREAM_STATE_CLOSED) { + if (this->session->is_outbound()) { + reset_header_after_decoding = true; + // return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); + // return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_STREAM_CLOSED, + // "recv_header to closed stream"); + } else { + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_STREAM_CLOSED, + "recv_header to closed stream"); + } } - } else { - // Create new stream - Http2Error error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); - stream = this->create_stream(stream_id, error); - new_stream = true; - if (!stream) { - return error; + } + + if (!http2_is_client_streamid(stream_id)) { + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR, + "recv headers bad client id"); + } + + if (!stream) { + if (reset_header_after_decoding) { + free_stream_after_decoding = true; + uint32_t const initial_local_stream_window = this->acknowledged_local_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE); + ink_assert(dynamic_cast(this->session->get_proxy_session())->is_outbound() == true); + stream = THREAD_ALLOC_INIT(http2StreamAllocator, this_ethread(), this->session->get_proxy_session(), stream_id, + this->peer_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE), initial_local_stream_window, + !STREAM_IS_REGISTERED); + } else { + // Create new stream + Http2Error error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); + stream = this->create_stream(stream_id, error); + new_stream = true; + if (!stream) { + return error; + } } } @@ -352,28 +396,37 @@ Http2ConnectionState::rcv_headers_frame(const Http2Frame &frame) if (frame.header().flags & HTTP2_FLAGS_HEADERS_END_HEADERS) { // NOTE: If there are END_HEADERS flag, decode stored Header Blocks. - if (!stream->change_state(HTTP2_FRAME_TYPE_HEADERS, frame.header().flags) && stream->has_trailing_header() == false) { + if (!stream->change_state(HTTP2_FRAME_TYPE_HEADERS, frame.header().flags)) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR, "recv headers end headers and not trailing header"); } - bool empty_request = false; - if (stream->has_trailing_header()) { + if (stream->trailing_header_is_possible()) { if (!(frame.header().flags & HTTP2_FLAGS_HEADERS_END_STREAM)) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR, "recv headers tailing header without endstream"); } - // If the flag has already been set before decoding header blocks, this is the trailing header. - // Set a flag to avoid initializing fetcher for now. - // Decoding header blocks is still needed to maintain a HPACK dynamic table. - // TODO: TS-3812 - empty_request = true; } - stream->mark_milestone(Http2StreamMilestone::START_DECODE_HEADERS); + if (stream->trailing_header_is_possible()) { + stream->reset_receive_headers(); + } else { + stream->mark_milestone(Http2StreamMilestone::START_DECODE_HEADERS); + } Http2ErrorCode result = stream->decode_header_blocks(*this->local_hpack_handle, this->acknowledged_local_settings.get(HTTP2_SETTINGS_HEADER_TABLE_SIZE)); + // If this was an outbound connection and the state was already closed, just clear the + // headers after processing. We just processed the heaer blocks to keep the dynamic table in + // sync with peer to avoid future HPACK compression errors + if (reset_header_after_decoding) { + stream->reset_receive_headers(); + if (free_stream_after_decoding) { + THREAD_FREE(stream, http2StreamAllocator, this_ethread()); + } + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); + } + if (result != Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { if (result == Http2ErrorCode::HTTP2_ERROR_COMPRESSION_ERROR) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_COMPRESSION_ERROR, @@ -394,15 +447,22 @@ Http2ConnectionState::rcv_headers_frame(const Http2Frame &frame) } // Set up the State Machine - if (!empty_request) { + if (!stream->is_outbound_connection() && !stream->trailing_header_is_possible()) { SCOPED_MUTEX_LOCK(stream_lock, stream->mutex, this_ethread()); stream->mark_milestone(Http2StreamMilestone::START_TXN); stream->new_transaction(frame.is_from_early_data()); // Send request header to SM stream->send_request(*this); } else { - // Signal VC_EVENT_READ_COMPLETE because received trailing header fields with END_STREAM flag - stream->signal_read_event(VC_EVENT_READ_COMPLETE); + // If this is a trailer, first signal to the SM that the body is done + if (stream->trailing_header_is_possible()) { + stream->set_expect_receive_trailer(); + // Propagate the trailer header + stream->send_request(*this); + } else { + // Propagate the response + stream->send_request(*this); + } } } else { // NOTE: Expect CONTINUATION Frame. Do NOT change state of stream or decode @@ -671,8 +731,8 @@ Http2ConnectionState::rcv_settings_frame(const Http2Frame &frame) // [RFC 7540] 6.5. Once all values have been applied, the recipient MUST // immediately emit a SETTINGS frame with the ACK flag set. Http2SettingsFrame ack_frame(HTTP2_CONNECTION_CONTROL_STREAM, HTTP2_FLAGS_SETTINGS_ACK); + Http2StreamDebug(this->session, stream_id, "Send SETTINGS ACK"); this->session->xmit(ack_frame); - return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); } @@ -821,7 +881,6 @@ Http2ConnectionState::rcv_window_update_frame(const Http2Frame &frame) if (error != Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, error, "Erroneous client window update"); } - this->restart_streams(); } else { // Stream level window update @@ -853,11 +912,11 @@ Http2ConnectionState::rcv_window_update_frame(const Http2Frame &frame) auto error = stream->increment_peer_rwnd(size); if (error != Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { - return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, error); + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, error, "Bad stream rwnd"); } ssize_t wnd = std::min(this->get_peer_rwnd_in(), stream->get_peer_rwnd()); - if (!stream->is_closed() && stream->get_state() == Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_REMOTE && wnd > 0) { + if (wnd > 0) { SCOPED_MUTEX_LOCK(lock, stream->mutex, this_ethread()); stream->restart_sending(); } @@ -1084,6 +1143,10 @@ Http2ConnectionState::send_connection_preface() Http2ConnectionSettings configured_settings; configured_settings.settings_from_configs(); + + // Communicate to the peer that we do not support PUSH_PROMISE + configured_settings.set(HTTP2_SETTINGS_ENABLE_PUSH, 0); + configured_settings.set(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, _adjust_concurrent_stream()); if (this->_has_dynamic_stream_window()) { @@ -1283,7 +1346,6 @@ Http2ConnectionState::main_event_handler(int event, void *edata) } } } - return 0; } @@ -1303,6 +1365,101 @@ Http2ConnectionState::state_closed(int event, void *edata) return 0; } +bool +Http2ConnectionState::is_peer_concurrent_stream_ub() const +{ + return peer_streams_count_in >= (peer_settings.get(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS)) * 0.9; +} + +bool +Http2ConnectionState::is_peer_concurrent_stream_lb() const +{ + return peer_streams_count_in <= (peer_settings.get(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS)) / 2; +} + +void +Http2ConnectionState::set_stream_id(Http2Stream *stream) +{ + if (stream->get_transaction_id() < 0) { + Http2StreamId stream_id = (latest_streamid_in == 0) ? 3 : latest_streamid_in + 2; + stream->set_transaction_id(stream_id); + latest_streamid_in = stream_id; + } +} + +Http2Stream * +Http2ConnectionState::create_initiating_stream(Http2Error &error) +{ + // first check if we've hit the active connection limit + if (!session->get_netvc()->add_to_active_queue()) { + error = Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_NO_ERROR, + "refused to create new stream, maxed out active connections"); + return nullptr; + } + + // In half_close state, TS doesn't create new stream. Because GOAWAY frame is sent to client + if (session->get_half_close_local_flag()) { + error = Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_REFUSED_STREAM, + "refused to create new stream, because session is in half_close state"); + return nullptr; + } + + // Endpoints MUST NOT exceed the limit set by their peer. An endpoint + // that receives a HEADERS frame that causes their advertised concurrent + // stream limit to be exceeded MUST treat this as a stream error. + int check_max_concurrent_limit; + int check_count; + check_count = peer_streams_count_in; + // If this is an outbound client stream, must check against the peer's max_concurrent + if (session->is_outbound()) { + check_max_concurrent_limit = peer_settings.get(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS); + } else { // Inbound client streamm check against our own max_connecurent limits + check_max_concurrent_limit = local_settings.get(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS); + } + ink_release_assert(check_max_concurrent_limit != 0); + + // If we haven't got the peers settings yet, just hope for the best + if (check_max_concurrent_limit >= 0) { + if (session->is_outbound() && Http2ConnectionState::is_peer_concurrent_stream_ub()) { + error = Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_REFUSED_STREAM, + "recv headers creating stream beyond max_concurrent limit"); + return nullptr; + } else if (check_count >= check_max_concurrent_limit) { + error = Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_REFUSED_STREAM, + "recv headers creating stream beyond max_concurrent limit"); + return nullptr; + } + } + + ink_assert(dynamic_cast(this->session->get_proxy_session())->is_outbound() == true); + uint32_t const initial_stream_window = this->acknowledged_local_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE); + Http2Stream *new_stream = + THREAD_ALLOC_INIT(http2StreamAllocator, this_ethread(), session->get_proxy_session(), -1, + peer_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE), initial_stream_window, STREAM_IS_REGISTERED); + + ink_assert(nullptr != new_stream); + ink_assert(!stream_list.in(new_stream)); + + stream_list.enqueue(new_stream); + ink_assert(peer_streams_count_in < UINT32_MAX); + ++peer_streams_count_in; + ++total_peer_streams_count; + + if (zombie_event != nullptr) { + zombie_event->cancel(); + zombie_event = nullptr; + } + + new_stream->mutex = new_ProxyMutex(); + new_stream->is_first_transaction_flag = get_stream_requests() == 0; + increment_stream_requests(); + + // Clear the session timeout. Let the transaction timeouts reign + session->get_proxy_session()->cancel_inactivity_timeout(); + + return new_stream; +} + Http2Stream * Http2ConnectionState::create_stream(Http2StreamId new_id, Http2Error &error) { @@ -1345,22 +1502,37 @@ Http2ConnectionState::create_stream(Http2StreamId new_id, Http2Error &error) // Endpoints MUST NOT exceed the limit set by their peer. An endpoint // that receives a HEADERS frame that causes their advertised concurrent // stream limit to be exceeded MUST treat this as a stream error. + int check_max_concurrent_limit = 0; + int check_count = 0; + int max_streams_stat = 0; if (is_client_streamid) { - if (peer_streams_count_in >= acknowledged_local_settings.get(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS)) { - HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_MAX_CONCURRENT_STREAMS_EXCEEDED_IN, this_ethread()); - error = Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_REFUSED_STREAM, - "recv headers creating inbound stream beyond max_concurrent limit"); - return nullptr; - } - } else { - if (peer_streams_count_out >= peer_settings.get(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS)) { - HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_MAX_CONCURRENT_STREAMS_EXCEEDED_OUT, this_ethread()); - error = Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_REFUSED_STREAM, - "recv headers creating outbound stream beyond max_concurrent limit"); - return nullptr; - } + check_count = peer_streams_count_in; + max_streams_stat = HTTP2_STAT_MAX_CONCURRENT_STREAMS_EXCEEDED_IN; + // If this is an outbound client stream, must check against the peer's max_concurrent + if (session->is_outbound()) { + check_max_concurrent_limit = peer_settings.get(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS); + } else { // Inbound client streamm check against our own max_connecurent limits + check_max_concurrent_limit = acknowledged_local_settings.get(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS); + } + } else { // Not a client stream (i.e. a push) + check_count = peer_streams_count_out; + max_streams_stat = HTTP2_STAT_MAX_CONCURRENT_STREAMS_EXCEEDED_OUT; + // If this is an outbound non-client stream, must check against the local max_concurrent + if (session->is_outbound()) { + check_max_concurrent_limit = acknowledged_local_settings.get(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS); + } else { // Inbound non-client streamm check against the peer's max_connecurent limits + check_max_concurrent_limit = peer_settings.get(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS); + } + } + // If we haven't got the peers settings yet, just hope for the best + if (check_max_concurrent_limit >= 0 && check_count >= check_max_concurrent_limit) { + HTTP2_INCREMENT_THREAD_DYN_STAT(max_streams_stat, this_ethread()); + error = Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_REFUSED_STREAM, + "recv headers creating stream beyond max_concurrent limit"); + return nullptr; } + ink_assert(dynamic_cast(this->session->get_proxy_session())->is_outbound() == false); uint32_t initial_stream_window = this->acknowledged_local_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE); uint32_t initial_stream_window_target = initial_stream_window; if (is_client_streamid && this->_has_dynamic_stream_window()) { @@ -1377,8 +1549,9 @@ Http2ConnectionState::create_stream(Http2StreamId new_id, Http2Error &error) // 6.9.3. initial_stream_window_target = this->_get_configured_receive_session_window_size_in() / (peer_streams_count_in.load() + 1); } - Http2Stream *new_stream = THREAD_ALLOC_INIT(http2StreamAllocator, this_ethread(), session->get_proxy_session(), new_id, - peer_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE), initial_stream_window); + Http2Stream *new_stream = + THREAD_ALLOC_INIT(http2StreamAllocator, this_ethread(), session->get_proxy_session(), new_id, + peer_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE), initial_stream_window, STREAM_IS_REGISTERED); ink_assert(nullptr != new_stream); ink_assert(!stream_list.in(new_stream)); @@ -1409,6 +1582,9 @@ Http2ConnectionState::create_stream(Http2StreamId new_id, Http2Error &error) } increment_stream_requests(); + // Clear the session timeout. Let the transaction timeouts reign + session->get_proxy_session()->cancel_inactivity_timeout(); + return new_stream; } @@ -1424,6 +1600,17 @@ Http2ConnectionState::find_stream(Http2StreamId id) const return nullptr; } +void +Http2ConnectionState::start_streams() +{ + Http2Stream *s = stream_list.head; + while (s) { + Http2Stream *next = static_cast(s->link.next); + s->reenable_write(); + s = next; + } +} + void Http2ConnectionState::restart_streams() { @@ -1435,7 +1622,6 @@ Http2ConnectionState::restart_streams() // It doesn't need to be initialized with rand() nor time(), and doesn't need to be accessed with a lock, because it doesn't // need that randomness and accuracy. static uint16_t starting_point = 0; - // Change the start point randomly for (int i = starting_point % total_peer_streams_count; i >= 0; --i) { end = static_cast(end->link.next ? end->link.next : stream_list.head); @@ -1445,16 +1631,14 @@ Http2ConnectionState::restart_streams() // Call send_response_body() for each streams while (s != end) { Http2Stream *next = static_cast(s->link.next ? s->link.next : stream_list.head); - if (!s->is_closed() && s->get_state() == Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_REMOTE && - std::min(this->get_peer_rwnd_in(), s->get_peer_rwnd()) > 0) { + if (std::min(this->get_peer_rwnd_in(), s->get_peer_rwnd()) > 0) { SCOPED_MUTEX_LOCK(lock, s->mutex, this_ethread()); s->restart_sending(); } ink_assert(s != next); s = next; } - if (!s->is_closed() && s->get_state() == Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_REMOTE && - std::min(this->get_peer_rwnd_in(), s->get_peer_rwnd()) > 0) { + if (std::min(this->get_peer_rwnd_in(), s->get_peer_rwnd()) > 0) { SCOPED_MUTEX_LOCK(lock, s->mutex, this_ethread()); s->restart_sending(); } @@ -1555,7 +1739,9 @@ Http2ConnectionState::delete_stream(Http2Stream *stream) REMEMBER(NO_EVENT, this->recursion); if (Http2::stream_priority_enabled) { - Http2DependencyTree::Node *node = stream->priority_node; + Http2DependencyTree::Node *node = stream->priority_node; + Http2DependencyTree::Node *node_by_id = this->dependency_tree->find(stream->get_id()); + ink_assert(node == node_by_id); if (node != nullptr) { if (node->active) { dependency_tree->deactivate(node, 0); @@ -1577,13 +1763,16 @@ Http2ConnectionState::delete_stream(Http2Stream *stream) stream_list.remove(stream); if (http2_is_client_streamid(stream->get_id())) { - ink_assert(peer_streams_count_in > 0); + ink_release_assert(peer_streams_count_in > 0); --peer_streams_count_in; + if (!fini_received && is_peer_concurrent_stream_lb()) { + session->add_session(); + } } else { ink_assert(peer_streams_count_out > 0); --peer_streams_count_out; } - // total_client_streams_count will be decremented in release_stream(), because it's a counter include streams in the process of + // total_peer_streams_count will be decremented in release_stream(), because it's a counter include streams in the process of // shutting down. stream->initiating_close(); @@ -1616,6 +1805,7 @@ Http2ConnectionState::release_stream() // If the number of clients is 0, HTTP2_SESSION_EVENT_FINI is not received or sent, and session is active, // then mark the connection as inactive session->do_clear_session_active(); + session->set_no_activity_timeout(); UnixNetVConnection *vc = static_cast(session->get_netvc()); if (vc && vc->active_timeout_in == 0) { // With heavy traffic, session could be destroyed. Do not touch session after this. @@ -1703,10 +1893,12 @@ Http2ConnectionState::send_data_frames_depends_on_priority() Http2Stream *stream = static_cast(node->t); ink_release_assert(stream != nullptr); + ink_release_assert(stream->priority_node == node); Http2StreamDebug(session, stream->get_id(), "top node, point=%d", node->point); size_t len = 0; Http2SendDataFrameResult result = send_a_data_frame(stream, len); + ink_release_assert(stream->priority_node != nullptr); switch (result) { case Http2SendDataFrameResult::NO_ERROR: { @@ -1715,9 +1907,8 @@ Http2ConnectionState::send_data_frames_depends_on_priority() dependency_tree->deactivate(node, len); } else { dependency_tree->update(node, len); - SCOPED_MUTEX_LOCK(stream_lock, stream->mutex, this_ethread()); - stream->signal_write_event(Http2Stream::CALL_UPDATE); + stream->signal_write_event(stream->is_write_vio_done() ? VC_EVENT_WRITE_COMPLETE : VC_EVENT_WRITE_READY); } break; } @@ -1758,6 +1949,12 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len if (resp_reader->is_read_avail_more_than(0)) { // We only need to check for window size when there is a payload if (window_size <= 0) { + if (session->is_outbound()) { + ip_port_text_buffer ipb; + const char *client_ip = ats_ip_ntop(session->get_proxy_session()->get_remote_addr(), ipb, sizeof(ipb)); + Warning("No window server_ip=%s session_wnd=%zd stream_wnd=%zd peer_initial_window=%u", client_ip, get_peer_rwnd_in(), + stream->get_peer_rwnd(), this->peer_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE)); + } Http2StreamDebug(this->session, stream->get_id(), "No window"); this->session->flush(); return Http2SendDataFrameResult::NO_WINDOW; @@ -1790,6 +1987,7 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len } if (stream->is_write_vio_done()) { + Http2StreamDebug(this->session, stream->get_id(), "End of Data Frame"); flags |= HTTP2_FLAGS_DATA_END_STREAM; } @@ -1798,8 +1996,8 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len stream->decrement_peer_rwnd(payload_length); // Create frame - Http2StreamDebug(session, stream->get_id(), "Send a DATA frame - client window con: %5zd stream: %5zd payload: %5zd", - _peer_rwnd_in, stream->get_peer_rwnd(), payload_length); + Http2StreamDebug(session, stream->get_id(), "Send a DATA frame - client window con: %5zd stream: %5zd payload: %5zd flags: 0x%x", + _peer_rwnd_in, stream->get_peer_rwnd(), payload_length, flags); Http2DataFrame data(stream->get_id(), flags, resp_reader, payload_length); this->session->xmit(data, flags & HTTP2_FLAGS_DATA_END_STREAM); @@ -1828,20 +2026,38 @@ Http2ConnectionState::send_data_frames(Http2Stream *stream) return; } + if (zombie_event != nullptr) { + zombie_event->cancel(); + zombie_event = nullptr; + } + size_t len = 0; Http2SendDataFrameResult result = Http2SendDataFrameResult::NO_ERROR; - while (result == Http2SendDataFrameResult::NO_ERROR) { - result = send_a_data_frame(stream, len); + bool more_data = true; + IOBufferReader *resp_reader = stream->get_data_reader_for_send(); + while (more_data && result == Http2SendDataFrameResult::NO_ERROR) { + result = send_a_data_frame(stream, len); + more_data = resp_reader->is_read_avail_more_than(0); if (result == Http2SendDataFrameResult::DONE) { - // Delete a stream immediately - // TODO its should not be deleted for a several time to handling - // RST_STREAM and WINDOW_UPDATE. - // See 'closed' state written at [RFC 7540] 5.1. - Http2StreamDebug(this->session, stream->get_id(), "Shutdown stream"); - stream->initiating_close(); + if (!stream->is_outbound_connection()) { + // Delete a stream immediately + // TODO its should not be deleted for a several time to handling + // RST_STREAM and WINDOW_UPDATE. + // See 'closed' state written at [RFC 7540] 5.1. + Http2StreamDebug(this->session, stream->get_id(), "Shutdown stream"); + stream->signal_write_event(VC_EVENT_WRITE_COMPLETE); + stream->do_io_close(); + } else if (stream->is_outbound_connection() && stream->is_write_vio_done()) { + stream->signal_write_event(VC_EVENT_WRITE_COMPLETE); + } else { + ink_release_assert(!"What case is this?"); + } } } + if (!more_data && result != Http2SendDataFrameResult::DONE) { + stream->signal_write_event(VC_EVENT_WRITE_READY); + } return; } @@ -1855,15 +2071,25 @@ Http2ConnectionState::send_headers_frame(Http2Stream *stream) Http2StreamDebug(session, stream->get_id(), "Send HEADERS frame"); - HTTPHdr *resp_hdr = &stream->_send_header; - http2_convert_header_from_1_1_to_2(resp_hdr); + // For outbound streams, set the ID if it has not yet already been set + // Need to defer setting the stream ID to avoid another later created stream + // sending out first. This may cause the peer to issue a stream or connection + // error (new stream less that the greatest we have seen so far) + this->set_stream_id(stream); + + HTTPHdr *send_hdr = stream->get_send_header(); + if (stream->expect_send_trailer()) { + // Which is a no-op conversion + } else { + http2_convert_header_from_1_1_to_2(send_hdr); + } - uint32_t buf_len = resp_hdr->length_get() * 2; // Make it double just in case + uint32_t buf_len = send_hdr->length_get() * 2; // Make it double just in case ts::LocalBuffer local_buffer(buf_len); uint8_t *buf = local_buffer.data(); stream->mark_milestone(Http2StreamMilestone::START_ENCODE_HEADERS); - Http2ErrorCode result = http2_encode_header_blocks(resp_hdr, buf, buf_len, &header_blocks_size, *(this->peer_hpack_handle), + Http2ErrorCode result = http2_encode_header_blocks(send_hdr, buf, buf_len, &header_blocks_size, *(this->peer_hpack_handle), peer_settings.get(HTTP2_SETTINGS_HEADER_TABLE_SIZE)); if (result != Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { return; @@ -1873,11 +2099,37 @@ Http2ConnectionState::send_headers_frame(Http2Stream *stream) if (header_blocks_size <= static_cast(BUFFER_SIZE_FOR_INDEX(buffer_size_index[HTTP2_FRAME_TYPE_HEADERS]))) { payload_length = header_blocks_size; flags |= HTTP2_FLAGS_HEADERS_END_HEADERS; - if ((resp_hdr->presence(MIME_PRESENCE_CONTENT_LENGTH) && resp_hdr->get_content_length() == 0) || - (!resp_hdr->expect_final_response() && stream->is_write_vio_done())) { - Http2StreamDebug(session, stream->get_id(), "END_STREAM"); - flags |= HTTP2_FLAGS_HEADERS_END_STREAM; - stream->send_end_stream = true; + if (stream->is_outbound_connection()) { // Will be sending a request_header + int method = send_hdr->method_get_wksidx(); + + // Set END_STREAM on request headers for POST, etc. methods combined with + // an explicit length 0. Some origins RST on request headers with + // explicit zero length and no end stream flag, causing the request to + // fail. We emulate chromium behaviour here prevent such RSTs. + bool content_method = method == HTTP_WKSIDX_POST || method == HTTP_WKSIDX_PUSH || method == HTTP_WKSIDX_PUT; + bool is_transfer_encoded = send_hdr->presence(MIME_PRESENCE_TRANSFER_ENCODING); + bool has_content_header = send_hdr->presence(MIME_PRESENCE_CONTENT_LENGTH); + bool explicit_zero_length = has_content_header && send_hdr->get_content_length() == 0; + + bool expect_content_stream = + is_transfer_encoded || // transfer encoded content length is unknown + (!content_method && has_content_header && !explicit_zero_length) || // non zero content with GET,etc + (content_method && !explicit_zero_length); // content-length >0 or empty with POST etc + + // send END_STREAM if we don't expect any content + if (!expect_content_stream) { + // TODO deal with the chunked encoding case + Http2StreamDebug(session, stream->get_id(), "request END_STREAM"); + flags |= HTTP2_FLAGS_HEADERS_END_STREAM; + stream->send_end_stream = true; + } + } else { + if ((send_hdr->presence(MIME_PRESENCE_CONTENT_LENGTH) && send_hdr->get_content_length() == 0) || + (!send_hdr->expect_final_response() && stream->is_write_vio_done())) { + Http2StreamDebug(session, stream->get_id(), "response END_STREAM"); + flags |= HTTP2_FLAGS_HEADERS_END_STREAM; + stream->send_end_stream = true; + } } stream->mark_milestone(Http2StreamMilestone::START_TX_HEADERS_FRAMES); } else { @@ -1895,6 +2147,7 @@ Http2ConnectionState::send_headers_frame(Http2Stream *stream) return; } + Http2StreamDebug(session, stream->get_id(), "Send HEADERS frame flags: 0x%x length: %d", flags, payload_length); Http2HeadersFrame headers(stream->get_id(), flags, buf, payload_length); this->session->xmit(headers); uint64_t sent = payload_length; @@ -2079,7 +2332,7 @@ Http2ConnectionState::send_settings_frame(const Http2ConnectionSettings &new_set Http2SettingsFrame settings(stream_id, HTTP2_FRAME_NO_FLAG, params, params_size); this->_outstanding_settings_frames_in.emplace(new_settings); - this->session->xmit(settings); + this->session->xmit(settings, true); } void @@ -2090,7 +2343,7 @@ Http2ConnectionState::_process_incoming_settings_ack_frame() this->_outstanding_settings_frames_in.size()); // Do not update this->acknowledged_local_settings yet as - // update_initial_server_rwnd relies upon it still pointing to the old value. + // update_initial_local_rwnd_in relies upon it still pointing to the old value. Http2ConnectionSettings const &old_settings = this->acknowledged_local_settings; Http2ConnectionSettings const &new_settings = this->_outstanding_settings_frames_in.front().get_outstanding_settings(); @@ -2285,12 +2538,13 @@ Http2ConnectionState::increment_peer_rwnd_in(size_t amount) this->_recent_rwnd_increment[this->_recent_rwnd_increment_index] = amount; ++this->_recent_rwnd_increment_index; this->_recent_rwnd_increment_index %= this->_recent_rwnd_increment.size(); - double sum = std::accumulate(this->_recent_rwnd_increment.begin(), this->_recent_rwnd_increment.end(), 0.0); - double avg = sum / this->_recent_rwnd_increment.size(); - if (avg < Http2::min_avg_window_update) { - HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_INSUFFICIENT_AVG_WINDOW_UPDATE, this_ethread()); - return Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM; - } + // SKH Causing problems with gRPC processing. Python example resulted in amount 8 + // double sum = std::accumulate(this->_recent_rwnd_increment.begin(), this->_recent_rwnd_increment.end(), 0.0); + // double avg = sum / this->_recent_rwnd_increment.size(); + // if (avg < Http2::min_avg_window_update) { + // HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_INSUFFICIENT_AVG_WINDOW_UPDATE, this_ethread()); + // return Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM; + //} return Http2ErrorCode::HTTP2_ERROR_NO_ERROR; } diff --git a/proxy/http2/Http2ConnectionState.h b/proxy/http2/Http2ConnectionState.h index 7c70b5a275a..0784d4cd3e9 100644 --- a/proxy/http2/Http2ConnectionState.h +++ b/proxy/http2/Http2ConnectionState.h @@ -122,8 +122,11 @@ class Http2ConnectionState : public Continuation // Stream control interfaces Http2Stream *create_stream(Http2StreamId new_id, Http2Error &error); + Http2Stream *create_initiating_stream(Http2Error &error); + void set_stream_id(Http2Stream *stream); Http2Stream *find_stream(Http2StreamId id) const; void restart_streams(); + void start_streams(); bool delete_stream(Http2Stream *stream); void release_stream(); void cleanup_streams(); @@ -139,6 +142,8 @@ class Http2ConnectionState : public Continuation Http2StreamId get_latest_stream_id_out() const; int get_stream_requests() const; void increment_stream_requests(); + bool is_peer_concurrent_stream_ub() const; + bool is_peer_concurrent_stream_lb() const; // Continuated header decoding Http2StreamId get_continued_stream_id() const; @@ -198,6 +203,9 @@ class Http2ConnectionState : public Continuation Http2ErrorCode increment_local_rwnd_in(size_t amount); Http2ErrorCode decrement_local_rwnd_in(size_t amount); + bool no_streams() const; + bool single_stream() const; + private: Http2Error rcv_data_frame(const Http2Frame &); Http2Error rcv_headers_frame(const Http2Frame &); diff --git a/proxy/http2/Http2ServerSession.cc b/proxy/http2/Http2ServerSession.cc new file mode 100644 index 00000000000..42bb4e8e1e9 --- /dev/null +++ b/proxy/http2/Http2ServerSession.cc @@ -0,0 +1,418 @@ +/** @file + + Http2ServerSession. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "Http2ServerSession.h" +#include "HttpDebugNames.h" +#include "tscore/ink_base64.h" +#include "Http2CommonSessionInternal.h" +#include "HttpSessionManager.h" + +ClassAllocator http2ServerSessionAllocator("http2ServerSessionAllocator"); + +static int +send_connection_event(Continuation *cont, int event, void *edata) +{ + SCOPED_MUTEX_LOCK(lock, cont->mutex, this_ethread()); + return cont->handleEvent(event, edata); +} + +Http2ServerSession::Http2ServerSession() = default; + +void +Http2ServerSession::destroy() +{ + if (!in_destroy) { + in_destroy = true; + write_vio = nullptr; + this->remove_session(); + this->release_outbound_connection_tracking(); + REMEMBER(NO_EVENT, this->recursion) + Http2SsnDebug("session destroy"); + if (_vc) { + _vc->do_io_close(); + _vc = nullptr; + } + free(); + } +} + +void +Http2ServerSession::free() +{ + auto mutex_thread = this->mutex->thread_holding; + if (Http2CommonSession::common_free(this)) { + HTTP2_DECREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_SERVER_SESSION_COUNT, mutex_thread); + THREAD_FREE(this, http2ServerSessionAllocator, mutex_thread); + } +} + +void +Http2ServerSession::start() +{ + SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); + + SET_HANDLER(&Http2ServerSession::main_event_handler); + HTTP2_SET_SESSION_HANDLER(&Http2ServerSession::state_start_frame_read); + + VIO *read_vio = this->do_io_read(this, INT64_MAX, this->read_buffer); + write_vio = this->do_io_write(this, INT64_MAX, this->_write_buffer_reader); + + this->connection_state.init(this); + + // 3.5 HTTP/2 Connection Preface. Upon establishment of a TCP connection and + // determination that HTTP/2 will be used by both peers, each endpoint MUST + // send a connection preface as a final confirmation ... + // This is the preface string sent by the client + this->write_buffer->write(HTTP2_CONNECTION_PREFACE, HTTP2_CONNECTION_PREFACE_LEN); + write_reenable(); + this->connection_state.send_connection_preface(); + Http2SsnDebug("Sent Connection Preface"); + + this->handleEvent(VC_EVENT_READ_READY, read_vio); +} + +void +Http2ServerSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader) +{ + ink_assert(new_vc->mutex->thread_holding == this_ethread()); + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_SERVER_SESSION_COUNT, new_vc->mutex->thread_holding); + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_TOTAL_SERVER_CONNECTION_COUNT, new_vc->mutex->thread_holding); + this->_milestones.mark(Http2SsnMilestone::OPEN); + + // Unique client session identifier. + this->con_id = ProxySession::next_connection_id(); + this->_vc = new_vc; + _vc->set_inactivity_timeout(HRTIME_SECONDS(Http2::accept_no_activity_timeout)); + this->schedule_event = nullptr; + this->mutex = new_vc->mutex; + + this->connection_state.mutex = this->mutex; + + // Since we're functioning as a client, we do not need to worry about + // TLSEarlyDataSupport. + + Http2SsnDebug("session born, netvc %p", this->_vc); + + this->_vc->set_tcp_congestion_control(CLIENT_SIDE); + + this->read_buffer = iobuf ? iobuf : new_MIOBuffer(HTTP2_HEADER_BUFFER_SIZE_INDEX); + this->read_buffer->water_mark = connection_state.local_settings.get(HTTP2_SETTINGS_MAX_FRAME_SIZE); + this->_read_buffer_reader = reader ? reader : this->read_buffer->alloc_reader(); + + // Set write buffer size to max size of TLS record (16KB) + // This block size is the buffer size that we pass to SSLWriteBuffer + auto buffer_block_size_index = iobuffer_size_to_index(Http2::write_buffer_block_size, MAX_BUFFER_SIZE_INDEX); + this->write_buffer = new_MIOBuffer(buffer_block_size_index); + this->_write_buffer_reader = this->write_buffer->alloc_reader(); + this->_write_size_threshold = index_to_buffer_size(buffer_block_size_index) * Http2::write_size_threshold; + + this->_handle_if_ssl(new_vc); + + do_api_callout(TS_HTTP_SSN_START_HOOK); + + this->add_session(); +} + +// implement that. After we send a GOAWAY, there +// are scenarios where we would like to complete the outstanding streams. + +void +Http2ServerSession::do_io_close(int alerrno) +{ + REMEMBER(NO_EVENT, this->recursion) + + if (!this->connection_state.is_state_closed()) { + Http2SsnDebug("session closed"); + this->remove_session(); + + ink_assert(this->mutex->thread_holding == this_ethread()); + send_connection_event(&this->connection_state, HTTP2_SESSION_EVENT_FINI, this); + + // Destroy will be called from connection_state.release_stream() once the number of active streams goes to 0 + } +} + +int +Http2ServerSession::main_event_handler(int event, void *edata) +{ + ink_assert(this->mutex->thread_holding == this_ethread()); + int retval; + + recursion++; + + Event *e = static_cast(edata); + if (e == schedule_event) { + schedule_event = nullptr; + } + + Http2SsnDebug("main_event_handler=%d edata=%p", event, edata); + + switch (event) { + case VC_EVENT_READ_COMPLETE: + case VC_EVENT_READ_READY: { + bool is_zombie = connection_state.get_zombie_event() != nullptr; + retval = (this->*session_handler)(event, edata); + if (is_zombie && connection_state.get_zombie_event() != nullptr) { + Warning("Processed read event for zombie session %" PRId64, connection_id()); + } + break; + } + + case HTTP2_SESSION_EVENT_REENABLE: + // VIO will be reenableed in this handler + retval = (this->*session_handler)(VC_EVENT_READ_READY, static_cast(e->cookie)); + // Clear the event after calling session_handler to not reschedule REENABLE in it + this->_reenable_event = nullptr; + break; + + case VC_EVENT_ACTIVE_TIMEOUT: + case VC_EVENT_INACTIVITY_TIMEOUT: + case VC_EVENT_ERROR: + case VC_EVENT_EOS: + this->set_dying_event(event); + this->do_io_close(); + retval = 0; + break; + + case VC_EVENT_WRITE_READY: + case VC_EVENT_WRITE_COMPLETE: + this->connection_state.restart_streams(); + if ((Thread::get_hrtime() >= this->_write_buffer_last_flush + HRTIME_MSECONDS(this->_write_time_threshold))) { + this->flush(); + } + + retval = 0; + break; + + case HTTP2_SESSION_EVENT_XMIT: + default: + Http2SsnDebug("unexpected event=%d edata=%p", event, edata); + ink_release_assert(0); + retval = 0; + break; + } + + if (!this->is_draining() && this->connection_state.get_shutdown_reason() == Http2ErrorCode::HTTP2_ERROR_MAX) { + this->connection_state.set_shutdown_state(HTTP2_SHUTDOWN_NONE); + } + + if (this->connection_state.get_shutdown_state() == HTTP2_SHUTDOWN_NONE) { + if (this->is_draining()) { // For a case we already checked Connection header and it didn't exist + Http2SsnDebug("Preparing for graceful shutdown because of draining state"); + this->connection_state.set_shutdown_state(HTTP2_SHUTDOWN_NOT_INITIATED); + } /*else if (this->connection_state.get_stream_error_rate() > + Http2::stream_error_rate_threshold) { // For a case many stream errors happened + ip_port_text_buffer ipb; + const char *client_ip = ats_ip_ntop(get_remote_addr(), ipb, sizeof(ipb)); + SiteThrottledWarning("HTTP/2 session error origin_ip=%s session_id=%" PRId64 + " closing a connection, because its stream error rate (%f) exceeded the threshold (%f)", + client_ip, connection_id(), this->connection_state.get_stream_error_rate(), Http2::stream_error_rate_threshold); + Http2SsnDebug("Preparing for graceful shutdown because of a high stream error rate"); + cause_of_death = Http2SessionCod::HIGH_ERROR_RATE; + this->connection_state.set_shutdown_state(HTTP2_SHUTDOWN_NOT_INITIATED, Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM); + } */ + } + + if (this->connection_state.get_shutdown_state() == HTTP2_SHUTDOWN_NOT_INITIATED) { + send_connection_event(&this->connection_state, HTTP2_SESSION_EVENT_SHUTDOWN_INIT, this); + } + + recursion--; + if (!connection_state.is_recursing() && this->recursion == 0 && kill_me) { + this->free(); + } + return retval; +} + +void +Http2ServerSession::increment_current_active_connections_stat() +{ + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_ACTIVE_SERVER_CONNECTION_COUNT, this_ethread()); +} + +void +Http2ServerSession::decrement_current_active_connections_stat() +{ + HTTP2_DECREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_ACTIVE_SERVER_CONNECTION_COUNT, this_ethread()); +} + +sockaddr const * +Http2ServerSession::get_remote_addr() const +{ + return _vc ? _vc->get_remote_addr() : &cached_client_addr.sa; +} + +sockaddr const * +Http2ServerSession::get_local_addr() +{ + return _vc ? _vc->get_local_addr() : &cached_local_addr.sa; +} + +int +Http2ServerSession::get_transact_count() const +{ + return connection_state.get_stream_requests(); +} + +const char * +Http2ServerSession::get_protocol_string() const +{ + return "http/2"; +} + +void +Http2ServerSession::release(ProxyTransaction *trans) +{ +} + +int +Http2ServerSession::populate_protocol(std::string_view *result, int size) const +{ + int retval = 0; + if (size > retval) { + result[retval++] = IP_PROTO_TAG_HTTP_2_0; + if (size > retval) { + retval += super::populate_protocol(result + retval, size - retval); + } + } + return retval; +} + +const char * +Http2ServerSession::protocol_contains(std::string_view prefix) const +{ + const char *retval = nullptr; + + if (prefix.size() <= IP_PROTO_TAG_HTTP_2_0.size() && strncmp(IP_PROTO_TAG_HTTP_2_0.data(), prefix.data(), prefix.size()) == 0) { + retval = IP_PROTO_TAG_HTTP_2_0.data(); + } else { + retval = super::protocol_contains(prefix); + } + return retval; +} + +ProxySession * +Http2ServerSession::get_proxy_session() +{ + return this; +} + +ProxyTransaction * +Http2ServerSession::new_transaction() +{ + this->set_session_active(); + + // Create a new stream/transaction + Http2Error error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); + Http2Stream *stream = connection_state.create_initiating_stream(error); + + if (!stream || connection_state.is_peer_concurrent_stream_ub()) { + if (error.cls != Http2ErrorClass::HTTP2_ERROR_CLASS_NONE) { + Error("HTTP/2 stream error code=0x%02x %s", static_cast(error.code), error.msg); + } + + remove_session(); + } + + return stream; +} + +void +Http2ServerSession::add_session() +{ + if (this->in_session_table) { + return; + } + Http2SsnDebug("Add session to pool"); + EThread *ethread = this_ethread(); + ServerSessionPool *pool = ethread->server_session_pool; + MUTEX_TRY_LOCK(lock, pool->mutex, ethread); + if (lock.is_locked()) { + pool->addSession(this); + this->in_session_table = true; + } +} + +void +Http2ServerSession::remove_session() +{ + if (!this->in_session_table) { + return; + } + Http2SsnDebug("Remove session from pool"); + EThread *ethread = this_ethread(); + ServerSessionPool *pool = ethread->server_session_pool; + MUTEX_TRY_LOCK(lock, pool->mutex, ethread); + if (lock.is_locked()) { + pool->removeSession(this); + in_session_table = false; + } else { + ink_release_assert(!"How did we not get the pool lock?"); + } +} + +bool +Http2ServerSession::is_multiplexing() const +{ + return true; +} + +bool +Http2ServerSession::is_outbound() const +{ + return true; +} + +void +Http2ServerSession::set_netvc(NetVConnection *netvc) +{ + super::set_netvc(netvc); + if (netvc == nullptr) { + write_vio = nullptr; + } +} + +void +Http2ServerSession::set_no_activity_timeout() +{ + // Only set if not previously set + if (this->_vc->get_inactivity_timeout() == 0) { + this->set_inactivity_timeout(HRTIME_SECONDS(Http2::no_activity_timeout_out)); + } +} + +HTTPVersion +Http2ServerSession::get_version(HTTPHdr &hdr) const +{ + return HTTP_2_0; +} + +IOBufferReader * +Http2ServerSession::get_remote_reader() +{ + return _read_buffer_reader; +} + +std::function create_h2_server_session = []() -> PoolableSession * { + return http2ServerSessionAllocator.alloc(); +}; diff --git a/proxy/http2/Http2ServerSession.h b/proxy/http2/Http2ServerSession.h new file mode 100644 index 00000000000..93d8f9fbf2f --- /dev/null +++ b/proxy/http2/Http2ServerSession.h @@ -0,0 +1,94 @@ +/** @file + + Http2ServerSession. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "Plugin.h" +#include "Http2CommonSession.h" +#include +#include "tscore/ink_inet.h" +#include "tscore/History.h" +#include "Milestones.h" +#include "PoolableSession.h" + +class Http2ServerSession : public PoolableSession, public Http2CommonSession +{ +public: + using super = PoolableSession; ///< Parent type. + using SessionHandler = int (Http2ServerSession::*)(int, void *); + + Http2ServerSession(); + + ///////////////////// + // Methods + + // Implement VConnection interface + void do_io_close(int lerrno = -1) override; + + // Implement ProxySession interface + void new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader) override; + void start() override; + void destroy() override; + void release(ProxyTransaction *trans) override; + void free() override; + ProxyTransaction *new_transaction() override; + + void add_session() override; + void remove_session(); + + //////////////////// + // Accessors + sockaddr const *get_remote_addr() const override; + sockaddr const *get_local_addr() override; + int get_transact_count() const override; + const char *get_protocol_string() const override; + int populate_protocol(std::string_view *result, int size) const override; + const char *protocol_contains(std::string_view prefix) const override; + HTTPVersion get_version(HTTPHdr &hdr) const override; + void increment_current_active_connections_stat() override; + void decrement_current_active_connections_stat() override; + IOBufferReader *get_remote_reader() override; + + ProxySession *get_proxy_session() override; + + // noncopyable + Http2ServerSession(Http2ServerSession &) = delete; + Http2ServerSession &operator=(const Http2ServerSession &) = delete; + + bool is_multiplexing() const override; + bool is_outbound() const override; + + void set_netvc(NetVConnection *netvc) override; + + void set_no_activity_timeout() override; + +private: + int main_event_handler(int, void *); + + IpEndpoint cached_client_addr; + IpEndpoint cached_local_addr; + + bool in_session_table = false; +}; + +extern ClassAllocator http2ServerSessionAllocator; diff --git a/proxy/http2/Http2Stream.cc b/proxy/http2/Http2Stream.cc index 33c96fbb643..c620b70b6d2 100644 --- a/proxy/http2/Http2Stream.cc +++ b/proxy/http2/Http2Stream.cc @@ -25,8 +25,10 @@ #include "HTTP2.h" #include "Http2ClientSession.h" +#include "Http2ServerSession.h" #include "HttpDebugNames.h" #include "HttpSM.h" +#include "tscore/HTTPVersion.h" #include @@ -40,21 +42,34 @@ ClassAllocator http2StreamAllocator("http2StreamAllocator"); -Http2Stream::Http2Stream(ProxySession *session, Http2StreamId sid, ssize_t initial_peer_rwnd, ssize_t initial_local_rwnd) - : super(session), _id(sid), _peer_rwnd(initial_peer_rwnd), _local_rwnd(initial_local_rwnd) +Http2Stream::Http2Stream(ProxySession *session, Http2StreamId sid, ssize_t initial_peer_rwnd, ssize_t initial_local_rwnd, + bool registered_stream) + : super(session), _id(sid), _registered_stream(registered_stream), _peer_rwnd(initial_peer_rwnd), _local_rwnd(initial_local_rwnd) { SET_HANDLER(&Http2Stream::main_event_handler); this->mark_milestone(Http2StreamMilestone::OPEN); - this->_sm = nullptr; - this->_thread = this_ethread(); - this->upstream_outbound_options = *(session->accept_options); + this->_sm = nullptr; + this->_thread = this_ethread(); + this->_state = Http2StreamState::HTTP2_STREAM_STATE_IDLE; + + auto const *proxy_session = get_proxy_ssn(); + ink_assert(proxy_session != nullptr); + auto const *h2_session = dynamic_cast(proxy_session); + ink_assert(h2_session != nullptr); + this->_is_outbound = h2_session->is_outbound(); this->_reader = this->_receive_buffer.alloc_reader(); - _receive_header.create(HTTP_TYPE_REQUEST); - _send_header.create(HTTP_TYPE_RESPONSE, HTTP_2_0); + if (this->is_outbound_connection()) { // Flip the sense of the expected headers. Fix naming later + _receive_header.create(HTTP_TYPE_RESPONSE); + _send_header.create(HTTP_TYPE_REQUEST, HTTP_2_0); + } else { + this->upstream_outbound_options = *(session->accept_options); + _receive_header.create(HTTP_TYPE_REQUEST); + _send_header.create(HTTP_TYPE_RESPONSE, HTTP_2_0); + } http_parser_init(&http_parser); } @@ -62,7 +77,16 @@ Http2Stream::Http2Stream(ProxySession *session, Http2StreamId sid, ssize_t initi Http2Stream::~Http2Stream() { REMEMBER(NO_EVENT, this->reentrancy_count); - Http2StreamDebug("Destroy stream, sent %" PRIu64 " bytes", this->bytes_sent); + Http2StreamDebug("Destroy stream, sent %" PRIu64 " bytes, registered: %s", this->bytes_sent, + (_registered_stream ? "true" : "false")); + + // In the case of a temporary stream used to parse the header to keep the HPACK + // up to date, there may not be a mutex. Nothing was set up, so nothing to + // clean up in the destructor + if (this->mutex == nullptr) { + return; + } + SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); // Clean up after yourself if this was an EOS ink_release_assert(this->closed); @@ -70,23 +94,26 @@ Http2Stream::~Http2Stream() uint64_t cid = 0; - // Safe to initiate SSN_CLOSE if this is the last stream - if (_proxy_ssn) { - cid = _proxy_ssn->connection_id(); + if (_registered_stream) { + // Safe to initiate SSN_CLOSE if this is the last stream + if (_proxy_ssn) { + cid = _proxy_ssn->connection_id(); - Http2ClientSession *h2_proxy_ssn = static_cast(_proxy_ssn); - SCOPED_MUTEX_LOCK(lock, h2_proxy_ssn->mutex, this_ethread()); - // Make sure the stream is removed from the stream list and priority tree - // In many cases, this has been called earlier, so this call is a no-op - h2_proxy_ssn->connection_state.delete_stream(this); + SCOPED_MUTEX_LOCK(lock, _proxy_ssn->mutex, this_ethread()); + Http2ConnectionState &connection_state = this->get_connection_state(); - h2_proxy_ssn->connection_state.decrement_peer_stream_count(); + // Make sure the stream is removed from the stream list and priority tree + // In many cases, this has been called earlier, so this call is a no-op + connection_state.delete_stream(this); - // Update session's stream counts, so it accurately goes into keep-alive state - h2_proxy_ssn->connection_state.release_stream(); + connection_state.decrement_peer_stream_count(); - // Do not access `_proxy_ssn` in below. It might be freed by `release_stream`. - } + // Update session's stream counts, so it accurately goes into keep-alive state + connection_state.release_stream(); + + // Do not access `_proxy_ssn` in below. It might be freed by `release_stream`. + } + } // Otherwise, not registered with the connection_state (i.e. a temporary stream used for HPACK header processing) // Clean up the write VIO in case of inactivity timeout this->do_io_write(nullptr, 0, nullptr); @@ -182,15 +209,17 @@ Http2Stream::main_event_handler(int event, void *edata) this->signal_write_event(event); } } else { - update_write_request(true); + this->update_write_request(true); } break; case VC_EVENT_READ_COMPLETE: + read_vio.nbytes = read_vio.ndone; + /* fall through */ case VC_EVENT_READ_READY: _timeout.update_inactivity(); if (e->cookie == &read_vio) { if (read_vio.mutex && read_vio.cont && this->_sm) { - signal_read_event(event); + this->signal_read_event(event); } } else { this->update_read_request(true); @@ -199,10 +228,14 @@ Http2Stream::main_event_handler(int event, void *edata) case VC_EVENT_EOS: if (e->cookie == &read_vio) { SCOPED_MUTEX_LOCK(lock, read_vio.mutex, this_ethread()); - read_vio.cont->handleEvent(VC_EVENT_EOS, &read_vio); + if (read_vio.cont) { + read_vio.cont->handleEvent(VC_EVENT_EOS, &read_vio); + } } else if (e->cookie == &write_vio) { SCOPED_MUTEX_LOCK(lock, write_vio.mutex, this_ethread()); - write_vio.cont->handleEvent(VC_EVENT_EOS, &write_vio); + if (write_vio.cont) { + write_vio.cont->handleEvent(VC_EVENT_EOS, &write_vio); + } } break; } @@ -216,8 +249,9 @@ Http2Stream::main_event_handler(int event, void *edata) Http2ErrorCode Http2Stream::decode_header_blocks(HpackHandle &hpack_handle, uint32_t maximum_table_size) { - Http2ErrorCode error = http2_decode_header_blocks(&_receive_header, header_blocks, header_blocks_length, nullptr, hpack_handle, - is_trailing_header, maximum_table_size); + Http2ErrorCode error = + http2_decode_header_blocks(&_receive_header, (const uint8_t *)header_blocks, header_blocks_length, nullptr, hpack_handle, + _trailing_header_is_possible, maximum_table_size, this->is_outbound_connection()); if (error != Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { Http2StreamDebug("Error decoding header blocks: %u", static_cast(error)); } @@ -227,16 +261,30 @@ Http2Stream::decode_header_blocks(HpackHandle &hpack_handle, uint32_t maximum_ta void Http2Stream::send_request(Http2ConnectionState &cstate) { - ink_release_assert(this->_sm != nullptr); - this->_http_sm_id = this->_sm->sm_id; + if (closed) { + return; + } + REMEMBER(NO_EVENT, this->reentrancy_count); // Convert header to HTTP/1.1 format if (http2_convert_header_from_2_to_1_1(&_receive_header) == PARSE_RESULT_ERROR) { - // There's no way to cause Bad Request directly at this time. - // Set an invalid method so it causes an error later. - _receive_header.method_set("\xffVOID", 1); + Http2StreamDebug("Error converting HTTP/2 headers to HTTP/1.1."); + if (_receive_header.type_get() == HTTP_TYPE_REQUEST) { + // There's no way to cause Bad Request directly at this time. + // Set an invalid method so it causes an error later. + _receive_header.method_set("\xffVOID", 1); + } } + if (this->expect_send_trailer()) { + // Send read complete to terminate previous data tunnel + this->read_vio.nbytes = this->read_vio.ndone; + this->signal_read_event(VC_EVENT_READ_COMPLETE); + } + + ink_release_assert(this->_sm != nullptr); + this->_http_sm_id = this->_sm->sm_id; + // Write header to a buffer. Borrowing logic from HttpSM::write_header_into_buffer. // Seems like a function like this ought to be in HTTPHdr directly int bufindex; @@ -250,7 +298,7 @@ Http2Stream::send_request(Http2ConnectionState &cstate) this->_receive_buffer.add_block(); block = this->_receive_buffer.get_current_block(); } - done = _receive_header.print(block->start(), block->write_avail(), &bufindex, &tmp); + done = _receive_header.print(block->end(), block->write_avail(), &bufindex, &tmp); dumpoffset += bufindex; this->_receive_buffer.fill(bufindex); if (!done) { @@ -267,7 +315,12 @@ Http2Stream::send_request(Http2ConnectionState &cstate) if (this->read_vio.nbytes > 0) { if (this->receive_end_stream) { this->read_vio.nbytes = bufindex; - this->signal_read_event(VC_EVENT_READ_COMPLETE); + this->read_vio.ndone = bufindex; + if (this->is_outbound_connection()) { + this->signal_read_event(VC_EVENT_EOS); + } else { + this->signal_read_event(VC_EVENT_READ_COMPLETE); + } } else { // End of header but not end of stream, must have some body frames coming this->has_body = true; @@ -300,6 +353,8 @@ Http2Stream::change_state(uint8_t type, uint8_t flags) } } else if (type == HTTP2_FRAME_TYPE_PUSH_PROMISE) { _state = Http2StreamState::HTTP2_STREAM_STATE_RESERVED_LOCAL; + } else if (type == HTTP2_FRAME_TYPE_RST_STREAM) { + _state = Http2StreamState::HTTP2_STREAM_STATE_CLOSED; } else { return false; } @@ -310,7 +365,11 @@ Http2Stream::change_state(uint8_t type, uint8_t flags) _state = Http2StreamState::HTTP2_STREAM_STATE_CLOSED; } else if (type == HTTP2_FRAME_TYPE_HEADERS || type == HTTP2_FRAME_TYPE_DATA) { if (receive_end_stream) { - _state = Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_REMOTE; + if (send_end_stream) { + _state = Http2StreamState::HTTP2_STREAM_STATE_CLOSED; + } else { + _state = Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_REMOTE; + } } else if (send_end_stream) { _state = Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_LOCAL; } else { @@ -343,10 +402,6 @@ Http2Stream::change_state(uint8_t type, uint8_t flags) case Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_LOCAL: if (type == HTTP2_FRAME_TYPE_RST_STREAM || receive_end_stream) { _state = Http2StreamState::HTTP2_STREAM_STATE_CLOSED; - } else { - // Error, set state closed - _state = Http2StreamState::HTTP2_STREAM_STATE_CLOSED; - return false; } break; @@ -359,10 +414,6 @@ Http2Stream::change_state(uint8_t type, uint8_t flags) } else if (type == HTTP2_FRAME_TYPE_CONTINUATION) { // w/o END_STREAM flag // No state change here. Expect a following DATA frame with END_STREAM flag. return true; - } else { - // Error, set state closed - _state = Http2StreamState::HTTP2_STREAM_STATE_CLOSED; - return false; } break; @@ -414,6 +465,7 @@ Http2Stream::do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *abuffe write_vio.ndone = 0; write_vio.vc_server = this; write_vio.op = VIO::WRITE; + _send_reader = abuffer; if (c != nullptr && nbytes > 0 && this->is_state_writeable()) { update_write_request(false); @@ -434,23 +486,22 @@ Http2Stream::do_io_close(int /* flags */) REMEMBER(NO_EVENT, this->reentrancy_count); Http2StreamDebug("do_io_close"); + // if (this->is_state_writeable()) { // Let the other end know we are going away + // this->get_connection_state().send_rst_stream_frame(_id, Http2ErrorCode::HTTP2_ERROR_NO_ERROR); + //} + // When we get here, the SM has initiated the shutdown. Either it received a WRITE_COMPLETE, or it is shutting down. Any // remaining IO operations back to client should be abandoned. The SM-side buffers backing these operations will be deleted // by the time this is called from transaction_done. closed = true; - if (_proxy_ssn && this->is_state_writeable()) { - // Make sure any trailing end of stream frames are sent - // We will be removed at send_data_frames or closing connection phase - Http2ClientSession *h2_proxy_ssn = static_cast(this->_proxy_ssn); - SCOPED_MUTEX_LOCK(lock, h2_proxy_ssn->mutex, this_ethread()); - h2_proxy_ssn->connection_state.send_data_frames(this); - } + // Adjust state, so we don't process any more data + _state = Http2StreamState::HTTP2_STREAM_STATE_CLOSED; _clear_timers(); clear_io_events(); - // Wait until transaction_done is called from HttpSM to signal that the TXN_CLOSE hook has been executed + // Otherwise, Wait until transaction_done is called from HttpSM to signal that the TXN_CLOSE hook has been executed } } @@ -466,7 +517,8 @@ Http2Stream::transaction_done() if (!closed) { do_io_close(); // Make sure we've been closed. If we didn't close the _proxy_ssn session better still be open } - ink_release_assert(closed || !static_cast(_proxy_ssn)->connection_state.is_state_closed()); + Http2ConnectionState &state = this->get_connection_state(); + ink_release_assert(closed || !state.is_state_closed()); _sm = nullptr; if (closed) { @@ -481,11 +533,11 @@ Http2Stream::transaction_done() void Http2Stream::terminate_if_possible() { - if (terminate_stream && reentrancy_count == 0) { + // if (terminate_stream && reentrancy_count == 0) { + if (reentrancy_count == 0 && closed && terminate_stream) { REMEMBER(NO_EVENT, this->reentrancy_count); - Http2ClientSession *h2_proxy_ssn = static_cast(this->_proxy_ssn); - SCOPED_MUTEX_LOCK(lock, h2_proxy_ssn->mutex, this_ethread()); + SCOPED_MUTEX_LOCK(lock, _proxy_ssn->mutex, this_ethread()); THREAD_FREE(this, http2StreamAllocator, this_ethread()); } } @@ -497,7 +549,8 @@ Http2Stream::initiating_close() if (!closed) { SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); REMEMBER(NO_EVENT, this->reentrancy_count); - Http2StreamDebug("initiating_close"); + Http2StreamDebug("initiating_close client_window=%" PRId64 " session_window=%" PRId64, _peer_rwnd, + this->get_connection_state().get_peer_rwnd_in()); // Set the state of the connection to closed // TODO - these states should be combined @@ -520,28 +573,34 @@ Http2Stream::initiating_close() bool sent_write_complete = false; if (_sm) { // Push out any last IO events - if (write_vio.cont) { + // First look for active write or read + if (write_vio.cont && write_vio.nbytes > 0 && write_vio.ndone == write_vio.nbytes && + (!is_outbound_connection() || get_state() == Http2StreamState::HTTP2_STREAM_STATE_OPEN)) { SCOPED_MUTEX_LOCK(lock, write_vio.mutex, this_ethread()); - // Are we done? - if (write_vio.nbytes > 0 && write_vio.nbytes == write_vio.ndone) { - Http2StreamDebug("handle write from destroy (event=%d)", VC_EVENT_WRITE_COMPLETE); - write_event = send_tracked_event(write_event, VC_EVENT_WRITE_COMPLETE, &write_vio); - } else { - write_event = send_tracked_event(write_event, VC_EVENT_EOS, &write_vio); - Http2StreamDebug("handle write from destroy (event=%d)", VC_EVENT_EOS); - } + Http2StreamDebug("Send tracked event VC_EVENT_WRITE_COMPLETE on write_vio. sm_id: %" PRId64, _sm->sm_id); + write_event = send_tracked_event(write_event, VC_EVENT_WRITE_COMPLETE, &write_vio); sent_write_complete = true; } - } - // Send EOS to let SM know that we aren't sticking around - if (_sm && read_vio.cont) { - // Only bother with the EOS if we haven't sent the write complete + if (!sent_write_complete) { - SCOPED_MUTEX_LOCK(lock, read_vio.mutex, this_ethread()); - Http2StreamDebug("send EOS to read cont"); - read_event = send_tracked_event(read_event, VC_EVENT_EOS, &read_vio); + if (write_vio.cont && write_vio.buffer.writer() && + (!is_outbound_connection() || get_state() == Http2StreamState::HTTP2_STREAM_STATE_OPEN || + get_state() == Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_LOCAL)) { + SCOPED_MUTEX_LOCK(lock, write_vio.mutex, this_ethread()); + Http2StreamDebug("Send tracked event VC_EVENT_EOS on write_vio. sm_id: %" PRId64, _sm->sm_id); + write_event = send_tracked_event(write_event, VC_EVENT_EOS, &write_vio); + } else if (read_vio.cont && read_vio.buffer.writer()) { + SCOPED_MUTEX_LOCK(lock, read_vio.mutex, this_ethread()); + Http2StreamDebug("Send tracked event VC_EVENT_EOS on read_vio. sm_id: %" PRId64, _sm->sm_id); + read_event = send_tracked_event(read_event, VC_EVENT_EOS, &read_vio); + } else { + Http2StreamDebug("send EOS to SM"); + // Just send EOS to the _sm + _sm->handleEvent(VC_EVENT_EOS, nullptr); + } } - } else if (!sent_write_complete) { + } else { + Http2StreamDebug("No SM to signal"); // Transaction is already gone or not started. Kill yourself terminate_stream = true; terminate_if_possible(); @@ -549,6 +608,12 @@ Http2Stream::initiating_close() } } +bool +Http2Stream::is_outbound_connection() const +{ + return _is_outbound; +} + /* Replace existing event only if the new event is different than the inprogress event */ Event * Http2Stream::send_tracked_event(Event *event, int send_event, VIO *vio) @@ -582,7 +647,7 @@ Http2Stream::update_read_request(bool call_update) ink_release_assert(this->_thread == this_ethread()); SCOPED_MUTEX_LOCK(lock, read_vio.mutex, this_ethread()); - if (read_vio.nbytes == 0) { + if (read_vio.nbytes == 0 || read_vio.is_disabled()) { return; } @@ -609,9 +674,23 @@ Http2Stream::update_read_request(bool call_update) void Http2Stream::restart_sending() { + // Make sure the stream is in a good state to be sending + if (this->is_closed()) { + return; + } if (!this->parsing_header_done) { + this->update_write_request(true); return; } + if (this->is_outbound_connection()) { + if (this->get_state() != Http2StreamState::HTTP2_STREAM_STATE_OPEN || write_vio.ntodo() == 0) { + return; + } + } else { + if (this->get_state() != Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_REMOTE) { + return; + } + } IOBufferReader *reader = this->get_data_reader_for_send(); if (reader && !reader->is_read_avail_more_than(0)) { @@ -629,7 +708,7 @@ void Http2Stream::update_write_request(bool call_update) { if (!this->is_state_writeable() || closed || _proxy_ssn == nullptr || write_vio.mutex == nullptr || - write_vio.get_reader() == nullptr) { + write_vio.get_reader() == nullptr || this->_send_reader == nullptr) { return; } @@ -639,26 +718,40 @@ Http2Stream::update_write_request(bool call_update) } ink_release_assert(this->_thread == this_ethread()); - Http2ClientSession *h2_proxy_ssn = static_cast(this->_proxy_ssn); + Http2StreamDebug("update_write_request parse_done=%d", parsing_header_done); + + Http2ConnectionState &connection_state = this->get_connection_state(); SCOPED_MUTEX_LOCK(lock, write_vio.mutex, this_ethread()); IOBufferReader *vio_reader = write_vio.get_reader(); - if (write_vio.ntodo() == 0 || !vio_reader->is_read_avail_more_than(0)) { + if (write_vio.ntodo() > 0 && (!vio_reader->is_read_avail_more_than(0) || + // If there is no window left, just give up now too + std::min(_peer_rwnd, this->get_connection_state().get_peer_rwnd_in()) == 0)) { + Http2StreamDebug("update_write_request give up without doing anything ntodo=%" PRId64 " is_read_avail=%d client_window=%" PRId64 + " session_window=%" PRId64, + write_vio.ntodo(), vio_reader->is_read_avail_more_than(0), _peer_rwnd, + this->get_connection_state().get_peer_rwnd_in()); return; } // Process the new data if (!this->parsing_header_done) { - // Still parsing the response_header + // Still parsing the request or response header int bytes_used = 0; - int state = this->_send_header.parse_resp(&http_parser, vio_reader, &bytes_used, false); - // HTTPHdr::parse_resp() consumed the vio_reader in above (consumed size is `bytes_used`) + int state; + if (this->is_outbound_connection()) { + state = this->_send_header.parse_req(&http_parser, this->_send_reader, &bytes_used, false); + } else { + state = this->_send_header.parse_resp(&http_parser, this->_send_reader, &bytes_used, false); + } + // HTTPHdr::parse_resp() consumed the send_reader in above write_vio.ndone += bytes_used; switch (state) { case PARSE_RESULT_DONE: { this->parsing_header_done = true; + Http2StreamDebug("update_write_request parsing done, read %d bytes", bytes_used); // Schedule session shutdown if response header has "Connection: close" MIMEField *field = this->_send_header.field_find(MIME_FIELD_CONNECTION, MIME_LEN_CONNECTION); @@ -666,31 +759,36 @@ Http2Stream::update_write_request(bool call_update) int len; const char *value = field->value_get(&len); if (memcmp(HTTP_VALUE_CLOSE, value, HTTP_LEN_CLOSE) == 0) { - SCOPED_MUTEX_LOCK(lock, h2_proxy_ssn->mutex, this_ethread()); - if (h2_proxy_ssn->connection_state.get_shutdown_state() == HTTP2_SHUTDOWN_NONE) { - h2_proxy_ssn->connection_state.set_shutdown_state(HTTP2_SHUTDOWN_NOT_INITIATED, Http2ErrorCode::HTTP2_ERROR_NO_ERROR); + SCOPED_MUTEX_LOCK(lock, _proxy_ssn->mutex, this_ethread()); + if (connection_state.get_shutdown_state() == HTTP2_SHUTDOWN_NONE) { + connection_state.set_shutdown_state(HTTP2_SHUTDOWN_NOT_INITIATED, Http2ErrorCode::HTTP2_ERROR_NO_ERROR); } } } { - SCOPED_MUTEX_LOCK(lock, h2_proxy_ssn->mutex, this_ethread()); + SCOPED_MUTEX_LOCK(lock, _proxy_ssn->mutex, this_ethread()); // Send the response header back - h2_proxy_ssn->connection_state.send_headers_frame(this); + connection_state.send_headers_frame(this); } // Roll back states of response header to read final response - if (this->_send_header.expect_final_response()) { + if (!this->is_outbound_connection() && this->_send_header.expect_final_response()) { this->parsing_header_done = false; + } + if (this->is_outbound_connection() || this->_send_header.expect_final_response()) { _send_header.destroy(); - _send_header.create(HTTP_TYPE_RESPONSE, HTTP_2_0); + _send_header.create(this->is_outbound_connection() ? HTTP_TYPE_REQUEST : HTTP_TYPE_RESPONSE, HTTP_2_0); http_parser_clear(&http_parser); http_parser_init(&http_parser); } + bool final_write = this->write_vio.ntodo() == 0; + if (final_write) { + this->signal_write_event(VC_EVENT_WRITE_COMPLETE, !CALL_UPDATE); + } - this->signal_write_event(call_update); - - if (vio_reader->is_read_avail_more_than(0)) { + if (!final_write && this->_send_reader->is_read_avail_more_than(0)) { + Http2StreamDebug("update_write_request done parsing, still more to send"); this->_milestones.mark(Http2StreamMilestone::START_TX_DATA_FRAMES); this->send_body(call_update); } @@ -698,8 +796,10 @@ Http2Stream::update_write_request(bool call_update) } case PARSE_RESULT_CONT: // Let it ride for next time + Http2StreamDebug("update_write_request still parsing, read %d bytes", bytes_used); break; default: + Http2StreamDebug("update_write_request state %d, read %d bytes", state, bytes_used); break; } } else { @@ -713,12 +813,18 @@ Http2Stream::update_write_request(bool call_update) void Http2Stream::signal_read_event(int event) { - if (this->read_vio.cont == nullptr || this->read_vio.cont->mutex == nullptr || this->read_vio.op == VIO::NONE) { + if (this->read_vio.cont == nullptr || this->read_vio.cont->mutex == nullptr || this->read_vio.op == VIO::NONE || + this->terminate_stream) { return; } + reentrancy_count++; MUTEX_TRY_LOCK(lock, read_vio.cont->mutex, this_ethread()); if (lock.is_locked()) { + if (read_event) { + read_event->cancel(); + read_event = nullptr; + } _timeout.update_inactivity(); this->read_vio.cont->handleEvent(event, &this->read_vio); } else { @@ -727,79 +833,82 @@ Http2Stream::signal_read_event(int event) } this->_read_vio_event = this_ethread()->schedule_in(this, retry_delay, event, &read_vio); } + reentrancy_count--; + // Clean stream up if the terminate flag is set and we are at the bottom of the handler stack + terminate_if_possible(); } void -Http2Stream::signal_write_event(int event) +Http2Stream::signal_write_event(int event, bool call_update) { // Don't signal a write event if in fact nothing was written if (this->write_vio.cont == nullptr || this->write_vio.cont->mutex == nullptr || this->write_vio.op == VIO::NONE || - this->write_vio.nbytes == 0) { - return; - } - - MUTEX_TRY_LOCK(lock, write_vio.cont->mutex, this_ethread()); - if (lock.is_locked()) { - _timeout.update_inactivity(); - this->write_vio.cont->handleEvent(event, &this->write_vio); - } else { - if (this->_write_vio_event) { - this->_write_vio_event->cancel(); - } - this->_write_vio_event = this_ethread()->schedule_in(this, retry_delay, event, &write_vio); - } -} - -void -Http2Stream::signal_write_event(bool call_update) -{ - if (this->write_vio.cont == nullptr || this->write_vio.op == VIO::NONE) { - return; - } - - if (this->write_vio.get_writer()->write_avail() == 0) { + this->terminate_stream) { return; } - int send_event = this->write_vio.ntodo() == 0 ? VC_EVENT_WRITE_COMPLETE : VC_EVENT_WRITE_READY; - + reentrancy_count++; if (call_update) { - // Coming from reenable. Safe to call the handler directly - if (write_vio.cont && this->_sm) { - write_vio.cont->handleEvent(send_event, &write_vio); + MUTEX_TRY_LOCK(lock, write_vio.cont->mutex, this_ethread()); + if (lock.is_locked()) { + if (write_event) { + write_event->cancel(); + write_event = nullptr; + } + _timeout.update_inactivity(); + this->write_vio.cont->handleEvent(event, &this->write_vio); + } else { + if (this->_write_vio_event) { + this->_write_vio_event->cancel(); + } + this->_write_vio_event = this_ethread()->schedule_in(this, retry_delay, event, &write_vio); } } else { // Called from do_io_write. Might still be setting up state. Send an event to let the dust settle - write_event = send_tracked_event(write_event, send_event, &write_vio); + write_event = send_tracked_event(write_event, event, &write_vio); } + reentrancy_count--; + // Clean stream up if the terminate flag is set and we are at the bottom of the handler stack + terminate_if_possible(); } bool Http2Stream::push_promise(URL &url, const MIMEField *accept_encoding) { - Http2ClientSession *h2_proxy_ssn = static_cast(this->_proxy_ssn); - SCOPED_MUTEX_LOCK(lock, h2_proxy_ssn->mutex, this_ethread()); - return h2_proxy_ssn->connection_state.send_push_promise_frame(this, url, accept_encoding); + SCOPED_MUTEX_LOCK(lock, _proxy_ssn->mutex, this_ethread()); + return this->get_connection_state().send_push_promise_frame(this, url, accept_encoding); } void Http2Stream::send_body(bool call_update) { - Http2ClientSession *h2_proxy_ssn = static_cast(this->_proxy_ssn); + Http2ConnectionState &connection_state = this->get_connection_state(); _timeout.update_inactivity(); + reentrancy_count++; + + SCOPED_MUTEX_LOCK(lock, _proxy_ssn->mutex, this_ethread()); if (Http2::stream_priority_enabled) { - SCOPED_MUTEX_LOCK(lock, h2_proxy_ssn->mutex, this_ethread()); - h2_proxy_ssn->connection_state.schedule_stream(this); + connection_state.schedule_stream(this); // signal_write_event() will be called from `Http2ConnectionState::send_data_frames_depends_on_priority()` // when write_vio is consumed } else { - SCOPED_MUTEX_LOCK(lock, h2_proxy_ssn->mutex, this_ethread()); - h2_proxy_ssn->connection_state.send_data_frames(this); - this->signal_write_event(call_update); + connection_state.send_data_frames(this); // XXX The call to signal_write_event can destroy/free the Http2Stream. // Don't modify the Http2Stream after calling this method. } + + reentrancy_count--; + terminate_if_possible(); +} + +void +Http2Stream::reenable_write() +{ + if (this->_proxy_ssn) { + SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); + update_write_request(true); + } } void @@ -810,14 +919,9 @@ Http2Stream::reenable(VIO *vio) SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); update_write_request(true); } else if (vio->op == VIO::READ) { - Http2ClientSession *h2_proxy_ssn = static_cast(this->_proxy_ssn); - { - SCOPED_MUTEX_LOCK(ssn_lock, h2_proxy_ssn->mutex, this_ethread()); - h2_proxy_ssn->connection_state.restart_receiving(this); - } - - SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); - update_read_request(true); + SCOPED_MUTEX_LOCK(ssn_lock, _proxy_ssn->mutex, this_ethread()); + Http2ConnectionState &connection_state = this->get_connection_state(); + connection_state.restart_receiving(this); } } } @@ -825,7 +929,7 @@ Http2Stream::reenable(VIO *vio) IOBufferReader * Http2Stream::get_data_reader_for_send() const { - return write_vio.get_reader(); + return this->_send_reader; } void @@ -903,14 +1007,23 @@ Http2Stream::release() void Http2Stream::increment_transactions_stat() { - HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_STREAM_COUNT, _thread); - HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_TOTAL_CLIENT_STREAM_COUNT, _thread); + if (this->is_outbound_connection()) { + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_SERVER_STREAM_COUNT, _thread); + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_TOTAL_SERVER_STREAM_COUNT, _thread); + } else { + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_STREAM_COUNT, _thread); + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_TOTAL_CLIENT_STREAM_COUNT, _thread); + } } void Http2Stream::decrement_transactions_stat() { - HTTP2_DECREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_STREAM_COUNT, _thread); + if (this->is_outbound_connection()) { + HTTP2_DECREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_SERVER_STREAM_COUNT, _thread); + } else { + HTTP2_DECREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_STREAM_COUNT, _thread); + } } ssize_t @@ -1016,3 +1129,68 @@ Http2Stream::has_request_body(int64_t content_length, bool is_chunked_set) const { return has_body; } + +Http2ConnectionState & +Http2Stream::get_connection_state() +{ + if (this->is_outbound_connection()) { + Http2ServerSession *session = static_cast(_proxy_ssn); + return session->connection_state; + } else { + Http2ClientSession *session = static_cast(_proxy_ssn); + return session->connection_state; + } +} + +bool +Http2Stream::is_read_closed() const +{ + return this->receive_end_stream; +} + +bool +Http2Stream::expect_send_trailer() const +{ + return this->_expect_send_trailer; +} + +void +Http2Stream::set_expect_send_trailer() +{ + _expect_send_trailer = true; + parsing_header_done = false; + reset_send_headers(); +} +bool +Http2Stream::expect_receive_trailer() const +{ + return this->_expect_receive_trailer; +} + +void +Http2Stream::set_expect_receive_trailer() +{ + _expect_receive_trailer = true; +} + +void +Http2Stream::set_rx_error_code(ProxyError e) +{ + if (!this->is_outbound_connection() && this->_sm) { + this->_sm->t_state.client_info.rx_error_code = e; + } +} + +void +Http2Stream::set_tx_error_code(ProxyError e) +{ + if (!this->is_outbound_connection() && this->_sm) { + this->_sm->t_state.client_info.tx_error_code = e; + } +} + +HTTPVersion +Http2Stream::get_version(HTTPHdr &hdr) const +{ + return HTTP_2_0; +} diff --git a/proxy/http2/Http2Stream.h b/proxy/http2/Http2Stream.h index ead1d60f733..87a0d95059e 100644 --- a/proxy/http2/Http2Stream.h +++ b/proxy/http2/Http2Stream.h @@ -35,7 +35,7 @@ class Http2Stream; class Http2ConnectionState; -typedef Http2DependencyTree::Tree DependencyTree; +using DependencyTree = Http2DependencyTree::Tree; enum class Http2StreamMilestone { OPEN = 0, @@ -48,6 +48,8 @@ enum class Http2StreamMilestone { LAST_ENTRY, }; +constexpr bool STREAM_IS_REGISTERED = true; + class Http2Stream : public ProxyTransaction { public: @@ -55,13 +57,15 @@ class Http2Stream : public ProxyTransaction using super = ProxyTransaction; ///< Parent type. Http2Stream() {} // Just to satisfy ClassAllocator - Http2Stream(ProxySession *session, Http2StreamId sid, ssize_t initial_peer_rwnd, ssize_t initial_local_rwnd); - ~Http2Stream(); + Http2Stream(ProxySession *session, Http2StreamId sid, ssize_t initial_peer_rwnd, ssize_t initial_local_rwnd, + bool registered_stream); + ~Http2Stream() override; int main_event_handler(int event, void *edata); void release() override; void reenable(VIO *vio) override; + void reenable_write(); void transaction_done() override; void @@ -72,17 +76,22 @@ class Http2Stream : public ProxyTransaction VIO *do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *abuffer, bool owner = false) override; void do_io_close(int lerrno = -1) override; + bool expect_send_trailer() const override; + void set_expect_send_trailer() override; + bool expect_receive_trailer() const override; + void set_expect_receive_trailer() override; + Http2ErrorCode decode_header_blocks(HpackHandle &hpack_handle, uint32_t maximum_table_size); void send_request(Http2ConnectionState &cstate); void initiating_close(); + bool is_outbound_connection() const; void terminate_if_possible(); void update_read_request(bool send_update); void update_write_request(bool send_update); void signal_read_event(int event); - void signal_write_event(int event); static constexpr auto CALL_UPDATE = true; - void signal_write_event(bool call_update = CALL_UPDATE); + void signal_write_event(int event, bool call_update = CALL_UPDATE); void restart_sending(); bool push_promise(URL &url, const MIMEField *accept_encoding); @@ -116,17 +125,31 @@ class Http2Stream : public ProxyTransaction bool is_first_transaction() const override; void increment_transactions_stat() override; void decrement_transactions_stat() override; + void set_transaction_id(int new_id); int get_transaction_id() const override; int get_transaction_priority_weight() const override; int get_transaction_priority_dependence() const override; + bool is_read_closed() const override; + + HTTPHdr * + get_send_header() + { + return &_send_header; + } + + void read_update(int count); + void read_done(); void clear_io_events(); bool is_state_writeable() const; bool is_closed() const; IOBufferReader *get_data_reader_for_send() const; + void set_rx_error_code(ProxyError e) override; + void set_tx_error_code(ProxyError e) override; bool has_request_body(int64_t content_length, bool is_chunked_set) const override; + HTTPVersion get_version(HTTPHdr &hdr) const override; void mark_milestone(Http2StreamMilestone type); @@ -139,15 +162,20 @@ class Http2Stream : public ProxyTransaction bool change_state(uint8_t type, uint8_t flags); void set_peer_rwnd(Http2WindowSize new_size); void set_local_rwnd(Http2WindowSize new_size); - bool has_trailing_header() const; + bool trailing_header_is_possible() const; + void set_trailing_header_is_possible(); void set_receive_headers(HTTPHdr &h2_headers); + void reset_receive_headers(); + void reset_send_headers(); MIOBuffer *read_vio_writer() const; int64_t read_vio_read_avail(); + bool read_enabled() const; ////////////////// // Variables uint8_t *header_blocks = nullptr; - uint32_t header_blocks_length = 0; // total length of header blocks (not include Padding or other fields) + uint32_t header_blocks_length = 0; // total length of header blocks (not include + // Padding or other fields) bool receive_end_stream = false; bool send_end_stream = false; @@ -156,10 +184,12 @@ class Http2Stream : public ProxyTransaction bool is_first_transaction_flag = false; HTTPHdr _send_header; + IOBufferReader *_send_reader = nullptr; Http2DependencyTree::Node *priority_node = nullptr; + Http2ConnectionState &get_connection_state(); + private: - bool response_is_data_available() const; Event *send_tracked_event(Event *event, int send_event, VIO *vio); void send_body(bool call_update); void _clear_timers(); @@ -180,15 +210,28 @@ class Http2Stream : public ProxyTransaction HTTPHdr _receive_header; MIOBuffer _receive_buffer = CLIENT_CONNECTION_FIRST_READ_BUFFER_SIZE_INDEX; - int64_t read_vio_nbytes; VIO read_vio; VIO write_vio; History _history; Milestones(Http2StreamMilestone::LAST_ENTRY)> _milestones; - bool is_trailing_header = false; - bool has_body = false; + bool _trailing_header_is_possible = false; + bool _expect_send_trailer = false; + bool _expect_receive_trailer = false; + + bool has_body = false; + + /** Whether this is an outbound (toward the origin) connection. + * + * We store this upon construction as a cached version of the session's + * is_outbound() call. In some circumstances we need this value after a + * session close in which is_outbound is not accessible. + */ + bool _is_outbound = false; + + /** Whether the stream has been registered with the connection state. */ + bool _registered_stream = true; // A brief discussion of similar flags and state variables: _state, closed, terminate_stream // @@ -265,6 +308,12 @@ Http2Stream::get_transaction_id() const return _id; } +inline void +Http2Stream::set_transaction_id(int new_id) +{ + _id = new_id; +} + inline Http2StreamState Http2Stream::get_state() const { @@ -284,9 +333,15 @@ Http2Stream::set_local_rwnd(Http2WindowSize new_size) } inline bool -Http2Stream::has_trailing_header() const +Http2Stream::trailing_header_is_possible() const { - return is_trailing_header; + return _trailing_header_is_possible; +} + +inline void +Http2Stream::set_trailing_header_is_possible() +{ + _trailing_header_is_possible = true; } inline void @@ -295,6 +350,20 @@ Http2Stream::set_receive_headers(HTTPHdr &h2_headers) _receive_header.copy(&h2_headers); } +inline void +Http2Stream::reset_receive_headers() +{ + this->_receive_header.destroy(); + this->_receive_header.create(HTTP_TYPE_RESPONSE); +} + +inline void +Http2Stream::reset_send_headers() +{ + this->_send_header.destroy(); + this->_send_header.create(HTTP_TYPE_RESPONSE); +} + // Check entire DATA payload length if content-length: header is exist inline void Http2Stream::increment_data_length(uint64_t length) @@ -306,6 +375,10 @@ inline bool Http2Stream::payload_length_is_valid() const { uint32_t content_length = _receive_header.get_content_length(); + if (content_length != 0 && content_length != data_length) { + Warning("Bad payload length content_length=%d data_legnth=%d session_id=%" PRId64, content_length, + static_cast(data_length), _proxy_ssn->connection_id()); + } return content_length == 0 || content_length == data_length; } @@ -313,7 +386,8 @@ inline bool Http2Stream::is_state_writeable() const { return _state == Http2StreamState::HTTP2_STREAM_STATE_OPEN || _state == Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_REMOTE || - _state == Http2StreamState::HTTP2_STREAM_STATE_RESERVED_LOCAL; + _state == Http2StreamState::HTTP2_STREAM_STATE_RESERVED_LOCAL || + (this->is_outbound_connection() && _state == Http2StreamState::HTTP2_STREAM_STATE_IDLE); } inline bool @@ -334,9 +408,27 @@ Http2Stream::read_vio_writer() const return this->read_vio.get_writer(); } +inline bool +Http2Stream::read_enabled() const +{ + return !this->read_vio.is_disabled(); +} + inline void Http2Stream::_clear_timers() { _timeout.cancel_active_timeout(); _timeout.cancel_inactive_timeout(); } + +inline void +Http2Stream::read_update(int count) +{ + read_vio.ndone += count; +} + +inline void +Http2Stream::read_done() +{ + read_vio.nbytes = read_vio.ndone; +} diff --git a/proxy/http2/Makefile.am b/proxy/http2/Makefile.am index 544dfcc4d8f..a6dae126a99 100644 --- a/proxy/http2/Makefile.am +++ b/proxy/http2/Makefile.am @@ -45,6 +45,8 @@ libhttp2_a_SOURCES = \ Http2ClientSession.h \ Http2CommonSession.cc \ Http2CommonSession.h \ + Http2ServerSession.cc \ + Http2ServerSession.h \ Http2ConnectionState.cc \ Http2ConnectionState.h \ Http2DebugNames.cc \ diff --git a/proxy/http2/unit_tests/test_HTTP2.cc b/proxy/http2/unit_tests/test_HTTP2.cc index 5ec532031e1..cb4dcf2e8a2 100644 --- a/proxy/http2/unit_tests/test_HTTP2.cc +++ b/proxy/http2/unit_tests/test_HTTP2.cc @@ -107,8 +107,8 @@ TEST_CASE("Convert HTTPHdr", "[HTTP2]") // check CHECK_THAT(buf, Catch::StartsWith("GET https://trafficserver.apache.org/index.html HTTP/1.1\r\n" - "Host: trafficserver.apache.org\r\n" "User-Agent: foobar\r\n" + "Host: trafficserver.apache.org\r\n" "\r\n")); } diff --git a/src/records/RecHttp.cc b/src/records/RecHttp.cc index cba63d8ac8f..fe48ab4805a 100644 --- a/src/records/RecHttp.cc +++ b/src/records/RecHttp.cc @@ -885,16 +885,10 @@ convert_alpn_to_wire_format(std::string_view protocols_sv, unsigned char *wire_f Error("Unknown protocol name in configured ALPN list: \"%.*s\"", static_cast(protocol.size()), protocol.data()); return false; } - // We currently only support HTTP/1.x protocols toward the origin. - if (!HTTP_PROTOCOL_SET.contains(protocol_index)) { - Error("Unsupported non-HTTP/1.x protocol name in configured ALPN list: \"%.*s\"", static_cast(protocol.size()), - protocol.data()); - return false; - } - // But not HTTP/0.9. - if (protocol_index == TS_ALPN_PROTOCOL_INDEX_HTTP_0_9) { - Error("Unsupported \"http/0.9\" protocol name in configured ALPN list: \"%.*s\"", static_cast(protocol.size()), - protocol.data()); + // Make sure the protocol is one of our supported protocols. + if (protocol_index == TS_ALPN_PROTOCOL_INDEX_HTTP_0_9 || + (!HTTP_PROTOCOL_SET.contains(protocol_index) && !HTTP2_PROTOCOL_SET.contains(protocol_index))) { + Error("Unsupported protocol name in configured ALPN list: %.*s", static_cast(protocol.size()), protocol.data()); return false; } diff --git a/src/records/RecordsConfig.cc b/src/records/RecordsConfig.cc index f2679fdfa62..d87809d5206 100644 --- a/src/records/RecordsConfig.cc +++ b/src/records/RecordsConfig.cc @@ -1311,6 +1311,8 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.http2.no_activity_timeout_in", RECD_INT, "120", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , + {RECT_CONFIG, "proxy.config.http2.no_activity_timeout_out", RECD_INT, "120", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} + , {RECT_CONFIG, "proxy.config.http2.active_timeout_in", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , {RECT_CONFIG, "proxy.config.http2.push_diary_size", RECD_INT, "256", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} diff --git a/src/records/unit_tests/test_RecHttp.cc b/src/records/unit_tests/test_RecHttp.cc index 6dcf4f0e0f5..ce4158b8952 100644 --- a/src/records/unit_tests/test_RecHttp.cc +++ b/src/records/unit_tests/test_RecHttp.cc @@ -165,13 +165,6 @@ std::vector convertAlpnToWireFormatTestCases = 0, false }, - { - "Single protocol: HTTP/2 (currently unsupported)", - "h2", - { 0 }, - 0, - false - }, { "Single protocol: HTTP/3 (currently unsupported)", "h3", @@ -179,13 +172,6 @@ std::vector convertAlpnToWireFormatTestCases = 0, false }, - { - "Both HTTP/1.1 and HTTP/2 (HTTP/2 is currently unsupported)", - "h2,http/1.1", - { 0 }, - 0, - false - }, // -------------------------------------------------------------------------- // Happy cases. // -------------------------------------------------------------------------- @@ -197,7 +183,14 @@ std::vector convertAlpnToWireFormatTestCases = true }, { - "Multiple protocols: HTTP/1.0, HTTP/1.1", + "Single protocol: HTTP/2", + "h2", + {0x02, 'h', '2'}, + 3, + true + }, + { + "Multiple protocols: HTTP/1.1, HTTP/1.0", "http/1.1,http/1.0", {0x08, 'h', 't', 't', 'p', '/', '1', '.', '1', 0x08, 'h', 't', 't', 'p', '/', '1', '.', '0'}, 18, @@ -210,6 +203,13 @@ std::vector convertAlpnToWireFormatTestCases = 18, true }, + { + "Both HTTP/2 and HTTP/1.1", + "h2,http/1.1", + {0x02, 'h', '2', 0x08, 'h', 't', 't', 'p', '/', '1', '.', '1'}, + 12, + true + }, }; // clang-format on diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index c7a59333060..f8a085e240e 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -41,7 +41,7 @@ #include "HTTP.h" #include "ProxySession.h" #include "Http2ClientSession.h" -#include "Http1ServerSession.h" +#include "PoolableSession.h" #include "HttpSM.h" #include "HttpConfig.h" #include "P_Net.h" @@ -4948,12 +4948,11 @@ TSHttpSsnClientVConnGet(TSHttpSsn ssnp) TSVConn TSHttpSsnServerVConnGet(TSHttpSsn ssnp) { - TSVConn vconn = nullptr; PoolableSession *ss = reinterpret_cast(ssnp); if (ss != nullptr) { - vconn = reinterpret_cast(ss->get_netvc()); + return reinterpret_cast(ss->get_netvc()); } - return vconn; + return nullptr; } TSVConn @@ -7851,9 +7850,8 @@ TSHttpTxnServerFdGet(TSHttpTxn txnp, int *fdp) sdk_assert(sdk_sanity_check_txn(txnp) == TS_SUCCESS); sdk_assert(sdk_sanity_check_null_ptr((void *)fdp) == TS_SUCCESS); - HttpSM *sm = reinterpret_cast(txnp); - *fdp = -1; - + HttpSM *sm = reinterpret_cast(txnp); + *fdp = -1; TSReturnCode retval = TS_ERROR; ProxyTransaction *ss = sm->get_server_txn(); if (ss != nullptr) { diff --git a/tests/gold_tests/chunked_encoding/chunked_encoding.test.py b/tests/gold_tests/chunked_encoding/chunked_encoding.test.py index 78410d90536..6aff817da5f 100644 --- a/tests/gold_tests/chunked_encoding/chunked_encoding.test.py +++ b/tests/gold_tests/chunked_encoding/chunked_encoding.test.py @@ -50,7 +50,7 @@ "body": "knock knock"} response_header2 = {"headers": "HTTP/1.1 200 OK\r\nServer: uServer\r\nConnection: close\r\nTransfer-Encoding: chunked\r\n\r\n", "timestamp": "1415926535.898", - "body": ""} + "body": "12345678901234567890"} request_header3 = { "headers": "POST / HTTP/1.1\r\nHost: www.yetanotherexample.com\r\nContent-Type: application/x-www-form-urlencoded\r\nContent-Length: 11\r\n\r\n", diff --git a/tests/gold_tests/h2/gold/nghttp_0_stdout.gold b/tests/gold_tests/h2/gold/nghttp_0_stdout.gold index e8e9acabd41..f19a43516d3 100644 --- a/tests/gold_tests/h2/gold/nghttp_0_stdout.gold +++ b/tests/gold_tests/h2/gold/nghttp_0_stdout.gold @@ -12,5 +12,3 @@ `` [``] recv (stream_id=1) :status: 200 `` -``; END_STREAM -`` diff --git a/tests/gold_tests/h2/h2origin.test.py b/tests/gold_tests/h2/h2origin.test.py new file mode 100644 index 00000000000..e4d3de67d85 --- /dev/null +++ b/tests/gold_tests/h2/h2origin.test.py @@ -0,0 +1,94 @@ +''' +Test communication to origin with H2 +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Test communication to origin with H2 +''' + +Test.ContinueOnFail = True + +# +# Communicate to origin with HTTP/2 +# +ts = Test.MakeATSProcess("ts", enable_tls="true") + +# add ssl materials like key, certificates for the server +ts.addDefaultSSLFiles() +replay_file = "replay/" +server = Test.MakeVerifierServerProcess("h2-origin", replay_file) +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http', + 'proxy.config.exec_thread.autoconfig': 0, + # Allow for more parallelism + 'proxy.config.exec_thread.limit': 4, + 'proxy.config.ssl.client.alpn_protocols': 'h2,http/1.1', + # Sticking with thread pool because global pool does not work with h2 + 'proxy.config.http.server_session_sharing.pool': 'thread', + 'proxy.config.http.server_session_sharing.match': 'ip,sni,cert', + 'proxy.config.ssl.client.verify.server.policy': 'PERMISSIVE', +}) + +ts.Disk.remap_config.AddLine( + 'map / https://127.0.0.1:{0}'.format(server.Variables.https_port) +) +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.logging_yaml.AddLines( + ''' +logging: + formats: + - name: testformat + format: '% % % % %<{uuid}cqh> % % % % % %' + logs: + - mode: ascii + format: testformat + filename: squid +'''.split("\n") +) + +tr = Test.AddTestRun("Test traffic to origin using HTTP/2") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(ts) +tr.AddVerifierClientProcess("client", replay_file, http_ports=[ts.Variables.port], https_ports=[ts.Variables.ssl_port]) +tr.StillRunningAfter = ts +tr.TimeOut = 60 + +# Just a check to flush out the traffic log until we have a clean shutdown for traffic_server +tr = Test.AddTestRun("Wait for the access log to write out") +tr.DelayStart = 10 +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'ls' +tr.Processes.Default.ReturnCode = 0 + +# UUIDs 1-4 should be http/1.1 clients and H2 origin +# UUIDs 5-9 should be http/2 clients and H2 origins +ts.Disk.squid_log.Content = Testers.ContainsExpression(" [1-4] http/1.1 http/2", "cases 1-4 request http/1.1") +ts.Disk.squid_log.Content += Testers.ExcludesExpression(" [1-4] http/2 http/2", "cases 1-4 request http/1.1") +ts.Disk.squid_log.Content += Testers.ContainsExpression(" 1[1-4] http/1.1 http/2", "cases 12-14 request http/1.1") +ts.Disk.squid_log.Content += Testers.ExcludesExpression(" 1[2-4] http/2 http/2", "cases 12-14 request http/1.1") +ts.Disk.squid_log.Content += Testers.ContainsExpression(" [5-9] http/2 http/2", "cases 5-11 request http/2") +ts.Disk.squid_log.Content += Testers.ExcludesExpression(" [5-9] http/1.1 http/2", "cases 5-11 request http/2") +ts.Disk.squid_log.Content += Testers.ContainsExpression(" 1[0-1] http/2 http/2", "cases 5-11 request http/2") +ts.Disk.squid_log.Content += Testers.ExcludesExpression(" 1[0-1] http/1.1 http/2", "cases 5-11 request http/2") diff --git a/tests/gold_tests/h2/h2origin_single_thread.test.py b/tests/gold_tests/h2/h2origin_single_thread.test.py new file mode 100644 index 00000000000..a1f80cb74ce --- /dev/null +++ b/tests/gold_tests/h2/h2origin_single_thread.test.py @@ -0,0 +1,90 @@ +''' +Test communication to origin with H2 +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Test communication to origin with H2 +''' + +Test.ContinueOnFail = True + +# +# Communicate to origin with HTTP/2 +# +ts = Test.MakeATSProcess("ts", enable_tls="true") + +# add ssl materials like key, certificates for the server +ts.addDefaultSSLFiles() +replay_file = "replay" +server = Test.MakeVerifierServerProcess("h2-origin", replay_file) +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http', + 'proxy.config.exec_thread.autoconfig': 0, + # Limiting ourselves to 1 thread to exercise origin reuse + 'proxy.config.exec_thread.limit': 1, + 'proxy.config.ssl.client.alpn_protocols': 'h2,http/1.1', + # Sticking with hybrid pool because global pool does not work with h2 + 'proxy.config.http.server_session_sharing.pool': 'hybrid', + 'proxy.config.http.server_session_sharing.match': 'hostonly', + 'proxy.config.ssl.client.verify.server.policy': 'PERMISSIVE', +}) + +ts.Disk.remap_config.AddLine( + 'map / https://127.0.0.1:{0}'.format(server.Variables.https_port) +) +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.logging_yaml.AddLines( + ''' +logging: + formats: + - name: testformat + format: '% % % % %<{uuid}cqh> % % % % % %' + logs: + - mode: ascii + format: testformat + filename: squid +'''.split("\n") +) + +tr = Test.AddTestRun("Test traffic to origin using HTTP/2") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(ts) +tr.AddVerifierClientProcess("client", replay_file, http_ports=[ts.Variables.port], https_ports=[ts.Variables.ssl_port]) +tr.StillRunningAfter = ts + +# Just a check to flush out the traffic log until we have a clean shutdown for traffic_server +tr = Test.AddTestRun("Wait for the access log to write out") +tr.DelayStart = 10 +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'ls' +tr.Processes.Default.ReturnCode = 0 + +# UUIDs 1-4 should be http/1.1 clients and H2 origin +# UUIDs 5-9 should be http/2 clients and H2 origins +ts.Disk.squid_log.Content = Testers.ContainsExpression(" [1-4] http/1.1 http/2", "cases 1-4 request http/1.1") +ts.Disk.squid_log.Content += Testers.ExcludesExpression(" [1-4] http/2 http/2", "cases 1-4 request http/1.1") +ts.Disk.squid_log.Content += Testers.ContainsExpression(" [5-9] http/2 http/2", "cases 5-9 request http/2") +ts.Disk.squid_log.Content += Testers.ExcludesExpression(" [5-9] http/1.1 http/2", "cases 5-9 request http/2") +ts.Disk.squid_log.Content += Testers.ContainsExpression(" http/2 1 1 1 [2-9]", "At least one case of origin reuse") diff --git a/tests/gold_tests/h2/h2spec.test.py b/tests/gold_tests/h2/h2spec.test.py index 37740dd7c96..31a274496ac 100644 --- a/tests/gold_tests/h2/h2spec.test.py +++ b/tests/gold_tests/h2/h2spec.test.py @@ -50,7 +50,7 @@ 'proxy.config.http.insert_response_via_str': 1, 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), - 'proxy.config.diags.debug.enabled': 0, + 'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'http', }) diff --git a/tests/gold_tests/h2/http2.test.py b/tests/gold_tests/h2/http2.test.py index 11e6fb109b7..3d203970228 100644 --- a/tests/gold_tests/h2/http2.test.py +++ b/tests/gold_tests/h2/http2.test.py @@ -26,7 +26,7 @@ Test.SkipUnless( Condition.HasCurlFeature('http2') ) -Test.ContinueOnFail = True +#Test.ContinueOnFail = True # ---- # Setup Origin Server @@ -192,6 +192,7 @@ post_body, ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = "gold/post_chunked.gold" +tr.TimeOut = 60 tr.StillRunningAfter = server # Test Case 7: Post with big chunked body @@ -202,6 +203,7 @@ ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = "gold/post_chunked.gold" +tr.TimeOut = 60 tr.StillRunningAfter = server # Test Case 8: Huge response header @@ -211,6 +213,7 @@ tr.Processes.Default.Streams.stdout = "gold/http2_8_stdout.gold" # Different versions of curl will have different cases for HTTP/2 field names. tr.Processes.Default.Streams.stderr = Testers.GoldFile("gold/http2_8_stderr.gold", case_insensitive=True) +tr.TimeOut = 60 tr.StillRunningAfter = server # Test Case 9: Header Only Response - e.g. 204 @@ -220,4 +223,5 @@ tr.Processes.Default.Streams.stdout = "gold/http2_9_stdout.gold" # Different versions of curl will have different cases for HTTP/2 field names. tr.Processes.Default.Streams.stderr = Testers.GoldFile("gold/http2_9_stderr.gold", case_insensitive=True) +tr.TimeOut = 60 tr.StillRunningAfter = server diff --git a/tests/gold_tests/h2/httpbin.test.py b/tests/gold_tests/h2/httpbin.test.py index 96b9631c872..a759fb85fbc 100644 --- a/tests/gold_tests/h2/httpbin.test.py +++ b/tests/gold_tests/h2/httpbin.test.py @@ -30,7 +30,7 @@ Condition.HasCurlFeature('http2'), Condition.HasProgram("shasum", "shasum need to be installed on system for this test to work"), ) -Test.ContinueOnFail = True +#Test.ContinueOnFail = True # ---- # Setup httpbin Origin Server @@ -57,7 +57,7 @@ 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'http2', + 'proxy.config.diags.debug.tags': 'http', }) ts.Disk.logging_yaml.AddLines( diff --git a/tests/gold_tests/h2/replay/h1-client-h2-origin.yaml b/tests/gold_tests/h2/replay/h1-client-h2-origin.yaml new file mode 100644 index 00000000000..93fa1cff5df --- /dev/null +++ b/tests/gold_tests/h2/replay/h1-client-h2-origin.yaml @@ -0,0 +1,596 @@ +meta: + version: '1.0' + +sessions: + - protocol: + - name: http + version: '1.1' + - name: tls + version: TLSv1.3 + sni: data.brian.example.com + proxy-verify-mode: 0 + proxy-provided-cert: true + - name: tcp + - name: ip + version: '4' + + transactions: + # + # Test 1: Zero length response. + # + - all: { headers: { fields: [[ uuid, 1 ]]}} + + client-request: + protocol: + - name: http + version: '1.1' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '1.1' + scheme: https + method: GET + url: /some/path + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + proxy-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: GET + url: /some/path + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + server-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + proxy-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + + # + # Test 2: Non-zero length response. + # + - all: { headers: { fields: [[ uuid, 2 ]]}} + + client-request: + protocol: + - name: http + version: '1.1' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '1.1' + scheme: https + method: GET + url: /some/path2 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + proxy-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: GET + url: /some/path2 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + server-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 16 ] + content: + encoding: plain + size: 16 + + proxy-response: + version: '1.1' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 16 ] + content: + encoding: plain + size: 16 + + - protocol: + - name: http + version: '1.1' + - name: tls + version: TLSv1.3 + sni: data.brian.example.com + proxy-verify-mode: 0 + proxy-provided-cert: true + - name: tcp + - name: ip + version: '4' + + transactions: + # + # Test 3: 8 byte post with a 404 response. + # + - all: { headers: { fields: [[ uuid, 3 ]]}} + + client-request: + protocol: + - name: http + version: '1.1' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '1.1' + scheme: https + method: POST + url: /some/path3 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 8 ] + content: + encoding: plain + size: 8 + + proxy-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path3 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 8 ] + content: + encoding: plain + size: 8 + + server-response: + version: '2' + status: 404 + reason: "Not Found" + headers: + encoding: esc_json + fields: + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + proxy-response: + version: '2' + status: 404 + reason: "Not Found" + headers: + encoding: esc_json + fields: + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + # + # Test 4: 32 byte POST with a 200 response. + # + - all: { headers: { fields: [[ uuid, 4 ]]}} + + client-request: + protocol: + - name: http + version: '1.1' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '1.1' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 32 ] + content: + encoding: plain + size: 32 + + proxy-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 32 ] + content: + encoding: plain + size: 32 + + server-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 32 ] + content: + encoding: plain + size: 32 + + proxy-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 32 ] + content: + encoding: plain + size: 32 + + # + # Test 5: 3200 byte POST with a 200 response. + # + - all: { headers: { fields: [[ uuid, 12 ]]}} + + client-request: + protocol: + - name: http + version: '1.1' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '1.1' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 3200 ] + content: + encoding: plain + size: 3200 + + proxy-request: + protocol: + - name: http + version: '1.1' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 3200 ] + content: + encoding: plain + size: 3200 + + server-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 1600 ] + content: + encoding: plain + size: 1600 + + proxy-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 1600 ] + content: + encoding: plain + size: 1600 + + # + # Test 6: large post body small response + # + - all: { headers: { fields: [[ uuid, 13 ]]}} + + client-request: + protocol: + - name: http + version: '1.1' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '1.1' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 3200 ] + content: + encoding: plain + size: 3200 + + proxy-request: + protocol: + - name: http + version: '1.1' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '1.1' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 3200 ] + content: + encoding: plain + size: 3200 + + server-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 16 ] + content: + encoding: plain + size: 16 + + proxy-response: + version: '1.1' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 16 ] + content: + encoding: plain + size: 16 + + # + # Test 7: small post body large response + # + - all: { headers: { fields: [[ uuid, 14 ]]}} + + client-request: + protocol: + - name: http + version: '1.1' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '1.1' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 32 ] + content: + encoding: plain + size: 32 + + proxy-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 32 ] + content: + encoding: plain + size: 32 + + server-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 3200 ] + content: + encoding: plain + size: 3200 + + proxy-response: + version: '1.1' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 3200 ] + content: + encoding: plain + size: 3200 diff --git a/tests/gold_tests/h2/replay/h2-origin.yaml b/tests/gold_tests/h2/replay/h2-origin.yaml new file mode 100644 index 00000000000..65b133375a4 --- /dev/null +++ b/tests/gold_tests/h2/replay/h2-origin.yaml @@ -0,0 +1,624 @@ +meta: + version: '1.0' + +sessions: + - protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.3 + sni: data.brian.example.com + proxy-verify-mode: 0 + proxy-provided-cert: true + - name: tcp + - name: ip + version: '4' + + transactions: + # + # Test 1: Zero length response. + # + - all: { headers: { fields: [[ uuid, 5 ]]}} + + client-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: GET + url: /some/path;arg=1;arg=2?foo + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + content: + encoding: plain + size: 0 + + proxy-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: GET + url: + - [ path, { value: /some/path;arg=1;arg=2, as: equal } ] + - [ query, { value: foo, as: equal } ] + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + server-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + proxy-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + + # + # Test 2: Non-zero length response. + # + - all: { headers: { fields: [[ uuid, 6 ]]}} + + client-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: GET + url: /some/path2 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + proxy-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: GET + url: + - [ path, { value: /some/path2, as: equal } ] + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + server-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 16 ] + content: + encoding: plain + size: 16 + + proxy-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 16 ] + content: + encoding: plain + size: 16 + + - protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.3 + sni: data.brian.example.com + proxy-verify-mode: 0 + proxy-provided-cert: true + - name: tcp + - name: ip + version: '4' + + transactions: + # + # Test 3: 8 byte post with a 404 response. + # + - all: { headers: { fields: [[ uuid, 7 ]]}} + + client-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path3?foo=bar + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 8 ] + content: + encoding: plain + size: 8 + + proxy-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: + - [ path, { value: /some/path3, as: equal }] + - [ query, { value: foo=bar, as: equal }] + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 8 ] + content: + encoding: plain + size: 8 + + server-response: + version: '2' + status: 404 + reason: "Not Found" + headers: + encoding: esc_json + fields: + - [ bob, 0 ] + content: + encoding: plain + size: 0 + + proxy-response: + version: '2' + status: 404 + reason: "Not Found" + headers: + encoding: esc_json + fields: + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + proxy-response: + version: '2' + status: 404 + reason: "Not Found" + headers: + encoding: esc_json + fields: + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + # + # Test 4: 32 byte POST with a 200 response. + # + - all: { headers: { fields: [[ uuid, 8 ]]}} + + client-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 32 ] + content: + encoding: plain + size: 32 + + proxy-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 32 ] + content: + encoding: plain + size: 32 + + server-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 32 ] + content: + encoding: plain + size: 32 + + proxy-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 32 ] + content: + encoding: plain + size: 32 + + # + # Test 5: 3200 byte POST with a 200 response. + # + - all: { headers: { fields: [[ uuid, 9 ]]}} + + client-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 3200 ] + content: + encoding: plain + size: 3200 + + proxy-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 3200 ] + content: + encoding: plain + size: 3200 + + server-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ bob, 1600 ] + content: + encoding: plain + size: 1600 + + proxy-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 1600 ] + content: + encoding: plain + size: 1600 + + proxy-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 1600 ] + content: + encoding: plain + size: 1600 + + # + # Test 6: large post body small response + # + - all: { headers: { fields: [[ uuid, 10 ]]}} + + client-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + content: + encoding: plain + size: 3200 + + proxy-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 3200 ] + content: + encoding: plain + size: 3200 + + server-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 16 ] + content: + encoding: plain + size: 16 + + proxy-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 16 ] + content: + encoding: plain + size: 16 + + # + # Test 7: small post body large response + # + - all: { headers: { fields: [[ uuid, 11 ]]}} + + client-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 32 ] + content: + encoding: plain + size: 32 + + proxy-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 32 ] + content: + encoding: plain + size: 32 + + server-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 3200 ] + content: + encoding: plain + size: 3200 + + proxy-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 3200 ] + content: + encoding: plain + size: 3200 diff --git a/tests/gold_tests/post_slow_server/gold/post_slow_server_max_requests_in_0_stderr.gold b/tests/gold_tests/post_slow_server/gold/post_slow_server_max_requests_in_0_stderr.gold index 0d5e92cc8ed..3c0d989168e 100644 --- a/tests/gold_tests/post_slow_server/gold/post_slow_server_max_requests_in_0_stderr.gold +++ b/tests/gold_tests/post_slow_server/gold/post_slow_server_max_requests_in_0_stderr.gold @@ -1,5 +1,5 @@ `` > POST / HTTP/1.1 `` -< HTTP/1.1 502 Broken pipe +< HTTP/1.1 502 Connection refused `` diff --git a/tests/gold_tests/redirect/redirect_post.test.py b/tests/gold_tests/redirect/redirect_post.test.py index 21182feed14..852590e3e4e 100644 --- a/tests/gold_tests/redirect/redirect_post.test.py +++ b/tests/gold_tests/redirect/redirect_post.test.py @@ -38,7 +38,8 @@ 'proxy.config.http.number_of_redirections': MAX_REDIRECT, 'proxy.config.http.post_copy_size': 919430601, 'proxy.config.http.redirect.actions': 'self:follow', # redirects to self are not followed by default - # 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http', }) redirect_request_header = { diff --git a/tests/gold_tests/timeout/tls_conn_timeout.test.py b/tests/gold_tests/timeout/tls_conn_timeout.test.py index 86da7ecc4ee..8187220c6cf 100644 --- a/tests/gold_tests/timeout/tls_conn_timeout.test.py +++ b/tests/gold_tests/timeout/tls_conn_timeout.test.py @@ -69,7 +69,7 @@ tr.Processes.Default.Command = 'curl -H"Connection:close" -d "bob" -i http://127.0.0.1:{0}/connect_blocked --tlsv1.2'.format( ts.Variables.port) tr.Processes.Default.Streams.All = Testers.ContainsExpression( - "HTTP/1.1 502 connect failed", "Connect failed") + "HTTP/1.1 502 Connection timed out", "Connect failed") tr.Processes.Default.ReturnCode = 0 tr.StillRunningAfter = delay_post_connect tr.StillRunningAfter = Test.Processes.ts @@ -93,7 +93,7 @@ tr.Processes.Default.Command = 'curl -H"Connection:close" -i http://127.0.0.1:{0}/get_connect_blocked --tlsv1.2'.format( ts.Variables.port) tr.Processes.Default.Streams.All = Testers.ContainsExpression( - "HTTP/1.1 502 connect failed", "Connect failed") + "HTTP/1.1 502 Connection timed out", "Connect failed") tr.Processes.Default.ReturnCode = 0 tr.StillRunningAfter = delay_get_connect diff --git a/tests/gold_tests/tls/tls_client_alpn_configuration.replay.yaml b/tests/gold_tests/tls/tls_client_alpn_configuration.replay.yaml index 9ebb7adf294..ba9fbef1a73 100644 --- a/tests/gold_tests/tls/tls_client_alpn_configuration.replay.yaml +++ b/tests/gold_tests/tls/tls_client_alpn_configuration.replay.yaml @@ -44,13 +44,13 @@ sessions: fields: - [ Host, www.example.com ] - [ Content-Length, 0 ] - - [ X-Request, alpn_request ] + - [ X-Request, alpn_http1_request ] - [ uuid, first-request ] proxy-request: headers: fields: - - [ X-Request, {value: 'alpn_request', as: equal } ] + - [ X-Request, {value: 'alpn_http1_request', as: equal } ] server-response: status: 200 @@ -59,13 +59,12 @@ sessions: fields: - [ Date, "Sat, 16 Mar 2019 03:11:36 GMT" ] - [ Content-Length, 36 ] - - [ Connection, keep-alive ] - - [ X-Response, alpn_response ] + - [ X-Response, alpn_http1_response ] proxy-response: headers: fields: - - [ X-Response, {value: 'alpn_response', as: equal } ] + - [ X-Response, {value: 'alpn_http1_response', as: equal } ] # HTTP/2 over TLS. - protocol: @@ -81,32 +80,36 @@ sessions: # This test has more to do with ALPN configuration than the transactions. The # following generates a simple request and response. - client-request: - method: GET - url: /some/path/2 - version: '1.1' headers: fields: - - [ Host, www.example.com ] + - [ :method, GET ] + - [ :scheme, https ] + - [ :authority, www.example.com ] + - [ :path, /some/path/2 ] - [ Content-Length, 0 ] - - [ X-Request, alpn_request ] - - [ uuid, first-request ] + - [ X-Request, alpn_http2_request ] + - [ uuid, second-request ] + content: + encoding: plain + size: 0 proxy-request: headers: fields: - - [ X-Request, {value: 'alpn_request', as: equal } ] + - [ X-Request, {value: 'alpn_http2_request', as: equal } ] server-response: - status: 200 - reason: OK - headers: - fields: - - [ Date, "Sat, 16 Mar 2019 03:11:36 GMT" ] - - [ Content-Length, 36 ] - - [ Connection, keep-alive ] - - [ X-Response, alpn_response ] + headers: + fields: + - [ :status, 200 ] + - [ Date, "Sat, 16 Mar 2019 03:11:36 GMT" ] + - [ Content-Length, 0 ] + - [ X-Response, alpn_http2_response ] + content: + encoding: plain + size: 0 proxy-response: headers: fields: - - [ X-Response, {value: 'alpn_response', as equal } ] + - [ X-Response, {value: 'alpn_http2_response', as equal } ] diff --git a/tests/gold_tests/tls/tls_client_alpn_configuration.test.py b/tests/gold_tests/tls/tls_client_alpn_configuration.test.py index be69463219a..7d9dc4562aa 100644 --- a/tests/gold_tests/tls/tls_client_alpn_configuration.test.py +++ b/tests/gold_tests/tls/tls_client_alpn_configuration.test.py @@ -111,7 +111,7 @@ def _configure_trafficserver( "proxy.config.ssl.client.verify.server.policy": 'PERMISSIVE', 'proxy.config.diags.debug.enabled': 3, - 'proxy.config.diags.debug.tags': 'ssl', + 'proxy.config.diags.debug.tags': 'ssl|http', }) if records_config_alpn is not None: @@ -158,7 +158,14 @@ def run(self): TestAlpnFunctionality._client_counter += 1 +# +# Test default configuration. +# TestAlpnFunctionality().run() + +# +# Test various valid ALPN configurations. +# TestAlpnFunctionality( records_config_alpn='http/1.1').run() TestAlpnFunctionality( @@ -166,18 +173,18 @@ def run(self): TestAlpnFunctionality( records_config_alpn='http/1.1', conf_remap_alpn='http/1.1,http/1.0').run() +TestAlpnFunctionality( + records_config_alpn='h2,http/1.1').run() +TestAlpnFunctionality( + records_config_alpn='h2').run() -# TODO: HTTP/2 to origin comes later. -# TestAlpnFunctionality( -# records_config_alpn='h2,http1.1').run() - +# +# Test malformed ALPN configurations. +# TestAlpnFunctionality( records_config_alpn='not_a_protocol', alpn_is_malformed=True).run() - -# Since we do not currently support ALPN with HTTP/2, this will be considered a -# malformed ALPN protocol. -# TODO: remove this when we support HTTP/2 to origin. +# Note that HTTP/3 to origin is not currently supported. TestAlpnFunctionality( - records_config_alpn='h2', + records_config_alpn='h3', alpn_is_malformed=True).run() From 331367a52986d2dfdf082b4d45dab0ad788dad3b Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Fri, 11 Nov 2022 22:51:46 +0000 Subject: [PATCH 02/16] Adding window size and flow control out parameters. --- doc/admin-guide/files/records.yaml.en.rst | 43 +++ proxy/http/HttpTunnel.cc | 1 + proxy/http2/HTTP2.cc | 55 ++-- proxy/http2/HTTP2.h | 9 +- proxy/http2/Http2ConnectionState.cc | 250 ++++++++++++------ proxy/http2/Http2ConnectionState.h | 48 ++-- proxy/http2/Http2Stream.cc | 8 +- src/records/RecordsConfig.cc | 10 + .../h2/http2_flow_control.replay.yaml | 20 +- .../gold_tests/h2/http2_flow_control.test.py | 191 ++++++++----- 10 files changed, 436 insertions(+), 199 deletions(-) diff --git a/doc/admin-guide/files/records.yaml.en.rst b/doc/admin-guide/files/records.yaml.en.rst index ff2bcce9347..926f80191ca 100644 --- a/doc/admin-guide/files/records.yaml.en.rst +++ b/doc/admin-guide/files/records.yaml.en.rst @@ -4194,6 +4194,16 @@ HTTP/2 Configuration Reloading this value affects only new HTTP/2 connections, not the ones already established. +.. ts:cv:: CONFIG proxy.config.http2.max_concurrent_streams_out INT 100 + :reloadable: + + The maximum number of concurrent streams per outbound connection. + +.. note:: + + Reloading this value affects only new HTTP/2 connections, not the + ones already established. + .. ts:cv:: CONFIG proxy.config.http2.min_concurrent_streams_in INT 10 :reloadable: @@ -4201,6 +4211,13 @@ HTTP/2 Configuration This is used when :ts:cv:`proxy.config.http2.max_active_streams_in` is set larger than ``0``. +.. ts:cv:: CONFIG proxy.config.http2.min_concurrent_streams_out INT 10 + :reloadable: + + The minimum number of concurrent streams per outbound connection. + This is used when :ts:cv:`proxy.config.http2.max_active_streams_out` is set + larger than ``0``. + .. ts:cv:: CONFIG proxy.config.http2.max_active_streams_in INT 0 :reloadable: @@ -4210,6 +4227,15 @@ HTTP/2 Configuration :ts:cv:`proxy.config.http2.min_concurrent_streams_in`. To disable, set to zero (``0``). +.. ts:cv:: CONFIG proxy.config.http2.max_active_streams_out INT 0 + :reloadable: + + Limits the maximum number of connection wide active streams. + When connection wide active streams are larger than this value, + SETTINGS_MAX_CONCURRENT_STREAMS will be reduced to + :ts:cv:`proxy.config.http2.min_concurrent_streams_out`. + To disable, set to zero (``0``). + .. ts:cv:: CONFIG proxy.config.http2.initial_window_size_in INT 65535 :reloadable: :units: bytes @@ -4220,6 +4246,16 @@ HTTP/2 Configuration :ts:cv:`proxy.config.http2.flow_control.policy_in` for how HTTP/2 stream and session windows are maintained over the lifetime of HTTP/2 sessions. +.. ts:cv:: CONFIG proxy.config.http2.initial_window_size_out INT 65535 + :reloadable: + :units: bytes + + The initial HTTP/2 stream window size for outbound connections that |TS| as a + client advertises to the peer. See IETF RFC 9113 section 5.2 for details + concerning HTTP/2 flow control. See + :ts:cv:`proxy.config.http2.flow_control.policy_out` for how HTTP/2 stream and + session windows are maintained over the lifetime of HTTP/2 sessions. + .. ts:cv:: CONFIG proxy.config.http2.flow_control.policy_in INT 0 :reloadable: @@ -4249,6 +4285,13 @@ HTTP/2 Configuration a way that shares the window equally among all concurrent streams. ===== =========================================================================================== +.. ts:cv:: CONFIG proxy.config.http2.flow_control.policy_out INT 0 + :reloadable: + + Specifies the mechanism |TS| uses to maintian flow control via the HTTP/2 + stream and session windows for outbound connections. See the corresponding :ts:cv:`proxy.config.http2.flow_control.policy_in` + configuration for details concerning how this configuration variable is used. + .. ts:cv:: CONFIG proxy.config.http2.max_frame_size INT 16384 :reloadable: :units: bytes diff --git a/proxy/http/HttpTunnel.cc b/proxy/http/HttpTunnel.cc index 1c1c515670b..9d6a6b32fd7 100644 --- a/proxy/http/HttpTunnel.cc +++ b/proxy/http/HttpTunnel.cc @@ -1015,6 +1015,7 @@ HttpTunnel::producer_run(HttpTunnelProducer *p) } else { Debug("http_tunnel", "Start read vio %ld bytes", producer_n); p->read_vio = p->vc->do_io_read(this, producer_n, p->read_buffer); + p->read_vio->reenable(); } } } else { diff --git a/proxy/http2/HTTP2.cc b/proxy/http2/HTTP2.cc index 6877d1abcef..e3a5a1b806d 100644 --- a/proxy/http2/HTTP2.cc +++ b/proxy/http2/HTTP2.cc @@ -586,35 +586,45 @@ uint32_t Http2::header_table_size = 4096; uint32_t Http2::max_header_list_size = 4294967295; uint32_t Http2::accept_no_activity_timeout = 120; uint32_t Http2::no_activity_timeout_in = 120; -uint32_t Http2::no_activity_timeout_out = 120; uint32_t Http2::active_timeout_in = 0; uint32_t Http2::push_diary_size = 256; uint32_t Http2::zombie_timeout_in = 0; -float Http2::stream_error_rate_threshold = 0.1; -uint32_t Http2::stream_error_sampling_threshold = 10; -uint32_t Http2::max_settings_per_frame = 7; -uint32_t Http2::max_settings_per_minute = 14; -uint32_t Http2::max_settings_frames_per_minute = 14; -uint32_t Http2::max_ping_frames_per_minute = 60; -uint32_t Http2::max_priority_frames_per_minute = 120; -float Http2::min_avg_window_update = 2560.0; -uint32_t Http2::con_slow_log_threshold = 0; -uint32_t Http2::stream_slow_log_threshold = 0; -uint32_t Http2::header_table_size_limit = 65536; -uint32_t Http2::write_buffer_block_size = 262144; -float Http2::write_size_threshold = 0.5; -uint32_t Http2::write_time_threshold = 100; -uint32_t Http2::buffer_water_mark = 0; + +uint32_t Http2::max_concurrent_streams_out = 100; +uint32_t Http2::min_concurrent_streams_out = 10; +uint32_t Http2::max_active_streams_out = 0; +uint32_t Http2::initial_window_size_out = 65535; +Http2FlowControlPolicy Http2::flow_control_policy_out = Http2FlowControlPolicy::STATIC_SESSION_AND_STATIC_STREAM; +uint32_t Http2::no_activity_timeout_out = 120; + +float Http2::stream_error_rate_threshold = 0.1; +uint32_t Http2::stream_error_sampling_threshold = 10; +uint32_t Http2::max_settings_per_frame = 7; +uint32_t Http2::max_settings_per_minute = 14; +uint32_t Http2::max_settings_frames_per_minute = 14; +uint32_t Http2::max_ping_frames_per_minute = 60; +uint32_t Http2::max_priority_frames_per_minute = 120; +float Http2::min_avg_window_update = 2560.0; +uint32_t Http2::con_slow_log_threshold = 0; +uint32_t Http2::stream_slow_log_threshold = 0; +uint32_t Http2::header_table_size_limit = 65536; +uint32_t Http2::write_buffer_block_size = 262144; +float Http2::write_size_threshold = 0.5; +uint32_t Http2::write_time_threshold = 100; +uint32_t Http2::buffer_water_mark = 0; void Http2::init() { REC_EstablishStaticConfigInt32U(max_concurrent_streams_in, "proxy.config.http2.max_concurrent_streams_in"); REC_EstablishStaticConfigInt32U(min_concurrent_streams_in, "proxy.config.http2.min_concurrent_streams_in"); + REC_EstablishStaticConfigInt32U(max_concurrent_streams_out, "proxy.config.http2.max_concurrent_streams_out"); + REC_EstablishStaticConfigInt32U(min_concurrent_streams_out, "proxy.config.http2.min_concurrent_streams_out"); + REC_EstablishStaticConfigInt32U(max_active_streams_in, "proxy.config.http2.max_active_streams_in"); REC_EstablishStaticConfigInt32U(stream_priority_enabled, "proxy.config.http2.stream_priority_enabled"); - REC_EstablishStaticConfigInt32U(initial_window_size_in, "proxy.config.http2.initial_window_size_in"); + REC_EstablishStaticConfigInt32U(initial_window_size_in, "proxy.config.http2.initial_window_size_in"); uint32_t flow_control_policy_in_int = 0; REC_EstablishStaticConfigInt32U(flow_control_policy_in_int, "proxy.config.http2.flow_control.policy_in"); if (flow_control_policy_in_int > 2) { @@ -623,6 +633,15 @@ Http2::init() } flow_control_policy_in = static_cast(flow_control_policy_in_int); + REC_EstablishStaticConfigInt32U(initial_window_size_out, "proxy.config.http2.initial_window_size_out"); + uint32_t flow_control_policy_out_int = 0; + REC_EstablishStaticConfigInt32U(flow_control_policy_out_int, "proxy.config.http2.flow_control.policy_out"); + if (flow_control_policy_out_int > 2) { + Error("Invalid value for proxy.config.http2.flow_control.policy_out: %d", flow_control_policy_out_int); + flow_control_policy_out_int = 0; + } + flow_control_policy_out = static_cast(flow_control_policy_out_int); + REC_EstablishStaticConfigInt32U(max_frame_size, "proxy.config.http2.max_frame_size"); REC_EstablishStaticConfigInt32U(header_table_size, "proxy.config.http2.header_table_size"); REC_EstablishStaticConfigInt32U(max_header_list_size, "proxy.config.http2.max_header_list_size"); @@ -651,6 +670,8 @@ Http2::init() // If any settings is broken, ATS should not start ink_release_assert(http2_settings_parameter_is_valid({HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, max_concurrent_streams_in})); ink_release_assert(http2_settings_parameter_is_valid({HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, min_concurrent_streams_in})); + ink_release_assert(http2_settings_parameter_is_valid({HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, max_concurrent_streams_out})); + ink_release_assert(http2_settings_parameter_is_valid({HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, min_concurrent_streams_out})); ink_release_assert(http2_settings_parameter_is_valid({HTTP2_SETTINGS_INITIAL_WINDOW_SIZE, initial_window_size_in})); ink_release_assert(http2_settings_parameter_is_valid({HTTP2_SETTINGS_MAX_FRAME_SIZE, max_frame_size})); ink_release_assert(http2_settings_parameter_is_valid({HTTP2_SETTINGS_HEADER_TABLE_SIZE, header_table_size})); diff --git a/proxy/http2/HTTP2.h b/proxy/http2/HTTP2.h index 814b10ff38f..65b5413ef84 100644 --- a/proxy/http2/HTTP2.h +++ b/proxy/http2/HTTP2.h @@ -395,10 +395,17 @@ class Http2 static uint32_t max_header_list_size; static uint32_t accept_no_activity_timeout; static uint32_t no_activity_timeout_in; - static uint32_t no_activity_timeout_out; static uint32_t active_timeout_in; static uint32_t push_diary_size; static uint32_t zombie_timeout_in; + + static uint32_t max_concurrent_streams_out; + static uint32_t min_concurrent_streams_out; + static uint32_t max_active_streams_out; + static uint32_t no_activity_timeout_out; + static uint32_t initial_window_size_out; + static Http2FlowControlPolicy flow_control_policy_out; + static float stream_error_rate_threshold; static uint32_t stream_error_sampling_threshold; static uint32_t max_settings_per_frame; diff --git a/proxy/http2/Http2ConnectionState.cc b/proxy/http2/Http2ConnectionState.cc index 818358dd184..c48b3952fec 100644 --- a/proxy/http2/Http2ConnectionState.cc +++ b/proxy/http2/Http2ConnectionState.cc @@ -35,6 +35,7 @@ #include "tscpp/util/PostScript.h" #include "tscpp/util/LocalBuffer.h" +#include #include #include @@ -173,7 +174,7 @@ Http2ConnectionState::rcv_data_frame(const Http2Frame &frame) } // Check whether Window Size is acceptable - if (!this->_local_rwnd_is_shrinking_in && this->get_local_rwnd_in() < payload_length) { + if (!this->_local_rwnd_is_shrinking && this->get_local_rwnd() < payload_length) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_FLOW_CONTROL_ERROR, "recv data this->local_rwnd < payload_length"); } @@ -183,14 +184,15 @@ Http2ConnectionState::rcv_data_frame(const Http2Frame &frame) } // Update Window size - this->decrement_local_rwnd_in(payload_length); + this->decrement_local_rwnd(payload_length); stream->decrement_local_rwnd(payload_length); if (is_debug_tag_set("http2_con")) { uint32_t const stream_window = this->acknowledged_local_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE); - uint32_t const session_window = this->_get_configured_receive_session_window_size_in(); - Http2StreamDebug(this->session, id, "Received DATA frame: rwnd con=%zd/%" PRId32 " stream=%zd/%" PRId32, - this->get_local_rwnd_in(), session_window, stream->get_local_rwnd(), stream_window); + uint32_t const session_window = this->_get_configured_receive_session_window_size(); + Http2StreamDebug(this->session, id, + "Received DATA frame: payload_length=%" PRId32 " rwnd con=%zd/%" PRId32 " stream=%zd/%" PRId32, payload_length, + this->get_local_rwnd(), session_window, stream->get_local_rwnd(), stream_window); } const uint32_t unpadded_length = payload_length - pad_length; @@ -554,7 +556,7 @@ Http2ConnectionState::rcv_priority_frame(const Http2Frame &frame) // Restrict number of inactive node in dependency tree smaller than max_concurrent_streams. // Current number of inactive node is size of tree minus active node count. - if (Http2::max_concurrent_streams_in > this->dependency_tree->size() - this->get_peer_stream_count() + 1) { + if (this->_get_configured_max_concurrent_streams() > this->dependency_tree->size() - this->get_peer_stream_count() + 1) { this->dependency_tree->add(priority.stream_dependency, stream_id, priority.weight, priority.exclusive_flag, nullptr); } } @@ -710,7 +712,7 @@ Http2ConnectionState::rcv_settings_frame(const Http2Frame &frame) // windows that it maintains by the difference between the new value and // the old value. if (param.id == HTTP2_SETTINGS_INITIAL_WINDOW_SIZE) { - this->update_initial_peer_rwnd_in(param.value); + this->update_initial_peer_rwnd(param.value); } this->peer_settings.set(static_cast(param.id), param.value); @@ -863,7 +865,7 @@ Http2ConnectionState::rcv_window_update_frame(const Http2Frame &frame) if (stream_id == HTTP2_CONNECTION_CONTROL_STREAM) { // Connection level window update Http2StreamDebug(this->session, stream_id, "Received WINDOW_UPDATE frame - updated to: %zd delta: %u", - (this->get_peer_rwnd_in() + size), size); + (this->get_peer_rwnd() + size), size); // A sender MUST NOT allow a flow-control window to exceed 2^31-1 // octets. If a sender receives a WINDOW_UPDATE that causes a flow- @@ -872,12 +874,12 @@ Http2ConnectionState::rcv_window_update_frame(const Http2Frame &frame) // sends a RST_STREAM with an error code of FLOW_CONTROL_ERROR; for the // connection, a GOAWAY frame with an error code of FLOW_CONTROL_ERROR // is sent. - if (size > HTTP2_MAX_WINDOW_SIZE - this->get_peer_rwnd_in()) { + if (size > HTTP2_MAX_WINDOW_SIZE - this->get_peer_rwnd()) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_FLOW_CONTROL_ERROR, "window update too big"); } - auto error = this->increment_peer_rwnd_in(size); + auto error = this->increment_peer_rwnd(size); if (error != Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, error, "Erroneous client window update"); } @@ -915,7 +917,7 @@ Http2ConnectionState::rcv_window_update_frame(const Http2Frame &frame) return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, error, "Bad stream rwnd"); } - ssize_t wnd = std::min(this->get_peer_rwnd_in(), stream->get_peer_rwnd()); + ssize_t wnd = std::min(this->get_peer_rwnd(), stream->get_peer_rwnd()); if (wnd > 0) { SCOPED_MUTEX_LOCK(lock, stream->mutex, this_ethread()); stream->restart_sending(); @@ -1033,6 +1035,64 @@ Http2ConnectionState::rcv_continuation_frame(const Http2Frame &frame) return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); } +//////// +// Configuration Getters. +// +uint32_t +Http2ConnectionState::_get_configured_max_concurrent_streams() const +{ + ink_assert(this->session != nullptr); + if (this->session->is_outbound()) { + return Http2::max_concurrent_streams_out; + } else { + return Http2::max_concurrent_streams_in; + } +} + +uint32_t +Http2ConnectionState::_get_configured_min_concurrent_streams() const +{ + ink_assert(this->session != nullptr); + if (this->session->is_outbound()) { + return Http2::min_concurrent_streams_out; + } else { + return Http2::min_concurrent_streams_in; + } +} + +uint32_t +Http2ConnectionState::_get_configured_max_active_streams() const +{ + ink_assert(this->session != nullptr); + if (this->session->is_outbound()) { + return Http2::max_active_streams_out; + } else { + return Http2::max_active_streams_in; + } +} + +uint32_t +Http2ConnectionState::_get_configured_initial_window_size() const +{ + ink_assert(this->session != nullptr); + if (this->session->is_outbound()) { + return Http2::initial_window_size_out; + } else { + return Http2::initial_window_size_in; + } +} + +Http2FlowControlPolicy +Http2ConnectionState::_get_configured_flow_control_policy() const +{ + ink_assert(this->session != nullptr); + if (this->session->is_outbound()) { + return Http2::flow_control_policy_out; + } else { + return Http2::flow_control_policy_in; + } +} + //////// // Http2ConnectionSettings // @@ -1050,13 +1110,18 @@ Http2ConnectionSettings::Http2ConnectionSettings() } void -Http2ConnectionSettings::settings_from_configs() +Http2ConnectionSettings::settings_from_configs(bool is_outbound) { - settings[indexof(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS)] = Http2::max_concurrent_streams_in; - settings[indexof(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE)] = Http2::initial_window_size_in; - settings[indexof(HTTP2_SETTINGS_MAX_FRAME_SIZE)] = Http2::max_frame_size; - settings[indexof(HTTP2_SETTINGS_HEADER_TABLE_SIZE)] = Http2::header_table_size; - settings[indexof(HTTP2_SETTINGS_MAX_HEADER_LIST_SIZE)] = Http2::max_header_list_size; + if (is_outbound) { + settings[indexof(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS)] = Http2::max_concurrent_streams_out; + settings[indexof(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE)] = Http2::initial_window_size_out; + } else { + settings[indexof(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS)] = Http2::max_concurrent_streams_in; + settings[indexof(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE)] = Http2::initial_window_size_in; + } + settings[indexof(HTTP2_SETTINGS_MAX_FRAME_SIZE)] = Http2::max_frame_size; + settings[indexof(HTTP2_SETTINGS_HEADER_TABLE_SIZE)] = Http2::header_table_size; + settings[indexof(HTTP2_SETTINGS_MAX_HEADER_LIST_SIZE)] = Http2::max_header_list_size; } unsigned @@ -1103,23 +1168,23 @@ void Http2ConnectionState::init(Http2CommonSession *ssn) { session = ssn; - uint32_t const configured_session_window = this->_get_configured_receive_session_window_size_in(); + uint32_t const configured_session_window = this->_get_configured_receive_session_window_size(); if (configured_session_window < HTTP2_INITIAL_WINDOW_SIZE) { // There is no HTTP/2 specified way to shrink the connection window size // other than to receive data and not send WINDOW_UPDATE frames for a // while. - this->_local_rwnd_in = HTTP2_INITIAL_WINDOW_SIZE; - this->_local_rwnd_is_shrinking_in = true; + this->_local_rwnd = HTTP2_INITIAL_WINDOW_SIZE; + this->_local_rwnd_is_shrinking = true; } else { - this->_local_rwnd_in = configured_session_window; - this->_local_rwnd_is_shrinking_in = false; + this->_local_rwnd = configured_session_window; + this->_local_rwnd_is_shrinking = false; } local_hpack_handle = new HpackHandle(HTTP2_HEADER_TABLE_SIZE); peer_hpack_handle = new HpackHandle(HTTP2_HEADER_TABLE_SIZE); if (Http2::stream_priority_enabled) { - dependency_tree = new DependencyTree(Http2::max_concurrent_streams_in); + dependency_tree = new DependencyTree(this->_get_configured_max_concurrent_streams()); } _cop = ActivityCop(this->mutex, &stream_list, 1); @@ -1142,19 +1207,22 @@ Http2ConnectionState::send_connection_preface() REMEMBER(NO_EVENT, this->recursion) Http2ConnectionSettings configured_settings; - configured_settings.settings_from_configs(); + configured_settings.settings_from_configs(session->is_outbound()); - // Communicate to the peer that we do not support PUSH_PROMISE + // We do not have PUSH_PROMISE implemented, so we communicate to the peer + // that they should not send such frames to us. RFC 9113 6.5.2 says that + // servers can send this too, but they must always set a value of 0. Thus we + // send a value of 0 for both inbound and outbound connections. configured_settings.set(HTTP2_SETTINGS_ENABLE_PUSH, 0); configured_settings.set(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, _adjust_concurrent_stream()); + uint32_t const configured_initial_window_size = this->_get_configured_receive_session_window_size(); if (this->_has_dynamic_stream_window()) { // Since this is the beginning of the connection and there are no streams // yet, we can just set the stream window size to fill the entire session // window size. - uint32_t const stream_window = this->_get_configured_receive_session_window_size_in(); - configured_settings.set(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE, stream_window); + configured_settings.set(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE, configured_initial_window_size); } send_settings_frame(configured_settings); @@ -1162,8 +1230,8 @@ Http2ConnectionState::send_connection_preface() // If the session window size is non-default, send a WINDOW_UPDATE right // away. Note that there is no session window size setting in HTTP/2. The // session window size is controlled entirely by WINDOW_UPDATE frames. - if (this->_get_configured_receive_session_window_size_in() > HTTP2_INITIAL_WINDOW_SIZE) { - auto const diff = this->_get_configured_receive_session_window_size_in() - HTTP2_INITIAL_WINDOW_SIZE; + if (configured_initial_window_size > HTTP2_INITIAL_WINDOW_SIZE) { + auto const diff = configured_initial_window_size - HTTP2_INITIAL_WINDOW_SIZE; Http2ConDebug(session, "Updating the session window with a WINDOW_UPDATE frame: %u", diff); send_window_update_frame(HTTP2_CONNECTION_CONTROL_STREAM, diff); } @@ -1457,6 +1525,17 @@ Http2ConnectionState::create_initiating_stream(Http2Error &error) // Clear the session timeout. Let the transaction timeouts reign session->get_proxy_session()->cancel_inactivity_timeout(); + if (session->is_outbound() && this->_has_dynamic_stream_window()) { + // See the comment in create_stream() concerning the difference between the + // initial window size and the target window size for dynamic stream window + // sizes. + Http2ConnectionSettings new_settings = local_settings; + uint32_t const initial_stream_window_target = + this->_get_configured_receive_session_window_size() / (peer_streams_count_in.load()); + new_settings.set(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE, initial_stream_window_target); + send_settings_frame(new_settings); + } + return new_stream; } @@ -1532,7 +1611,7 @@ Http2ConnectionState::create_stream(Http2StreamId new_id, Http2Error &error) return nullptr; } - ink_assert(dynamic_cast(this->session->get_proxy_session())->is_outbound() == false); + ink_release_assert(dynamic_cast(this->session->get_proxy_session())->is_outbound() == false); uint32_t initial_stream_window = this->acknowledged_local_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE); uint32_t initial_stream_window_target = initial_stream_window; if (is_client_streamid && this->_has_dynamic_stream_window()) { @@ -1547,7 +1626,7 @@ Http2ConnectionState::create_stream(Http2StreamId new_id, Http2Error &error) // // The situation of dynamic stream window sizes is described in [RFC 9113] // 6.9.3. - initial_stream_window_target = this->_get_configured_receive_session_window_size_in() / (peer_streams_count_in.load() + 1); + initial_stream_window_target = this->_get_configured_receive_session_window_size() / (peer_streams_count_in.load() + 1); } Http2Stream *new_stream = THREAD_ALLOC_INIT(http2StreamAllocator, this_ethread(), session->get_proxy_session(), new_id, @@ -1631,14 +1710,14 @@ Http2ConnectionState::restart_streams() // Call send_response_body() for each streams while (s != end) { Http2Stream *next = static_cast(s->link.next ? s->link.next : stream_list.head); - if (std::min(this->get_peer_rwnd_in(), s->get_peer_rwnd()) > 0) { + if (std::min(this->get_peer_rwnd(), s->get_peer_rwnd()) > 0) { SCOPED_MUTEX_LOCK(lock, s->mutex, this_ethread()); s->restart_sending(); } ink_assert(s != next); s = next; } - if (std::min(this->get_peer_rwnd_in(), s->get_peer_rwnd()) > 0) { + if (std::min(this->get_peer_rwnd(), s->get_peer_rwnd()) > 0) { SCOPED_MUTEX_LOCK(lock, s->mutex, this_ethread()); s->restart_sending(); } @@ -1651,14 +1730,14 @@ void Http2ConnectionState::restart_receiving(Http2Stream *stream) { // Connection level WINDOW UPDATE - uint32_t const configured_session_window = this->_get_configured_receive_session_window_size_in(); + uint32_t const configured_session_window = this->_get_configured_receive_session_window_size(); uint32_t const min_session_window = std::min(configured_session_window, this->acknowledged_local_settings.get(HTTP2_SETTINGS_MAX_FRAME_SIZE)); - if (this->get_local_rwnd_in() < min_session_window) { - Http2WindowSize diff_size = configured_session_window - this->get_local_rwnd_in(); + if (this->get_local_rwnd() < min_session_window) { + Http2WindowSize diff_size = configured_session_window - this->get_local_rwnd(); if (diff_size > 0) { - this->increment_local_rwnd_in(diff_size); - this->_local_rwnd_is_shrinking_in = false; + this->increment_local_rwnd(diff_size); + this->_local_rwnd_is_shrinking = false; this->send_window_update_frame(HTTP2_CONNECTION_CONTROL_STREAM, diff_size); } } @@ -1670,12 +1749,8 @@ Http2ConnectionState::restart_receiving(Http2Stream *stream) return; } - // If read_vio is buffering data, do not fully update window uint32_t const initial_stream_window = this->acknowledged_local_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE); int64_t data_size = stream->read_vio_read_avail(); - if (data_size >= initial_stream_window) { - return; - } Http2WindowSize diff_size = 0; if (stream->get_local_rwnd() < 0) { @@ -1685,7 +1760,7 @@ Http2ConnectionState::restart_receiving(Http2Stream *stream) // target initial_stream_window size. diff_size = initial_stream_window - stream->get_local_rwnd(); } else { - diff_size = initial_stream_window - std::max(static_cast(stream->get_local_rwnd()), data_size); + diff_size = initial_stream_window - std::min(static_cast(stream->get_local_rwnd()), data_size); } // Dynamic stream window sizes may result in negative values. In this case, @@ -1821,7 +1896,7 @@ Http2ConnectionState::release_stream() } void -Http2ConnectionState::update_initial_peer_rwnd_in(Http2WindowSize new_size) +Http2ConnectionState::update_initial_peer_rwnd(Http2WindowSize new_size) { // Update stream level window sizes for (Http2Stream *s = stream_list.head; s; s = static_cast(s->link.next)) { @@ -1842,7 +1917,7 @@ Http2ConnectionState::update_initial_peer_rwnd_in(Http2WindowSize new_size) } void -Http2ConnectionState::update_initial_local_rwnd_in(Http2WindowSize new_size) +Http2ConnectionState::update_initial_local_rwnd(Http2WindowSize new_size) { // Update stream level window sizes for (Http2Stream *s = stream_list.head; s; s = static_cast(s->link.next)) { @@ -1887,7 +1962,7 @@ Http2ConnectionState::send_data_frames_depends_on_priority() Http2DependencyTree::Node *node = dependency_tree->top(); // No node to send or no connection level window left - if (node == nullptr || _peer_rwnd_in <= 0) { + if (node == nullptr || _peer_rwnd <= 0) { return; } @@ -1930,7 +2005,7 @@ Http2ConnectionState::send_data_frames_depends_on_priority() Http2SendDataFrameResult Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_length) { - const ssize_t window_size = std::min(this->get_peer_rwnd_in(), stream->get_peer_rwnd()); + const ssize_t window_size = std::min(this->get_peer_rwnd(), stream->get_peer_rwnd()); const size_t buf_len = BUFFER_SIZE_FOR_INDEX(buffer_size_index[HTTP2_FRAME_TYPE_DATA]); const size_t write_available_size = std::min(buf_len, static_cast(window_size)); payload_length = 0; @@ -1952,7 +2027,7 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len if (session->is_outbound()) { ip_port_text_buffer ipb; const char *client_ip = ats_ip_ntop(session->get_proxy_session()->get_remote_addr(), ipb, sizeof(ipb)); - Warning("No window server_ip=%s session_wnd=%zd stream_wnd=%zd peer_initial_window=%u", client_ip, get_peer_rwnd_in(), + Warning("No window server_ip=%s session_wnd=%zd stream_wnd=%zd peer_initial_window=%u", client_ip, get_peer_rwnd(), stream->get_peer_rwnd(), this->peer_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE)); } Http2StreamDebug(this->session, stream->get_id(), "No window"); @@ -1969,8 +2044,11 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len payload_length = 0; } + // For HTTP/2 sessions, is_write_high_water() returning true correlates to + // our write buffer exceeding HTTP2_SETTINGS_MAX_FRAME_SIZE. Thus we will + // hold off on processing the payload until the write buffer is drained. if (payload_length > 0 && this->session->is_write_high_water()) { - Http2StreamDebug(this->session, stream->get_id(), "Not write avail"); + Http2StreamDebug(this->session, stream->get_id(), "Not write avail, payload_length=%zu", payload_length); this->session->flush(); return Http2SendDataFrameResult::NOT_WRITE_AVAIL; } @@ -1992,15 +2070,15 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len } // Update window size - this->decrement_peer_rwnd_in(payload_length); + this->decrement_peer_rwnd(payload_length); stream->decrement_peer_rwnd(payload_length); // Create frame Http2StreamDebug(session, stream->get_id(), "Send a DATA frame - client window con: %5zd stream: %5zd payload: %5zd flags: 0x%x", - _peer_rwnd_in, stream->get_peer_rwnd(), payload_length, flags); + _peer_rwnd, stream->get_peer_rwnd(), payload_length, flags); Http2DataFrame data(stream->get_id(), flags, resp_reader, payload_length); - this->session->xmit(data, flags & HTTP2_FLAGS_DATA_END_STREAM); + this->session->xmit(data); if (flags & HTTP2_FLAGS_DATA_END_STREAM) { Http2StreamDebug(session, stream->get_id(), "END_STREAM"); @@ -2176,6 +2254,8 @@ Http2ConnectionState::send_push_promise_frame(Http2Stream *stream, URL &url, con int payload_length = 0; uint8_t flags = 0x00; + // It makes no sense to send a PUSH_PROMISE toward the server. + ink_release_assert(!this->session->is_outbound()); if (peer_settings.get(HTTP2_SETTINGS_ENABLE_PUSH) == 0) { return false; } @@ -2331,7 +2411,7 @@ Http2ConnectionState::send_settings_frame(const Http2ConnectionSettings &new_set Http2SettingsFrame settings(stream_id, HTTP2_FRAME_NO_FLAG, params, params_size); - this->_outstanding_settings_frames_in.emplace(new_settings); + this->_outstanding_settings_frames.emplace(new_settings); this->session->xmit(settings, true); } @@ -2340,12 +2420,12 @@ Http2ConnectionState::_process_incoming_settings_ack_frame() { constexpr Http2StreamId stream_id = HTTP2_CONNECTION_CONTROL_STREAM; Http2StreamDebug(session, stream_id, "Processing SETTINGS ACK frame with a queue size of %zu", - this->_outstanding_settings_frames_in.size()); + this->_outstanding_settings_frames.size()); // Do not update this->acknowledged_local_settings yet as - // update_initial_local_rwnd_in relies upon it still pointing to the old value. + // update_initial_local_rwnd relies upon it still pointing to the old value. Http2ConnectionSettings const &old_settings = this->acknowledged_local_settings; - Http2ConnectionSettings const &new_settings = this->_outstanding_settings_frames_in.front().get_outstanding_settings(); + Http2ConnectionSettings const &new_settings = this->_outstanding_settings_frames.front().get_outstanding_settings(); for (int i = HTTP2_SETTINGS_HEADER_TABLE_SIZE; i < HTTP2_SETTINGS_MAX; ++i) { Http2SettingsIdentifier id = static_cast(i); @@ -2361,11 +2441,11 @@ Http2ConnectionState::_process_incoming_settings_ack_frame() if (id == HTTP2_SETTINGS_INITIAL_WINDOW_SIZE) { // Update all the streams for the newly acknowledged window size. - this->update_initial_local_rwnd_in(new_value); + this->update_initial_local_rwnd(new_value); } } this->acknowledged_local_settings = new_settings; - this->_outstanding_settings_frames_in.pop(); + this->_outstanding_settings_frames.pop(); } void @@ -2466,9 +2546,13 @@ Http2ConnectionState::get_received_priority_frame_count() unsigned Http2ConnectionState::_adjust_concurrent_stream() { - if (Http2::max_active_streams_in == 0) { + uint32_t const max_concurrent_streams = this->_get_configured_max_concurrent_streams(); + uint32_t const max_active_streams = this->_get_configured_max_active_streams(); + uint32_t const min_concurrent_streams = this->_get_configured_min_concurrent_streams(); + + if (max_active_streams == 0) { // Throttling down is disabled. - return Http2::max_concurrent_streams_in; + return max_concurrent_streams; } int64_t current_client_streams = 0; @@ -2476,43 +2560,43 @@ Http2ConnectionState::_adjust_concurrent_stream() Http2ConDebug(session, "current client streams: %" PRId64, current_client_streams); - if (current_client_streams >= Http2::max_active_streams_in) { + if (current_client_streams >= max_active_streams) { if (!Http2::throttling) { Warning("too many streams: %" PRId64 ", reduce SETTINGS_MAX_CONCURRENT_STREAMS to %d", current_client_streams, - Http2::min_concurrent_streams_in); + min_concurrent_streams); Http2::throttling = true; } - return Http2::min_concurrent_streams_in; + return min_concurrent_streams; } else { if (Http2::throttling) { - Note("revert SETTINGS_MAX_CONCURRENT_STREAMS to %d", Http2::max_concurrent_streams_in); + Note("revert SETTINGS_MAX_CONCURRENT_STREAMS to %d", max_concurrent_streams); Http2::throttling = false; } } - return Http2::max_concurrent_streams_in; + return max_concurrent_streams; } uint32_t -Http2ConnectionState::_get_configured_receive_session_window_size_in() const +Http2ConnectionState::_get_configured_receive_session_window_size() const { - switch (Http2::flow_control_policy_in) { + switch (this->_get_configured_flow_control_policy()) { case Http2FlowControlPolicy::STATIC_SESSION_AND_STATIC_STREAM: - return Http2::initial_window_size_in; + return this->_get_configured_initial_window_size(); case Http2FlowControlPolicy::LARGE_SESSION_AND_STATIC_STREAM: case Http2FlowControlPolicy::LARGE_SESSION_AND_DYNAMIC_STREAM: - return Http2::initial_window_size_in * Http2::max_concurrent_streams_in; + return this->_get_configured_initial_window_size() * this->_get_configured_max_concurrent_streams(); } // This is unreachable, but adding a return here quiets a compiler warning. - return Http2::initial_window_size_in; + return this->_get_configured_initial_window_size(); } bool Http2ConnectionState::_has_dynamic_stream_window() const { - switch (Http2::flow_control_policy_in) { + switch (this->_get_configured_flow_control_policy()) { case Http2FlowControlPolicy::STATIC_SESSION_AND_STATIC_STREAM: case Http2FlowControlPolicy::LARGE_SESSION_AND_STATIC_STREAM: return false; @@ -2525,15 +2609,15 @@ Http2ConnectionState::_has_dynamic_stream_window() const } ssize_t -Http2ConnectionState::get_peer_rwnd_in() const +Http2ConnectionState::get_peer_rwnd() const { - return this->_peer_rwnd_in; + return this->_peer_rwnd; } Http2ErrorCode -Http2ConnectionState::increment_peer_rwnd_in(size_t amount) +Http2ConnectionState::increment_peer_rwnd(size_t amount) { - this->_peer_rwnd_in += amount; + this->_peer_rwnd += amount; this->_recent_rwnd_increment[this->_recent_rwnd_increment_index] = amount; ++this->_recent_rwnd_increment_index; @@ -2549,28 +2633,28 @@ Http2ConnectionState::increment_peer_rwnd_in(size_t amount) } Http2ErrorCode -Http2ConnectionState::decrement_peer_rwnd_in(size_t amount) +Http2ConnectionState::decrement_peer_rwnd(size_t amount) { - this->_peer_rwnd_in -= amount; + this->_peer_rwnd -= amount; return Http2ErrorCode::HTTP2_ERROR_NO_ERROR; } ssize_t -Http2ConnectionState::get_local_rwnd_in() const +Http2ConnectionState::get_local_rwnd() const { - return this->_local_rwnd_in; + return this->_local_rwnd; } Http2ErrorCode -Http2ConnectionState::increment_local_rwnd_in(size_t amount) +Http2ConnectionState::increment_local_rwnd(size_t amount) { - this->_local_rwnd_in += amount; + this->_local_rwnd += amount; return Http2ErrorCode::HTTP2_ERROR_NO_ERROR; } Http2ErrorCode -Http2ConnectionState::decrement_local_rwnd_in(size_t amount) +Http2ConnectionState::decrement_local_rwnd(size_t amount) { - this->_local_rwnd_in -= amount; + this->_local_rwnd -= amount; return Http2ErrorCode::HTTP2_ERROR_NO_ERROR; } diff --git a/proxy/http2/Http2ConnectionState.h b/proxy/http2/Http2ConnectionState.h index 0784d4cd3e9..fe62136ed92 100644 --- a/proxy/http2/Http2ConnectionState.h +++ b/proxy/http2/Http2ConnectionState.h @@ -53,7 +53,7 @@ class Http2ConnectionSettings public: Http2ConnectionSettings(); - void settings_from_configs(); + void settings_from_configs(bool is_outbound); unsigned get(Http2SettingsIdentifier id) const; unsigned set(Http2SettingsIdentifier id, unsigned value); @@ -133,10 +133,10 @@ class Http2ConnectionState : public Continuation void restart_receiving(Http2Stream *stream); /** Update all streams for the peer's newly dictated stream window size. */ - void update_initial_peer_rwnd_in(Http2WindowSize new_size); + void update_initial_peer_rwnd(Http2WindowSize new_size); /** Update all streams for our newly dictated stream window size. */ - void update_initial_local_rwnd_in(Http2WindowSize new_size); + void update_initial_local_rwnd(Http2WindowSize new_size); Http2StreamId get_latest_stream_id_in() const; Http2StreamId get_latest_stream_id_out() const; @@ -196,12 +196,12 @@ class Http2ConnectionState : public Continuation void increment_received_priority_frame_count(); uint32_t get_received_priority_frame_count(); - ssize_t get_peer_rwnd_in() const; - Http2ErrorCode increment_peer_rwnd_in(size_t amount); - Http2ErrorCode decrement_peer_rwnd_in(size_t amount); - ssize_t get_local_rwnd_in() const; - Http2ErrorCode increment_local_rwnd_in(size_t amount); - Http2ErrorCode decrement_local_rwnd_in(size_t amount); + ssize_t get_peer_rwnd() const; + Http2ErrorCode increment_peer_rwnd(size_t amount); + Http2ErrorCode decrement_peer_rwnd(size_t amount); + ssize_t get_local_rwnd() const; + Http2ErrorCode increment_local_rwnd(size_t amount); + Http2ErrorCode decrement_local_rwnd(size_t amount); bool no_streams() const; bool single_stream() const; @@ -241,13 +241,22 @@ class Http2ConnectionState : public Continuation */ void _process_incoming_settings_ack_frame(); - /** Calculate the initial session window size that we communicate to peers. + // Getters for stream control configurations that retrieve the inbound or + // outbound values per the configured session. + uint32_t _get_configured_max_concurrent_streams() const; + uint32_t _get_configured_min_concurrent_streams() const; + uint32_t _get_configured_max_active_streams() const; + uint32_t _get_configured_initial_window_size() const; + Http2FlowControlPolicy _get_configured_flow_control_policy() const; + + /** Calculate the initial session window size that we communicate to inbound + * peers. * * @return The initial receive window size. */ - uint32_t _get_configured_receive_session_window_size_in() const; + uint32_t _get_configured_receive_session_window_size() const; - /** Whether our stream window can change over the lifetime of a session. + /** Whether the stream window can change over the lifetime of a session. * * @return @c true if the stream window can change, @c false otherwise. */ @@ -279,8 +288,8 @@ class Http2ConnectionState : public Continuation // Connection level window size - /** The client-side session level window that we have to respect when we send - * data to the peer. + /** The session level window that we have to respect when we send data to the + * peer. * * This is the session window configured by the peer via WINDOW_UPDATE * frames. Per specification, this defaults to HTTP2_INITIAL_WINDOW_SIZE (see @@ -289,17 +298,16 @@ class Http2ConnectionState : public Continuation * specification. When we receive WINDOW_UPDATE frames, we increment this * value. */ - ssize_t _peer_rwnd_in = HTTP2_INITIAL_WINDOW_SIZE; + ssize_t _peer_rwnd = HTTP2_INITIAL_WINDOW_SIZE; - /** The session window we maintain with the client-side peer via - * WINDOW_UPDATE frames. + /** The session window we maintain with the peer via WINDOW_UPDATE frames. * * We maintain the window we expect the peer to respect by sending * WINDOW_UPDATE frames to the peer. As we receive data, we decrement this * value, as we send WINDOW_UPDATE frames, we increment it. If it reaches * zero, we generate a connection-level error. */ - ssize_t _local_rwnd_in = 0; + ssize_t _local_rwnd = 0; /** Whether the client-side session window is in a shrinking state before we * send the first WINDOW_UPDATE frame. @@ -311,7 +319,7 @@ class Http2ConnectionState : public Continuation * window gets to the desired size, we start maintaining the window via * WINDOW_UPDATE frames. */ - bool _local_rwnd_is_shrinking_in = false; + bool _local_rwnd_is_shrinking = false; std::array _recent_rwnd_increment = {SIZE_MAX, SIZE_MAX, SIZE_MAX, SIZE_MAX, SIZE_MAX}; int _recent_rwnd_increment_index = 0; @@ -367,7 +375,7 @@ class Http2ConnectionState : public Continuation /** The queue of SETTINGS frames that we have sent but have not yet been * acknowledged by the peer. */ - std::queue _outstanding_settings_frames_in; + std::queue _outstanding_settings_frames; // NOTE: Id of stream which MUST receive CONTINUATION frame. // - [RFC 7540] 6.2 HEADERS diff --git a/proxy/http2/Http2Stream.cc b/proxy/http2/Http2Stream.cc index c620b70b6d2..ae3275ce96d 100644 --- a/proxy/http2/Http2Stream.cc +++ b/proxy/http2/Http2Stream.cc @@ -550,7 +550,7 @@ Http2Stream::initiating_close() SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); REMEMBER(NO_EVENT, this->reentrancy_count); Http2StreamDebug("initiating_close client_window=%" PRId64 " session_window=%" PRId64, _peer_rwnd, - this->get_connection_state().get_peer_rwnd_in()); + this->get_connection_state().get_peer_rwnd()); // Set the state of the connection to closed // TODO - these states should be combined @@ -726,12 +726,12 @@ Http2Stream::update_write_request(bool call_update) IOBufferReader *vio_reader = write_vio.get_reader(); if (write_vio.ntodo() > 0 && (!vio_reader->is_read_avail_more_than(0) || - // If there is no window left, just give up now too - std::min(_peer_rwnd, this->get_connection_state().get_peer_rwnd_in()) == 0)) { + // If there is no window left, just give up now too until we receive a WINDOW_UPDATE. + std::min(_peer_rwnd, this->get_connection_state().get_peer_rwnd()) == 0)) { Http2StreamDebug("update_write_request give up without doing anything ntodo=%" PRId64 " is_read_avail=%d client_window=%" PRId64 " session_window=%" PRId64, write_vio.ntodo(), vio_reader->is_read_avail_more_than(0), _peer_rwnd, - this->get_connection_state().get_peer_rwnd_in()); + this->get_connection_state().get_peer_rwnd()); return; } diff --git a/src/records/RecordsConfig.cc b/src/records/RecordsConfig.cc index d87809d5206..e7106a6812a 100644 --- a/src/records/RecordsConfig.cc +++ b/src/records/RecordsConfig.cc @@ -1293,14 +1293,24 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.http2.max_concurrent_streams_in", RECD_INT, "100", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , + {RECT_CONFIG, "proxy.config.http2.max_concurrent_streams_out", RECD_INT, "100", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} + , {RECT_CONFIG, "proxy.config.http2.min_concurrent_streams_in", RECD_INT, "10", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , + {RECT_CONFIG, "proxy.config.http2.min_concurrent_streams_out", RECD_INT, "10", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} + , {RECT_CONFIG, "proxy.config.http2.max_active_streams_in", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , + {RECT_CONFIG, "proxy.config.http2.max_active_streams_out", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} + , {RECT_CONFIG, "proxy.config.http2.initial_window_size_in", RECD_INT, "65535", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , + {RECT_CONFIG, "proxy.config.http2.initial_window_size_out", RECD_INT, "65535", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} + , {RECT_CONFIG, "proxy.config.http2.flow_control.policy_in", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "[0-2]", RECA_NULL} , + {RECT_CONFIG, "proxy.config.http2.flow_control.policy_out", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "[0-2]", RECA_NULL} + , {RECT_CONFIG, "proxy.config.http2.max_frame_size", RECD_INT, "16384", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , {RECT_CONFIG, "proxy.config.http2.header_table_size", RECD_INT, "4096", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} diff --git a/tests/gold_tests/h2/http2_flow_control.replay.yaml b/tests/gold_tests/h2/http2_flow_control.replay.yaml index f8eca55d59e..4714ad26a4d 100644 --- a/tests/gold_tests/h2/http2_flow_control.replay.yaml +++ b/tests/gold_tests/h2/http2_flow_control.replay.yaml @@ -88,9 +88,9 @@ sessions: headers: fields: - [ X-Response, first-response ] - - [ Content-Length, 28 ] + - [ Content-Length, 1200 ] content: - size: 28 + size: 1200 proxy-response: headers: @@ -120,9 +120,9 @@ sessions: headers: fields: - [ X-Response, second-response ] - - [ Content-Length, 28 ] + - [ Content-Length, 1200 ] content: - size: 28 + size: 1200 proxy-response: headers: @@ -152,9 +152,9 @@ sessions: headers: fields: - [ X-Response, third-response ] - - [ Content-Length, 28 ] + - [ Content-Length, 1200 ] content: - size: 28 + size: 1200 proxy-response: headers: @@ -190,9 +190,9 @@ sessions: headers: fields: - [ X-Response, fourth-response ] - - [ Content-Length, 28 ] + - [ Content-Length, 120000 ] content: - size: 28 + size: 120000 proxy-response: headers: @@ -226,9 +226,9 @@ sessions: headers: fields: - [ X-Response, fifth-response ] - - [ Content-Length, 28 ] + - [ Content-Length, 10000 ] content: - size: 28 + size: 10000 proxy-response: headers: diff --git a/tests/gold_tests/h2/http2_flow_control.test.py b/tests/gold_tests/h2/http2_flow_control.test.py index 03e1058f47b..ae4f8c2a665 100644 --- a/tests/gold_tests/h2/http2_flow_control.test.py +++ b/tests/gold_tests/h2/http2_flow_control.test.py @@ -33,7 +33,7 @@ class Http2FlowControlTest: _flow_control_policy_is_malformed: bool = False _default_initial_window_size: int = 65535 - _default_max_concurrent_streams_in: int = 100 + _default_max_concurrent_streams: int = 100 _default_flow_control_policy: int = 0 _dns_counter: int = 0 @@ -41,28 +41,34 @@ class Http2FlowControlTest: _ts_counter: int = 0 _client_counter: int = 0 + IS_OUTBOUND = True + IS_INBOUND = False + + IS_HTTP2_TO_ORIGIN = True + IS_HTTP1_TO_ORIGIN = False + def __init__( self, description: str, initial_window_size: Optional[int] = None, - max_concurrent_streams_in: Optional[int] = None, + max_concurrent_streams: Optional[int] = None, flow_control_policy: Optional[int] = None): """Declare the various test Processes. :param description: A description of the test. :param initial_window_size: The value with which to configure the - proxy.config.http2.initial_window_size_in ATS parameter in the + proxy.config.http2.initial_window_size_(in|out) ATS parameter in the records.yaml file. If the paramenter is None, then no window size will be explicitly set and ATS will use the default value. - :param max_concurrent_streams_in: The value with which to configure the - proxy.config.http2.max_concurrent_streams_in ATS parameter in the + :param max_concurrent_streams: The value with which to configure the + proxy.config.http2.max_concurrent_streams_(in|out) ATS parameter in the records.yaml file. If the paramenter is None, then no window size will be explicitly set and ATS will use the default value. :param flow_control_policy: The value with which to configure the - proxy.config.http2.flow_control.policy_in ATS parameter the + proxy.config.http2.flow_control.policy_(in|out) ATS parameter the records.yaml file. If the paramenter is None, then no policy configuration will be explicitly set and ATS will use the default value. @@ -74,10 +80,10 @@ def __init__( initial_window_size if initial_window_size is not None else self._default_initial_window_size) - self._max_concurrent_streams_in = max_concurrent_streams_in - self._expected_max_concurrent_streams_in = ( - max_concurrent_streams_in if max_concurrent_streams_in is not None - else self._default_max_concurrent_streams_in) + self._max_concurrent_streams = max_concurrent_streams + self._expected_max_concurrent_streams = ( + max_concurrent_streams if max_concurrent_streams is not None + else self._default_max_concurrent_streams) self._flow_control_policy = flow_control_policy self._expected_flow_control_policy = ( @@ -88,27 +94,23 @@ def __init__( self._flow_control_policy is not None and self._flow_control_policy not in self._valid_policy_values) - self._dns = self._configure_dns() - self._server = self._configure_server() - self._ts = self._configure_trafficserver() - - def _configure_dns(self): + def _configure_dns(self, tr: 'TestRun') -> 'Process': """Configure the DNS.""" - dns = Test.MakeDNServer(f'dns-{Http2FlowControlTest._dns_counter}') + dns = tr.MakeDNServer(f'dns-{Http2FlowControlTest._dns_counter}') Http2FlowControlTest._dns_counter += 1 return dns - def _configure_server(self): + def _configure_server(self, tr: 'TestRun') -> 'Process': """Configure the test server.""" - server = Test.MakeVerifierServerProcess( + server = tr.AddVerifierServerProcess( f'server-{Http2FlowControlTest._server_counter}', self._replay_file) Http2FlowControlTest._server_counter += 1 return server - def _configure_trafficserver(self): + def _configure_trafficserver(self, tr: 'TestRun', is_outbound: bool, is_http2_to_orign: bool) -> 'Process': """Configure a Traffic Server process.""" - ts = Test.MakeATSProcess( + ts = tr.MakeATSProcess( f'ts-{Http2FlowControlTest._ts_counter}', enable_tls=True, enable_cache=False) @@ -125,19 +127,36 @@ def _configure_trafficserver(self): 'proxy.config.diags.debug.tags': 'http', }) + if is_http2_to_orign: + ts.Disk.records_config.update({ + 'proxy.config.ssl.client.alpn_protocols': 'h2,http/1.1', + }) + if self._initial_window_size is not None: + if is_outbound: + configuration = 'proxy.config.http2.initial_window_size_out' + else: + configuration = 'proxy.config.http2.initial_window_size_in' ts.Disk.records_config.update({ - 'proxy.config.http2.initial_window_size_in': self._initial_window_size, + configuration: self._initial_window_size, }) if self._flow_control_policy is not None: + if is_outbound: + configuration = 'proxy.config.http2.flow_control.policy_out' + else: + configuration = 'proxy.config.http2.flow_control.policy_in' ts.Disk.records_config.update({ - 'proxy.config.http2.flow_control.policy_in': self._flow_control_policy, + configuration: self._flow_control_policy, }) - if self._max_concurrent_streams_in is not None: + if self._max_concurrent_streams is not None: + if is_outbound: + configuration = 'proxy.config.http2.max_concurrent_streams_out' + else: + configuration = 'proxy.config.http2.max_concurrent_streams_in' ts.Disk.records_config.update({ - 'proxy.config.http2.max_concurrent_streams_in': self._max_concurrent_streams_in, + configuration: self._max_concurrent_streams, }) ts.Disk.ssl_multicert_config.AddLine( @@ -145,13 +164,17 @@ def _configure_trafficserver(self): ) ts.Disk.remap_config.AddLine( - f'map / http://127.0.0.1:{self._server.Variables.http_port}' + f'map / https://127.0.0.1:{self._server.Variables.https_port}' ) if self._flow_control_policy_is_malformed: + if is_outbound: + configuration = 'proxy.config.http2.flow_control.policy_out' + else: + configuration = 'proxy.config.http2.flow_control.policy_in' ts.Disk.diags_log.Content = Testers.ContainsExpression( - "ERROR.*proxy.config.http2.flow_control.policy_in", - "There should be an about an invalid flow control policy.") + f"ERROR.*{configuration}", + "There should be an error about an invalid flow control policy.") return ts @@ -166,36 +189,39 @@ def _configure_client(self, tr): https_ports=[self._ts.Variables.ssl_port]) Http2FlowControlTest._client_counter += 1 + def _configure_log_expectations(self, host): + """Configure the log expectations for the client or server.""" + hostname = "server" if host == self._server else "client" if self._flow_control_policy_is_malformed: # Since we're just testing ATS configuration errors, there's no # need to set up client expectations. return # ATS currently always sends a MAX_CONCURRENT_STREAMS setting. - tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( - f'MAX_CONCURRENT_STREAMS:{self._expected_max_concurrent_streams_in}', - "Client should receive a MAX_CONCURRENT_STREAMS setting.") + host.Streams.stdout += Testers.ContainsExpression( + f'MAX_CONCURRENT_STREAMS:{self._expected_max_concurrent_streams}', + f"{hostname} should receive a MAX_CONCURRENT_STREAMS setting.") if self._initial_window_size is not None: - tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( + host.Streams.stdout += Testers.ContainsExpression( f'INITIAL_WINDOW_SIZE:{self._expected_initial_stream_window_size}', - "Client should receive an INITIAL_WINDOW_SIZE setting.") + f"{hostname} should receive an INITIAL_WINDOW_SIZE setting.") if self._expected_flow_control_policy == 0: update_window_size = ( self._expected_initial_stream_window_size - self._default_initial_window_size) if update_window_size > 0: - tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( + host.Streams.stdout += Testers.ContainsExpression( f'WINDOW_UPDATE.*id 0: {update_window_size}', - "Client should receive a session WINDOW_UPDATE.") + f"{hostname} should receive a session WINDOW_UPDATE.") if self._expected_flow_control_policy in (1, 2): # Verify the larger window size. session_window_size = ( self._expected_initial_stream_window_size * - self._expected_max_concurrent_streams_in) + self._expected_max_concurrent_streams) # ATS will send a WINDOW_UPDATE frame to the client to increase # the session window size to the configured value from the default @@ -206,9 +232,9 @@ def _configure_client(self, tr): # A WINDOW_UPDATE can only increase the window size. So make sure that # the new window size is greater than the default window size. if update_window_size > Http2FlowControlTest._default_initial_window_size: - tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( + host.Streams.stdout += Testers.ContainsExpression( f'WINDOW_UPDATE.*id 0: {update_window_size}', - "Client should receive an initial session WINDOW_UPDATE.") + f"{hostname} should receive an initial session WINDOW_UPDATE.") else: # Our test traffic is large enough that eventually we should # send a session WINDOW_UPDATE frame for the smaller window. @@ -216,47 +242,83 @@ def _configure_client(self, tr): # session window may not receive a 100 byte WINDOW_UPDATE frame # if the client is sending DATA frames in 10 byte chunks due to # a smaller stream window. - tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( + host.Streams.stdout += Testers.ContainsExpression( 'WINDOW_UPDATE.*id 0: ', - "Client should receive a session WINDOW_UPDATE.") + f"{hostname} should receive a session WINDOW_UPDATE.") if self._expected_flow_control_policy == 2: # Verify the streams window sizes get updated. stream_window_1 = session_window_size stream_window_2 = int(session_window_size / 2) stream_window_3 = int(session_window_size / 3) - tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( + host.Streams.stdout += Testers.ContainsExpression( (f'INITIAL_WINDOW_SIZE:{stream_window_1}.*' f'INITIAL_WINDOW_SIZE:{stream_window_2}.*' f'INITIAL_WINDOW_SIZE:{stream_window_3}'), - "Client should stream receive window updates", + f"{hostname} should stream receive window updates", reflags=re.DOTALL | re.MULTILINE) if self._expected_initial_stream_window_size < 1000: + first_id = 5 if self._server else 3 + + # WINDOW_UPDATE timing is different between the server and the + # client. Toward the origin we send SETTINGS frames to update the + # INITIAL_WINDOW_SIZE with the headers so they are received earlier + # than with the client, wherein we send the updated + # INITIAL_WINDOW_SIZE after receiving headers from the client. + if self._server and self._expected_flow_control_policy == 2: + window_update_size = 33 + else: + window_update_size = self._expected_initial_stream_window_size # For the smaller session window sizes, we expect WINDOW_UPDATE frames. - tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( - f'WINDOW_UPDATE.*id 3: {self._expected_initial_stream_window_size}', - "Client should receive a stream WINDOW_UPDATE.") - - tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( - f'WINDOW_UPDATE.*id 5: {self._expected_initial_stream_window_size}', - "Client should receive a stream WINDOW_UPDATE.") - - tr.Processes.Default.Streams.stdout += Testers.ContainsExpression( - f'WINDOW_UPDATE.*id 7: {self._expected_initial_stream_window_size}', - "Client should receive a stream WINDOW_UPDATE.") - - def run(self): - """Configure the TestRun.""" - tr = Test.AddTestRun(self._description) + host.Streams.stdout += Testers.ContainsExpression( + f'WINDOW_UPDATE.*id {first_id}: {window_update_size}', + f"{hostname} should receive a stream WINDOW_UPDATE.") + + host.Streams.stdout += Testers.ContainsExpression( + f'WINDOW_UPDATE.*id {first_id + 2}: {window_update_size}', + f"{hostname} should receive a stream WINDOW_UPDATE.") + + host.Streams.stdout += Testers.ContainsExpression( + f'WINDOW_UPDATE.*id {first_id + 4}: {window_update_size}', + f"{hostname} should receive a stream WINDOW_UPDATE.") + + def _configure_test_run_common(self, tr, is_outbound: bool, is_http2_to_origin: bool): + """Perform the common Process configuration.""" + self._dns = self._configure_dns(tr) + self._server = self._configure_server(tr) + self._ts = self._configure_trafficserver(tr, is_outbound, is_http2_to_origin) if not self._flow_control_policy_is_malformed: self._configure_client(tr) tr.Processes.Default.StartBefore(self._dns) tr.Processes.Default.StartBefore(self._server) - tr.StillRunningAfter = self._ts else: tr.Processes.Default.Command = "true" tr.Processes.Default.StartBefore(self._ts) + tr.TimeOut = 20 + + def _configure_inbound_http1_to_origin_test_run(self): + """Configure the TestRun for inbound stream configuration.""" + tr = Test.AddTestRun(f'{self._description} - inbound') + self._configure_test_run_common(tr, self.IS_INBOUND, self.IS_HTTP1_TO_ORIGIN) + self._configure_log_expectations(tr.Processes.Default) + + def _configure_inbound_http2_to_origin_test_run(self): + """Configure the TestRun for inbound stream configuration.""" + tr = Test.AddTestRun(f'{self._description} - inbound') + self._configure_test_run_common(tr, self.IS_INBOUND, self.IS_HTTP2_TO_ORIGIN) + self._configure_log_expectations(tr.Processes.Default) + + def _configure_outbound_test_run(self): + """Configure the TestRun outbound stream configuration.""" + tr = Test.AddTestRun(f'{self._description} - outbound') + self._configure_test_run_common(tr, self.IS_OUTBOUND, self.IS_HTTP2_TO_ORIGIN) + self._configure_log_expectations(self._server) + + def run(self): + self._configure_inbound_http1_to_origin_test_run() + self._configure_inbound_http2_to_origin_test_run() + self._configure_outbound_test_run() # @@ -266,18 +328,18 @@ def run(self): test.run() # -# Configuring max_concurrent_streams_in. +# Configuring max_concurrent_streams_(in|out). # test = Http2FlowControlTest( - description="Configure max_concurrent_streams_in", - max_concurrent_streams_in=53) + description="Configure max_concurrent_streams", + max_concurrent_streams=53) test.run() # # Configuring initial_window_size. # test = Http2FlowControlTest( - description="Configure a larger initial_window_size_in", + description="Configure a larger initial_window_size_(in|out)", initial_window_size=100123) test.run() @@ -288,20 +350,21 @@ def run(self): description="Configure an unrecognized flow_control.in.policy", flow_control_policy=23) test.run() + test = Http2FlowControlTest( - description="Flow control policy 0 (default): small initial_window_size_in", + description="Flow control policy 0 (default): small initial_window_size_(in|out)", initial_window_size=500, # The default is 65 KB. flow_control_policy=0) test.run() test = Http2FlowControlTest( description="Flow control policy 1: 100 byte session, 10 byte stream windows", - max_concurrent_streams_in=10, + max_concurrent_streams=10, initial_window_size=10, flow_control_policy=1) test.run() test = Http2FlowControlTest( description="Flow control policy 2: 100 byte session, dynamic stream windows", - max_concurrent_streams_in=10, + max_concurrent_streams=10, initial_window_size=10, flow_control_policy=2) test.run() From ed43c95fbcd04ab0f214c4db876a067a29dbc21d Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Sat, 19 Nov 2022 21:19:07 +0000 Subject: [PATCH 03/16] Handle set_connect_fail in ConnectingEntry correctly When ConnectingEntry handled a connection failure, it could clear the error value passed to set_connect_fail. This would incorrectly result in the server not being marked down by hostdb. This patch ensures that the error code passed to set_connect_fail is always an error value when handling a connection failure. --- proxy/http/ConnectingEntry.cc | 2 +- proxy/http/HttpTransact.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/proxy/http/ConnectingEntry.cc b/proxy/http/ConnectingEntry.cc index 2bd0bd2ada0..255aa90f412 100644 --- a/proxy/http/ConnectingEntry.cc +++ b/proxy/http/ConnectingEntry.cc @@ -117,7 +117,7 @@ ConnectingEntry::state_http_server_open(int event, void *data) int lerrno = EIO; if (_netvc != nullptr) { vc_provided_cert = _netvc->provided_cert(); - lerrno = _netvc->lerrno; + lerrno = _netvc->lerrno == 0 ? lerrno : _netvc->lerrno; _netvc->do_io_close(); } while (!_connect_sms.empty()) { diff --git a/proxy/http/HttpTransact.h b/proxy/http/HttpTransact.h index d4b47062ea4..67d98a34fb8 100644 --- a/proxy/http/HttpTransact.h +++ b/proxy/http/HttpTransact.h @@ -926,6 +926,7 @@ class HttpTransact void set_connect_fail(int e) { + int const original_connect_result = this->current.server->connect_result; if (e == EUSERS) { // EUSERS is used when the number of connections exceeds the configured // limit. Since this is not a network connectivity issue with the @@ -938,7 +939,7 @@ class HttpTransact if (e != EIO) { this->cause_of_death_errno = e; } - Debug("http", "Setting upstream connection failure %d to %d", e, this->current.server->connect_result); + Debug("http", "Setting upstream connection failure %d to %d", original_connect_result, this->current.server->connect_result); } MgmtInt From 8b78d0a3f5f4c3dd1e26b907955551bd50ae30ae Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Tue, 22 Nov 2022 17:01:30 +0000 Subject: [PATCH 04/16] Send WINDOW_UPDATE for session when stream ends for other streams --- proxy/http2/Http2ConnectionState.cc | 21 +- .../h2/http2_flow_control.replay.yaml | 67 +++- .../gold_tests/h2/http2_flow_control.test.py | 114 ++++--- .../h2/http2_flow_control_chunked.replay.yaml | 304 ++++++++++++++++++ 4 files changed, 463 insertions(+), 43 deletions(-) create mode 100644 tests/gold_tests/h2/http2_flow_control_chunked.replay.yaml diff --git a/proxy/http2/Http2ConnectionState.cc b/proxy/http2/Http2ConnectionState.cc index c48b3952fec..29878d123ba 100644 --- a/proxy/http2/Http2ConnectionState.cc +++ b/proxy/http2/Http2ConnectionState.cc @@ -232,6 +232,12 @@ Http2ConnectionState::rcv_data_frame(const Http2Frame &frame) if (stream->read_enabled()) { if (frame.header().flags & HTTP2_FLAGS_DATA_END_STREAM) { + if (this->get_peer_stream_count() > 1 && this->get_local_rwnd() == 0) { + // This final DATA frame for this stream consumed all the bytes for the + // session window. Send a WINDOW_UPDATE frame in order to open up the + // session window for other streams. + restart_receiving(nullptr); + } stream->signal_read_event(VC_EVENT_READ_COMPLETE); } else { stream->signal_read_event(VC_EVENT_READ_READY); @@ -1717,6 +1723,9 @@ Http2ConnectionState::restart_streams() ink_assert(s != next); s = next; } + + // The above stopped at end, so we need to call send_response_body() one + // last time for the stream pointed to by end. if (std::min(this->get_peer_rwnd(), s->get_peer_rwnd()) > 0) { SCOPED_MUTEX_LOCK(lock, s->mutex, this_ethread()); s->restart_sending(); @@ -2026,11 +2035,15 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len if (window_size <= 0) { if (session->is_outbound()) { ip_port_text_buffer ipb; - const char *client_ip = ats_ip_ntop(session->get_proxy_session()->get_remote_addr(), ipb, sizeof(ipb)); - Warning("No window server_ip=%s session_wnd=%zd stream_wnd=%zd peer_initial_window=%u", client_ip, get_peer_rwnd(), + const char *server_ip = ats_ip_ntop(session->get_proxy_session()->get_remote_addr(), ipb, sizeof(ipb)); + // Warn the user to give them visibility that their server-side + // connection is being limited by their server's flow control. Maybe + // they can make adjustments. + Warning("No window server_ip=%s session_wnd=%zd stream_wnd=%zd peer_initial_window=%u", server_ip, get_peer_rwnd(), stream->get_peer_rwnd(), this->peer_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE)); } - Http2StreamDebug(this->session, stream->get_id(), "No window"); + Http2StreamDebug(this->session, stream->get_id(), "No window session_wnd=%zd stream_wnd=%zd peer_initial_window=%u", + get_peer_rwnd(), stream->get_peer_rwnd(), this->peer_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE)); this->session->flush(); return Http2SendDataFrameResult::NO_WINDOW; } @@ -2074,7 +2087,7 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len stream->decrement_peer_rwnd(payload_length); // Create frame - Http2StreamDebug(session, stream->get_id(), "Send a DATA frame - client window con: %5zd stream: %5zd payload: %5zd flags: 0x%x", + Http2StreamDebug(session, stream->get_id(), "Send a DATA frame - peer window con: %5zd stream: %5zd payload: %5zd flags: 0x%x", _peer_rwnd, stream->get_peer_rwnd(), payload_length, flags); Http2DataFrame data(stream->get_id(), flags, resp_reader, payload_length); diff --git a/tests/gold_tests/h2/http2_flow_control.replay.yaml b/tests/gold_tests/h2/http2_flow_control.replay.yaml index 4714ad26a4d..e99d6ed3a7c 100644 --- a/tests/gold_tests/h2/http2_flow_control.replay.yaml +++ b/tests/gold_tests/h2/http2_flow_control.replay.yaml @@ -64,7 +64,7 @@ sessions: - [ X-Response, { value: 'zero-response', as: equal } ] - client-request: - delay: 500ms + await: zero-request headers: fields: @@ -235,3 +235,68 @@ sessions: fields: - [ X-Response, {value: 'fifth-response', as: equal } ] + - client-request: + # Populate the cache with a large response. + + headers: + fields: + - [ :method, GET ] + - [ :scheme, https ] + - [ :authority, www.example.com ] + - [ :path, /sixth-request ] + - [ uuid, sixth-request ] + - [ X-Request, sixth-request ] + + proxy-request: + headers: + fields: + - [ X-Request, {value: 'sixth-request', as: equal } ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ X-Response, sixth-response ] + - [ Cache-Control, max-age=3600 ] + - [ Content-Length, 120000 ] + content: + size: 120000 + + proxy-response: + headers: + fields: + - [ X-Response, {value: 'sixth-response', as: equal } ] + + + # Retrieve an item from the cache. /sixth-request should have been cached in + # the previous transaction. + - client-request: + + # Give the above transaction enough time to finish. + await: sixth-request + + # Add some time to ensure that the sixth-request response is cached. + delay: 100ms + headers: + fields: + - [ :method, GET ] + - [ :scheme, https ] + - [ :authority, www.example.com ] + - [ :path, /sixth-request ] + - [ uuid, sixth-request-cached ] + - [ X-Request, sixth-request-cached ] + content: + size: 0 + + # Configure an error response which we don't expect to receive from the + # server because this should be served out of the cache. + server-response: + status: 500 + reason: Bad Request + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response, {value: 'sixth-response', as: equal } ] diff --git a/tests/gold_tests/h2/http2_flow_control.test.py b/tests/gold_tests/h2/http2_flow_control.test.py index ae4f8c2a665..00f78ed1fe7 100644 --- a/tests/gold_tests/h2/http2_flow_control.test.py +++ b/tests/gold_tests/h2/http2_flow_control.test.py @@ -16,8 +16,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import re +from enum import Enum from typing import List, Optional @@ -28,6 +28,7 @@ class Http2FlowControlTest: """Define an object to test HTTP/2 flow control behavior.""" _replay_file: str = 'http2_flow_control.replay.yaml' + _replay_chunked_file: str = 'http2_flow_control_chunked.replay.yaml' _valid_policy_values: List[int] = list(range(0, 3)) _flow_control_policy: Optional[int] = None _flow_control_policy_is_malformed: bool = False @@ -47,6 +48,13 @@ class Http2FlowControlTest: IS_HTTP2_TO_ORIGIN = True IS_HTTP1_TO_ORIGIN = False + class ServerType(Enum): + """Define the type of server to use in a TestRun.""" + + HTTP1_CONTENT_LENGTH = 0 + HTTP1_CHUNKED = 1 + HTTP2 = 2 + def __init__( self, description: str, @@ -100,20 +108,26 @@ def _configure_dns(self, tr: 'TestRun') -> 'Process': Http2FlowControlTest._dns_counter += 1 return dns - def _configure_server(self, tr: 'TestRun') -> 'Process': + def _configure_server(self, tr: 'TestRun', + server_type: ServerType) -> 'Process': """Configure the test server.""" + if server_type == self.ServerType.HTTP1_CHUNKED: + replay_file = self._replay_chunked_file + else: + replay_file = self._replay_file + server = tr.AddVerifierServerProcess( f'server-{Http2FlowControlTest._server_counter}', - self._replay_file) + replay_file) Http2FlowControlTest._server_counter += 1 return server - def _configure_trafficserver(self, tr: 'TestRun', is_outbound: bool, is_http2_to_orign: bool) -> 'Process': + def _configure_trafficserver(self, tr: 'TestRun', is_outbound: bool, + server_type: ServerType) -> 'Process': """Configure a Traffic Server process.""" ts = tr.MakeATSProcess( f'ts-{Http2FlowControlTest._ts_counter}', - enable_tls=True, - enable_cache=False) + enable_tls=True) Http2FlowControlTest._ts_counter += 1 ts.addDefaultSSLFiles() @@ -122,12 +136,14 @@ def _configure_trafficserver(self, tr: 'TestRun', is_outbound: bool, is_http2_to 'proxy.config.ssl.server.private_key.path': f'{ts.Variables.SSLDir}', 'proxy.config.ssl.client.verify.server.policy': 'PERMISSIVE', 'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(self._dns.Variables.Port), + 'proxy.config.dns.resolv_conf': 'NULL', + 'proxy.config.http.insert_age_in_response': 0, 'proxy.config.diags.debug.enabled': 3, 'proxy.config.diags.debug.tags': 'http', }) - if is_http2_to_orign: + if server_type == self.ServerType.HTTP2: ts.Disk.records_config.update({ 'proxy.config.ssl.client.alpn_protocols': 'h2,http/1.1', }) @@ -174,11 +190,11 @@ def _configure_trafficserver(self, tr: 'TestRun', is_outbound: bool, is_http2_to configuration = 'proxy.config.http2.flow_control.policy_in' ts.Disk.diags_log.Content = Testers.ContainsExpression( f"ERROR.*{configuration}", - "There should be an error about an invalid flow control policy.") + "Expected an error about an invalid flow control policy.") return ts - def _configure_client(self, tr): + def _configure_client(self, tr, ): """Configure a client process. :param tr: The TestRun to associate the client with. @@ -251,25 +267,35 @@ def _configure_log_expectations(self, host): stream_window_1 = session_window_size stream_window_2 = int(session_window_size / 2) stream_window_3 = int(session_window_size / 3) - host.Streams.stdout += Testers.ContainsExpression( - (f'INITIAL_WINDOW_SIZE:{stream_window_1}.*' - f'INITIAL_WINDOW_SIZE:{stream_window_2}.*' - f'INITIAL_WINDOW_SIZE:{stream_window_3}'), - f"{hostname} should stream receive window updates", - reflags=re.DOTALL | re.MULTILINE) + if self._server: + # Toward the server, there is a potential race condition + # between sending of first-request and the sending of the + # SETTINGS frame which reduces the stream window size. + # Allow for either scenario. + host.Streams.stdout += Testers.ContainsExpression( + (f'INITIAL_WINDOW_SIZE:{stream_window_1}.*' + f'INITIAL_WINDOW_SIZE:{stream_window_2}.*'), + f"{hostname} should stream receive window updates", + reflags=re.DOTALL | re.MULTILINE) + else: + host.Streams.stdout += Testers.ContainsExpression( + (f'INITIAL_WINDOW_SIZE:{stream_window_1}.*' + f'INITIAL_WINDOW_SIZE:{stream_window_2}.*' + f'INITIAL_WINDOW_SIZE:{stream_window_3}'), + f"{hostname} should stream receive window updates", + reflags=re.DOTALL | re.MULTILINE) if self._expected_initial_stream_window_size < 1000: first_id = 5 if self._server else 3 - # WINDOW_UPDATE timing is different between the server and the - # client. Toward the origin we send SETTINGS frames to update the - # INITIAL_WINDOW_SIZE with the headers so they are received earlier - # than with the client, wherein we send the updated - # INITIAL_WINDOW_SIZE after receiving headers from the client. if self._server and self._expected_flow_control_policy == 2: - window_update_size = 33 + # Toward the server, there is a potential race condition + # between sending of first-request and the sending of the + # SETTINGS frame which reduces the stream window size. Allow + # for either scenario. + window_update_size = f'33|{self._expected_initial_stream_window_size}' else: - window_update_size = self._expected_initial_stream_window_size + window_update_size = f'{self._expected_initial_stream_window_size}' # For the smaller session window sizes, we expect WINDOW_UPDATE frames. host.Streams.stdout += Testers.ContainsExpression( f'WINDOW_UPDATE.*id {first_id}: {window_update_size}', @@ -283,11 +309,12 @@ def _configure_log_expectations(self, host): f'WINDOW_UPDATE.*id {first_id + 4}: {window_update_size}', f"{hostname} should receive a stream WINDOW_UPDATE.") - def _configure_test_run_common(self, tr, is_outbound: bool, is_http2_to_origin: bool): + def _configure_test_run_common(self, tr, is_outbound: bool, + server_type: ServerType) -> None: """Perform the common Process configuration.""" self._dns = self._configure_dns(tr) - self._server = self._configure_server(tr) - self._ts = self._configure_trafficserver(tr, is_outbound, is_http2_to_origin) + self._server = self._configure_server(tr, server_type) + self._ts = self._configure_trafficserver(tr, is_outbound, server_type) if not self._flow_control_policy_is_malformed: self._configure_client(tr) tr.Processes.Default.StartBefore(self._dns) @@ -297,25 +324,36 @@ def _configure_test_run_common(self, tr, is_outbound: bool, is_http2_to_origin: tr.Processes.Default.StartBefore(self._ts) tr.TimeOut = 20 - def _configure_inbound_http1_to_origin_test_run(self): + def _configure_inbound_http1_to_origin_test_run(self) -> None: """Configure the TestRun for inbound stream configuration.""" - tr = Test.AddTestRun(f'{self._description} - inbound') - self._configure_test_run_common(tr, self.IS_INBOUND, self.IS_HTTP1_TO_ORIGIN) + tr = Test.AddTestRun(f'{self._description} - inbound, ' + 'HTTP/1 Content-Length origin') + self._configure_test_run_common(tr, self.IS_INBOUND, + self.ServerType.HTTP1_CONTENT_LENGTH) + self._configure_log_expectations(tr.Processes.Default) + + tr = Test.AddTestRun(f'{self._description} - inbound, ' + 'HTTP/1 chunked origin') + self._configure_test_run_common(tr, self.IS_INBOUND, + self.ServerType.HTTP1_CHUNKED) self._configure_log_expectations(tr.Processes.Default) - def _configure_inbound_http2_to_origin_test_run(self): + def _configure_inbound_http2_to_origin_test_run(self) -> None: """Configure the TestRun for inbound stream configuration.""" - tr = Test.AddTestRun(f'{self._description} - inbound') - self._configure_test_run_common(tr, self.IS_INBOUND, self.IS_HTTP2_TO_ORIGIN) + tr = Test.AddTestRun(f'{self._description} - inbound, HTTP/2 origin') + self._configure_test_run_common(tr, self.IS_INBOUND, + self.ServerType.HTTP2) self._configure_log_expectations(tr.Processes.Default) - def _configure_outbound_test_run(self): + def _configure_outbound_test_run(self) -> None: """Configure the TestRun outbound stream configuration.""" - tr = Test.AddTestRun(f'{self._description} - outbound') - self._configure_test_run_common(tr, self.IS_OUTBOUND, self.IS_HTTP2_TO_ORIGIN) + tr = Test.AddTestRun(f'{self._description} - outbound, HTTP/2 origin') + self._configure_test_run_common(tr, self.IS_OUTBOUND, + self.ServerType.HTTP2) self._configure_log_expectations(self._server) - def run(self): + def run(self) -> None: + """Configure the test run for various origin side configurations.""" self._configure_inbound_http1_to_origin_test_run() self._configure_inbound_http2_to_origin_test_run() self._configure_outbound_test_run() @@ -352,18 +390,18 @@ def run(self): test.run() test = Http2FlowControlTest( - description="Flow control policy 0 (default): small initial_window_size_(in|out)", + description="Flow control policy 0 (default): small initial_window_size", initial_window_size=500, # The default is 65 KB. flow_control_policy=0) test.run() test = Http2FlowControlTest( - description="Flow control policy 1: 100 byte session, 10 byte stream windows", + description="Flow control policy 1: 100 byte session, 10 byte streams", max_concurrent_streams=10, initial_window_size=10, flow_control_policy=1) test.run() test = Http2FlowControlTest( - description="Flow control policy 2: 100 byte session, dynamic stream windows", + description="Flow control policy 2: 100 byte session, dynamic streams", max_concurrent_streams=10, initial_window_size=10, flow_control_policy=2) diff --git a/tests/gold_tests/h2/http2_flow_control_chunked.replay.yaml b/tests/gold_tests/h2/http2_flow_control_chunked.replay.yaml new file mode 100644 index 00000000000..b30ff37c696 --- /dev/null +++ b/tests/gold_tests/h2/http2_flow_control_chunked.replay.yaml @@ -0,0 +1,304 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +meta: + version: "1.0" + +# This replay file generates an HTTP/2 session with three streams in order to +# verify that ATS generates the expected SETTINGS and WINDOW_UPDATE frames. + +sessions: + +- protocol: + - name: http + version: 2 + - name: tls + sni: www.example.com + - name: tcp + - name: ip + + transactions: + + - client-request: + headers: + fields: + - [ :method, GET ] + - [ :scheme, https ] + - [ :authority, www.example.com ] + - [ :path, /zero-request ] + - [ uuid, zero-request ] + - [ X-Request, zero-request ] + - [ Content-Length, 0 ] + + proxy-request: + headers: + fields: + - [ X-Request, { value: 'zero-request', as: equal } ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ X-Response, zero-response ] + - [ Transfer-Encoding, chunked ] + content: + size: 28 + + proxy-response: + headers: + fields: + - [ X-Response, { value: 'zero-response', as: equal } ] + + - client-request: + await: zero-request + + headers: + fields: + - [ :method, POST ] + - [ :scheme, https ] + - [ :authority, www.example.com ] + - [ :path, /first-request ] + - [ uuid, first-request ] + - [ X-Request, first-request ] + content: + size: 1200 + + proxy-request: + headers: + fields: + - [ X-Request, { value: 'first-request', as: equal } ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ X-Response, first-response ] + - [ Transfer-Encoding, chunked ] + content: + size: 1200 + + proxy-response: + headers: + fields: + - [ X-Response, { value: 'first-response', as: equal } ] + + - client-request: + headers: + fields: + - [ :method, POST ] + - [ :scheme, https ] + - [ :authority, www.example.com ] + - [ :path, /second-request ] + - [ uuid, second-request ] + - [ X-Request, second-request ] + content: + size: 1200 + + proxy-request: + headers: + fields: + - [ X-Request, {value: 'second-request', as: equal } ] + + # Intermix a Content-Length encoding just to make sure they interact well + # with each other. + server-response: + status: 200 + reason: OK + headers: + fields: + - [ X-Response, second-response ] + - [ Content-Length, 1200 ] + content: + size: 1200 + + proxy-response: + headers: + fields: + - [ X-Response, {value: 'second-response', as: equal } ] + + - client-request: + headers: + fields: + - [ :method, POST ] + - [ :scheme, https ] + - [ :authority, www.example.com ] + - [ :path, /third-request ] + - [ uuid, third-request ] + - [ X-Request, third-request ] + content: + size: 1200 + + proxy-request: + headers: + fields: + - [ X-Request, {value: 'third-request', as: equal } ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ X-Response, third-response ] + - [ Transfer-Encoding, chunked ] + content: + size: 1200 + + proxy-response: + headers: + fields: + - [ X-Response, {value: 'third-response', as: equal } ] + + - client-request: + # Intentionally test a stream after the three other parallel POST + # requests. + delay: 500ms + + headers: + fields: + - [ :method, POST ] + - [ :scheme, https ] + - [ :authority, www.example.com ] + - [ :path, /fourth-request ] + - [ uuid, fourth-request ] + - [ X-Request, fourth-request ] + content: + # Send a very large DATA frame so that we exceed the 65,535 window + # size of most of the test runs. + size: 120000 + + proxy-request: + headers: + fields: + - [ X-Request, {value: 'fourth-request', as: equal } ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ X-Response, fourth-response ] + - [ Transfer-Encoding, chunked ] + content: + size: 120000 + + proxy-response: + headers: + fields: + - [ X-Response, {value: 'fourth-response', as: equal } ] + + - client-request: + # Give the above request time to process and give us an opportunity to + # receive any other WINDOW_UPDATE frames. + delay: 500ms + + headers: + fields: + - [ :method, POST ] + - [ :scheme, https ] + - [ :authority, www.example.com ] + - [ :path, /fifth-request ] + - [ uuid, fifth-request ] + - [ X-Request, fifth-request ] + content: + size: 10000 + + proxy-request: + headers: + fields: + - [ X-Request, {value: 'fifth-request', as: equal } ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ X-Response, fifth-response ] + - [ Transfer-Encoding, chunked ] + content: + size: 10000 + + proxy-response: + headers: + fields: + - [ X-Response, {value: 'fifth-response', as: equal } ] + + - client-request: + # Populate the cache with a large response. + + headers: + fields: + - [ :method, GET ] + - [ :scheme, https ] + - [ :authority, www.example.com ] + - [ :path, /sixth-request ] + - [ uuid, sixth-request ] + - [ X-Request, sixth-request ] + + proxy-request: + headers: + fields: + - [ X-Request, {value: 'sixth-request', as: equal } ] + + server-response: + status: 200 + reason: OK + headers: + fields: + - [ X-Response, sixth-response ] + - [ Cache-Control, max-age=3600 ] + - [ Transfer-Encoding, chunked ] + content: + size: 120000 + + proxy-response: + headers: + fields: + - [ X-Response, {value: 'sixth-response', as: equal } ] + + + # Retrieve an item from the cache. /sixth-request should have been cached in + # the previous transaction. + - client-request: + + # Give the above transaction enough time to finish. + await: sixth-request + + # Add some time to ensure that the sixth-request response is cached. + delay: 100ms + headers: + fields: + - [ :method, GET ] + - [ :scheme, https ] + - [ :authority, www.example.com ] + - [ :path, /sixth-request ] + - [ uuid, sixth-request-cached ] + - [ X-Request, sixth-request-cached ] + content: + size: 0 + + # Configure an error response which we don't expect to receive from the + # server because this should be served out of the cache. + server-response: + status: 500 + reason: Bad Request + + proxy-response: + status: 200 + headers: + fields: + - [ X-Response, {value: 'sixth-response', as: equal } ] From 9b92b1dafbf0fc0a642639732a989ad75110f50a Mon Sep 17 00:00:00 2001 From: Fei Deng Date: Wed, 1 Feb 2023 17:25:43 -0600 Subject: [PATCH 05/16] pass through RST_STREAM frame --- proxy/http2/Http2ConnectionState.cc | 4 +- proxy/http2/Http2Stream.cc | 10 +- .../h2/gold/server_after_headers.gold | 4 + tests/gold_tests/h2/h2origin.test.py | 2 +- .../h2/h2origin_single_thread.test.py | 2 +- tests/gold_tests/h2/http2_rst_stream.test.py | 199 ++++++++++++++++++ .../h1-client-h2-origin.yaml | 0 .../h2-origin.yaml | 0 .../http2_rst_stream_client_after_data.yaml | 54 +++++ ...http2_rst_stream_client_after_headers.yaml | 54 +++++ ...http2_rst_stream_server_after_headers.yaml | 48 +++++ 11 files changed, 371 insertions(+), 6 deletions(-) create mode 100644 tests/gold_tests/h2/gold/server_after_headers.gold create mode 100644 tests/gold_tests/h2/http2_rst_stream.test.py rename tests/gold_tests/h2/{replay => replay_h2origin}/h1-client-h2-origin.yaml (100%) rename tests/gold_tests/h2/{replay => replay_h2origin}/h2-origin.yaml (100%) create mode 100644 tests/gold_tests/h2/replay_rst_stream/http2_rst_stream_client_after_data.yaml create mode 100644 tests/gold_tests/h2/replay_rst_stream/http2_rst_stream_client_after_headers.yaml create mode 100644 tests/gold_tests/h2/replay_rst_stream/http2_rst_stream_server_after_headers.yaml diff --git a/proxy/http2/Http2ConnectionState.cc b/proxy/http2/Http2ConnectionState.cc index 29878d123ba..7e3cd6bd488 100644 --- a/proxy/http2/Http2ConnectionState.cc +++ b/proxy/http2/Http2ConnectionState.cc @@ -621,9 +621,10 @@ Http2ConnectionState::rcv_rst_stream_frame(const Http2Frame &frame) } if (stream != nullptr) { - Http2StreamDebug(this->session, stream_id, "RST_STREAM: Error Code: %u", rst_stream.error_code); + Http2StreamDebug(this->session, stream_id, "Parsed RST_STREAM: Error Code: %u", rst_stream.error_code); stream->set_rx_error_code({ProxyErrorClass::TXN, static_cast(rst_stream.error_code)}); + stream->signal_read_event(VC_EVENT_EOS); stream->initiating_close(); } @@ -2392,6 +2393,7 @@ Http2ConnectionState::send_rst_stream_frame(Http2StreamId id, Http2ErrorCode ec) } } + Http2StreamDebug(session, id, "Sending RST_STREAM: Error Code: %u", static_cast(ec)); Http2RstStreamFrame rst_stream(id, static_cast(ec)); this->session->xmit(rst_stream); } diff --git a/proxy/http2/Http2Stream.cc b/proxy/http2/Http2Stream.cc index ae3275ce96d..23fbe3bd635 100644 --- a/proxy/http2/Http2Stream.cc +++ b/proxy/http2/Http2Stream.cc @@ -486,9 +486,9 @@ Http2Stream::do_io_close(int /* flags */) REMEMBER(NO_EVENT, this->reentrancy_count); Http2StreamDebug("do_io_close"); - // if (this->is_state_writeable()) { // Let the other end know we are going away - // this->get_connection_state().send_rst_stream_frame(_id, Http2ErrorCode::HTTP2_ERROR_NO_ERROR); - //} + if (this->is_state_writeable()) { // Let the other end know we are going away + this->get_connection_state().send_rst_stream_frame(_id, Http2ErrorCode::HTTP2_ERROR_NO_ERROR); + } // When we get here, the SM has initiated the shutdown. Either it received a WRITE_COMPLETE, or it is shutting down. Any // remaining IO operations back to client should be abandoned. The SM-side buffers backing these operations will be deleted @@ -552,6 +552,10 @@ Http2Stream::initiating_close() Http2StreamDebug("initiating_close client_window=%" PRId64 " session_window=%" PRId64, _peer_rwnd, this->get_connection_state().get_peer_rwnd()); + if (this->is_state_writeable()) { // Let the other end know we are going away + this->get_connection_state().send_rst_stream_frame(_id, Http2ErrorCode::HTTP2_ERROR_NO_ERROR); + } + // Set the state of the connection to closed // TODO - these states should be combined closed = true; diff --git a/tests/gold_tests/h2/gold/server_after_headers.gold b/tests/gold_tests/h2/gold/server_after_headers.gold new file mode 100644 index 00000000000..555c2dbca0d --- /dev/null +++ b/tests/gold_tests/h2/gold/server_after_headers.gold @@ -0,0 +1,4 @@ +`` +``Submitting RST_STREAM frame for key 1 after HEADERS frame with error code ENHANCE_YOUR_CALM. +``Sent RST_STREAM frame for key 1 on stream 3. +`` diff --git a/tests/gold_tests/h2/h2origin.test.py b/tests/gold_tests/h2/h2origin.test.py index e4d3de67d85..23ebbac887f 100644 --- a/tests/gold_tests/h2/h2origin.test.py +++ b/tests/gold_tests/h2/h2origin.test.py @@ -30,7 +30,7 @@ # add ssl materials like key, certificates for the server ts.addDefaultSSLFiles() -replay_file = "replay/" +replay_file = "replay_h2origin/" server = Test.MakeVerifierServerProcess("h2-origin", replay_file) ts.Disk.records_config.update({ 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), diff --git a/tests/gold_tests/h2/h2origin_single_thread.test.py b/tests/gold_tests/h2/h2origin_single_thread.test.py index a1f80cb74ce..7e10ba06cbe 100644 --- a/tests/gold_tests/h2/h2origin_single_thread.test.py +++ b/tests/gold_tests/h2/h2origin_single_thread.test.py @@ -30,7 +30,7 @@ # add ssl materials like key, certificates for the server ts.addDefaultSSLFiles() -replay_file = "replay" +replay_file = "replay_h2origin" server = Test.MakeVerifierServerProcess("h2-origin", replay_file) ts.Disk.records_config.update({ 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), diff --git a/tests/gold_tests/h2/http2_rst_stream.test.py b/tests/gold_tests/h2/http2_rst_stream.test.py new file mode 100644 index 00000000000..c4e634fc1f0 --- /dev/null +++ b/tests/gold_tests/h2/http2_rst_stream.test.py @@ -0,0 +1,199 @@ +''' +Abort HTTP/2 connection using RST_STREAM frame. +''' + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Abort HTTP/2 connection using RST_STREAM frame. +''' + +Test.SkipUnless( + Condition.HasOpenSSLVersion('1.1.1'), + Condition.HasProxyVerifierVersion('2.5.2') +) + +# +# Client sends RST_STREAM after DATA frame +# +ts = Test.MakeATSProcess("ts0", enable_tls=True) +replay_file = "replay_rst_stream/http2_rst_stream_client_after_data.yaml" +server = Test.MakeVerifierServerProcess("server0", replay_file) +ts.addDefaultSSLFiles() +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': f'{ts.Variables.SSLDir}', + 'proxy.config.ssl.server.private_key.path': f'{ts.Variables.SSLDir}', + 'proxy.config.ssl.client.verify.server.policy': 'PERMISSIVE', + 'proxy.config.diags.debug.enabled': 3, + 'proxy.config.diags.debug.tags': 'http', + 'proxy.config.exec_thread.autoconfig': 0, + 'proxy.config.exec_thread.limit': 4, + 'proxy.config.ssl.client.alpn_protocols': 'h2,http/1.1', + 'proxy.config.http.server_session_sharing.pool': 'thread', + 'proxy.config.http.server_session_sharing.match': 'ip,sni,cert', +}) +ts.Disk.remap_config.AddLine( + f'map / https://127.0.0.1:{server.Variables.https_port}' +) +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +tr = Test.AddTestRun('Client sends RST_STREAM after DATA frame') +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(ts) +tr.AddVerifierClientProcess("client0", replay_file, http_ports=[ts.Variables.port], https_ports=[ts.Variables.ssl_port]) + +tr.Processes.Default.Streams.All += Testers.ContainsExpression( + 'Submitting RST_STREAM frame for key 1 after DATA frame with error code INTERNAL_ERROR.', + 'Detect client abort flag.') + +tr.Processes.Default.Streams.All += Testers.ContainsExpression( + 'Sent RST_STREAM frame for key 1 on stream 1', + 'Send RST_STREAM frame.') + +server.Streams.All += Testers.ExcludesExpression( + 'RST_STREAM', + 'Server is not affected.') + +ts.Disk.traffic_out.Content += Testers.ContainsExpression( + 'Received HEADERS frame', + 'Received HEADERS frame.') + +ts.Disk.traffic_out.Content += Testers.ContainsExpression( + 'Received DATA frame', + 'Received DATA frame.') + +ts.Disk.traffic_out.Content += Testers.ContainsExpression( + 'Received RST_STREAM frame', + 'Received RST_STREAM frame.') + +ts.Disk.traffic_out.Content += Testers.ContainsExpression( + 'RST_STREAM: Error Code: 2', + 'Error Code: ') + +# +# Client sends RST_STREAM after HEADERS frame +# +ts = Test.MakeATSProcess("ts1", enable_tls=True) +replay_file = "replay_rst_stream/http2_rst_stream_client_after_headers.yaml" +server = Test.MakeVerifierServerProcess("server1", replay_file) +ts.addDefaultSSLFiles() +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': f'{ts.Variables.SSLDir}', + 'proxy.config.ssl.server.private_key.path': f'{ts.Variables.SSLDir}', + 'proxy.config.ssl.client.verify.server.policy': 'PERMISSIVE', + 'proxy.config.diags.debug.enabled': 3, + 'proxy.config.diags.debug.tags': 'http', + 'proxy.config.exec_thread.autoconfig': 0, + 'proxy.config.exec_thread.limit': 4, + 'proxy.config.ssl.client.alpn_protocols': 'h2,http/1.1', + 'proxy.config.http.server_session_sharing.pool': 'thread', + 'proxy.config.http.server_session_sharing.match': 'ip,sni,cert', +}) +ts.Disk.remap_config.AddLine( + f'map / https://127.0.0.1:{server.Variables.https_port}' +) +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +tr = Test.AddTestRun('Client sends RST_STREAM after HEADERS frame') +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(ts) +tr.AddVerifierClientProcess("client1", replay_file, http_ports=[ts.Variables.port], https_ports=[ts.Variables.ssl_port]) + +tr.Processes.Default.Streams.All += Testers.ContainsExpression( + 'Submitting RST_STREAM frame for key 1 after HEADERS frame with error code STREAM_CLOSED.', + 'Detect client abort flag.') + +tr.Processes.Default.Streams.All += Testers.ContainsExpression( + 'Sent RST_STREAM frame for key 1 on stream 1', + 'Send RST_STREAM frame.') + +server.Streams.All += Testers.ExcludesExpression( + 'RST_STREAM', + 'Server is not affected.') + +ts.Disk.traffic_out.Content += Testers.ContainsExpression( + 'Received HEADERS frame', + 'Received HEADERS frame.') + +ts.Disk.traffic_out.Content += Testers.ExcludesExpression( + 'Received DATA frame', + 'Received DATA frame.') + +ts.Disk.traffic_out.Content += Testers.ContainsExpression( + 'Received RST_STREAM frame', + 'Received RST_STREAM frame.') + +ts.Disk.traffic_out.Content += Testers.ContainsExpression( + 'RST_STREAM: Error Code: 5', + 'Error Code: ') + +# +# Server sends RST_STREAM after HEADERS frame +# +ts = Test.MakeATSProcess("ts2", enable_tls=True) +replay_file = "replay_rst_stream/http2_rst_stream_server_after_headers.yaml" +server = Test.MakeVerifierServerProcess("server2", replay_file) +ts.addDefaultSSLFiles() +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': f'{ts.Variables.SSLDir}', + 'proxy.config.ssl.server.private_key.path': f'{ts.Variables.SSLDir}', + 'proxy.config.ssl.client.verify.server.policy': 'PERMISSIVE', + 'proxy.config.diags.debug.enabled': 3, + 'proxy.config.diags.debug.tags': 'http', + 'proxy.config.exec_thread.autoconfig': 0, + 'proxy.config.exec_thread.limit': 1, + 'proxy.config.ssl.client.alpn_protocols': 'h2,http/1.1', + 'proxy.config.http.server_session_sharing.pool': 'thread', + 'proxy.config.http.server_session_sharing.match': 'ip,sni,cert', +}) +ts.Disk.remap_config.AddLine( + f'map / https://127.0.0.1:{server.Variables.https_port}' +) +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +tr = Test.AddTestRun('Server sends RST_STREAM after HEADERS frame') +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(ts) +tr.AddVerifierClientProcess("client2", replay_file, http_ports=[ts.Variables.port], https_ports=[ts.Variables.ssl_port]) + +tr.Processes.Default.Streams.All += Testers.ContainsExpression( + 'HTTP/2 stream is closed with id: 1', + 'Client is not affected.') + +server.Streams.All += "gold/server_after_headers.gold" + +ts.Disk.traffic_out.Content += Testers.ContainsExpression( + 'Received RST_STREAM frame', + 'Received RST_STREAM frame.') + +ts.Disk.traffic_out.Content += Testers.ContainsExpression( + 'Send RST_STREAM frame', + 'Send RST_STREAM frame.') + +ts.Disk.traffic_out.Content += Testers.ContainsExpression( + 'Parsed RST_STREAM: Error Code: 11', + 'Error Code: ') + +ts.Disk.traffic_out.Content += Testers.ContainsExpression( + 'Sending RST_STREAM: Error Code: 0', + 'Error Code: ') diff --git a/tests/gold_tests/h2/replay/h1-client-h2-origin.yaml b/tests/gold_tests/h2/replay_h2origin/h1-client-h2-origin.yaml similarity index 100% rename from tests/gold_tests/h2/replay/h1-client-h2-origin.yaml rename to tests/gold_tests/h2/replay_h2origin/h1-client-h2-origin.yaml diff --git a/tests/gold_tests/h2/replay/h2-origin.yaml b/tests/gold_tests/h2/replay_h2origin/h2-origin.yaml similarity index 100% rename from tests/gold_tests/h2/replay/h2-origin.yaml rename to tests/gold_tests/h2/replay_h2origin/h2-origin.yaml diff --git a/tests/gold_tests/h2/replay_rst_stream/http2_rst_stream_client_after_data.yaml b/tests/gold_tests/h2/replay_rst_stream/http2_rst_stream_client_after_data.yaml new file mode 100644 index 00000000000..d505778f5ea --- /dev/null +++ b/tests/gold_tests/h2/replay_rst_stream/http2_rst_stream_client_after_data.yaml @@ -0,0 +1,54 @@ +meta: + version: '1.0' +sessions: +- protocol: + - name: http + version: 2 + - name: tls + sni: test_sni + - name: tcp + - name: ip + version: 4 + transactions: + - client-request: + frames: + - HEADERS: + headers: + fields: + - [:method, POST] + - [:scheme, https] + - [:authority, example.data.com] + - [:path, /a/path] + - [Content-Type, text/html] + - [Content-Length, '11'] + - [uuid, 1] + - DATA: + content: + encoding: plain + data: client_test + size: 11 + - RST_STREAM: + error-code: INTERNAL_ERROR + + proxy-request: + content: + encoding: plain + data: client_test + verify: {as: equal} + + server-response: + headers: + fields: + - [:status, 200] + - [Content-Type, text/html] + - [Content-Length, '11'] + content: + encoding: plain + data: server_test + size: 11 + + proxy-response: + content: + encoding: plain + data: server_test + verify: {as: equal} diff --git a/tests/gold_tests/h2/replay_rst_stream/http2_rst_stream_client_after_headers.yaml b/tests/gold_tests/h2/replay_rst_stream/http2_rst_stream_client_after_headers.yaml new file mode 100644 index 00000000000..58415705a00 --- /dev/null +++ b/tests/gold_tests/h2/replay_rst_stream/http2_rst_stream_client_after_headers.yaml @@ -0,0 +1,54 @@ +meta: + version: '1.0' +sessions: +- protocol: + - name: http + version: 2 + - name: tls + sni: test_sni + - name: tcp + - name: ip + version: 4 + transactions: + - client-request: + frames: + - HEADERS: + headers: + fields: + - [:method, POST] + - [:scheme, https] + - [:authority, example.data.com] + - [:path, /a/path] + - [Content-Type, text/html] + - [Content-Length, '11'] + - [uuid, 1] + - RST_STREAM: + error-code: STREAM_CLOSED + - DATA: + content: + encoding: plain + data: client_test + size: 11 + + proxy-request: + content: + encoding: plain + data: client_test + verify: {as: equal} + + server-response: + headers: + fields: + - [:status, 200] + - [Content-Type, text/html] + - [Content-Length, '11'] + content: + encoding: plain + data: server_test + size: 11 + + proxy-response: + content: + encoding: plain + data: server_test + verify: {as: equal} diff --git a/tests/gold_tests/h2/replay_rst_stream/http2_rst_stream_server_after_headers.yaml b/tests/gold_tests/h2/replay_rst_stream/http2_rst_stream_server_after_headers.yaml new file mode 100644 index 00000000000..1ce5c320382 --- /dev/null +++ b/tests/gold_tests/h2/replay_rst_stream/http2_rst_stream_server_after_headers.yaml @@ -0,0 +1,48 @@ +meta: + version: '1.0' +sessions: +- protocol: + - name: http + version: 2 + - name: tls + sni: test_sni + - name: tcp + - name: ip + version: 4 + transactions: + - client-request: + headers: + fields: + - [:method, POST] + - [:scheme, https] + - [:authority, example.data.com] + - [:path, /a/path] + - [Content-Type, text/html] + - [Content-Length, '11'] + - [uuid, 1] + content: + encoding: plain + data: client_test + size: 11 + + proxy-request: + content: + encoding: plain + data: client_test + verify: {as: equal} + + server-response: + frames: + - HEADERS: + headers: + fields: + - [:status, 200] + - [Content-Type, text/html] + - [Content-Length, '11'] + - RST_STREAM: + error-code: ENHANCE_YOUR_CALM + - DATA: + content: + encoding: plain + data: server_test + size: 11 From 6c4d341f6f92fb5a1da9ba6bbccb845b61d3219c Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Wed, 1 Feb 2023 22:14:57 +0000 Subject: [PATCH 06/16] set_cause_of_death_errno --- proxy/http/HttpSM.cc | 6 +++ .../gold_tests/slow_post/server_abort.test.py | 51 +++++++++++++++++++ .../slow_post/test_secrets/aaa-signed.key | 27 ++++++++++ .../slow_post/test_secrets/aaa-signed.pem | 16 ++++++ 4 files changed, 100 insertions(+) create mode 100644 tests/gold_tests/slow_post/server_abort.test.py create mode 100644 tests/gold_tests/slow_post/test_secrets/aaa-signed.key create mode 100644 tests/gold_tests/slow_post/test_secrets/aaa-signed.pem diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index ba92d11d085..b481c7b02d0 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -1965,6 +1965,12 @@ HttpSM::state_http_server_open(int event, void *data) _netvc->do_io_close(); _netvc = nullptr; } + if (t_state.cause_of_death_errno == -UNKNOWN_INTERNAL_ERROR) { + // We set this to 0 because otherwise + // HttpTransact::retry_server_connection_not_open will raise an assertion + // if the value is the default UNKNOWN_INTERNAL_ERROR. + t_state.cause_of_death_errno = 0; + } /* If we get this error in transparent mode, then we simply can't bind to the 4-tuple to make the connection. There's no hope of retries succeeding in the near future. The best option is to just shut down the connection without further comment. The diff --git a/tests/gold_tests/slow_post/server_abort.test.py b/tests/gold_tests/slow_post/server_abort.test.py new file mode 100644 index 00000000000..1b80ac031a3 --- /dev/null +++ b/tests/gold_tests/slow_post/server_abort.test.py @@ -0,0 +1,51 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import re + +Test.Summary = ''' +AuTest with bad configuration of microserver to simulate server aborting the connection unexpectedly +''' +ts = Test.MakeATSProcess("ts", enable_tls=True) +# note the microserver by default is not configured to use ssl +server = Test.MakeOriginServer("server") +ts.Disk.remap_config.AddLine( + # The following config tells ATS to do tls with the origin server on a + # non-tls port. This is misconfigured intentionally to trigger an exception + # on the origin server so that it aborts the connection upon receiving a + # request + 'map / https://127.0.0.1:{0}'.format(server.Variables.Port)) +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=aaa-signed.pem ssl_key_name=aaa-signed.key' +) +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.tags': 'http|dns', + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.ssl.server.cert.path': f'{Test.TestDirectory}/test_secrets', + 'proxy.config.ssl.server.private_key.path': f'{Test.TestDirectory}/test_secrets', +}) + +tr = Test.AddTestRun() +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(ts) +tr.Processes.Default.Command = "curl -v -k -H \"host: foo.com\" https://127.0.0.1:{0}".format(ts.Variables.ssl_port) +tr.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts +server.Streams.stderr += Testers.ContainsExpression( + "UnicodeDecodeError", + "Verify that the server raises an exception when processing the request.") diff --git a/tests/gold_tests/slow_post/test_secrets/aaa-signed.key b/tests/gold_tests/slow_post/test_secrets/aaa-signed.key new file mode 100644 index 00000000000..64090deaafb --- /dev/null +++ b/tests/gold_tests/slow_post/test_secrets/aaa-signed.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA4Uim3ZOB1IfLWxpSjQ60dq2j7oVi5fW8idDg3zZOxBv2NlTm +Ca4uFtwW+Jhv4CSed/7ggoPvtuxHvTy4w2rwxFpM29sInRjQdJJ/gftIIkaEqZ5c +qleGBsaG5CLDFSPejJ2+rSY0FWg2/F9GxljV6BNgO0ukv3AjeIGpRdZF3mJIozb3 +fU3/XOrgDfCt6IH9ZBPHhRA1DzkuBtBkStDgXVYr3bzfhmVb9tMKZRJjLUPjKfa2 +4ninFKXl/2S6/RHSRcLWde3U4IizmfepXiIFi2j49UGiDzCq3sKCvMsAahwwx8fR +FEMMwV6oWJM0rgzoz8YEPBC2oRO1FCIkHOYHEQIDAQABAoIBAAqIUwTY+KDvGFrS +CDoADf/ebmOgaNdHfeETmu/UoioZBJHVtkuNkSoQcCJ/PfvEuoPxrp1rfbGXqmL2 +i8zXGxqS/jTpMKXnmxdYIg35qY2wrlMfzEVKgkGe1n+kAGrkmmsIlPmTZ6v4i1mR +OsXbMWUAQueCydkJbR8dMMTLF8klyLge9G8M3Bm2lqtQnridsRULMbNs22JOQnah +C/4oOrD7tZLLhMnSzvWP92POZ6a/2vO6ou6rvg6zu9mMw6yNRYSqwyLzztbyhG6Y +M9ER7v8YJBPfSy+nAQ8VCA9cZb96Ybxwc7BecJAWgg1+tSlak0BNj1qUg4oroSpK +bRAciIECgYEA99as1TNaPsveRv9nTpIVl4gXpfixQMx0gpnivYNZW2XpCx5COuML +t98Lu7Hg6bknukGwWzNGRGFX+3Bx+zsK6+VuFlBxANH9kUzkY9KFonvb3xWbDivG +2bf+9a3oORNsl2GVkCwhoD/1y0aPR2xZvpEZDAf0apQ1cQ+rikNc31UCgYEA6LPV +86rDXRfXwwHlcUgEpyJ6kJ6J1OpmsbN8eLm+99TNf8lp974ewXjJkTm6NYOGS11y +lboAh0IxoUKy0Fuuf0eTmOa+2dee52Np9pElZ6daCX8eTSAOL46GzDCaKfBopYdv +EWGaU+W/k+Eod5ygEJrZHPRJ5+YrhbHy+o6KcM0CgYEA0zG9sCSFh7Oko62rM/oq +qilPtaBaM9TGiDBoVoRilg8e6tmLKLEn4DUSw4xOE/0zDHZDuUPVYhntppdomeTz +ZpfpGtzLnx5SzQnQKfxQ4mhXsh+wNQA7AHbZrjPXCyQxSkLe96+TrAI1C1cCa6O6 +SjlNNcJllpjbfZAT5suGjc0CgYBVr4KkyshNSy5DvDsET4SHFocTIY2XPQi7fl/j +BGJxV4aj+0Jt2y/wBc4TD7KladzVe39p6qevJoyn2KuHVXsXmv+aWb0E8gStJ0op +ZKDlXhYlUQ2TUK5ojI7OOUdLEh82dHxNZicxpXO5vDrucFnwQ1SW+M0N+w8jl7bk +0//eMQKBgQD14laufpbwz6KEj/nihyGHfMCNIV4IgBtSkQIJ4+xM8fjTd4YoLLWn +wwkFDOvMC5TE8WMVf17Eyo9N6D9OfOwHqoRkC93bcRA/5GCvhO4cnOmzLITAr4m5 +wESxUCaACyXrBZVzRKupZEzsRrVskUC1WbQA2SHAJ7Kx5iAW3d7KuA== +-----END RSA PRIVATE KEY----- diff --git a/tests/gold_tests/slow_post/test_secrets/aaa-signed.pem b/tests/gold_tests/slow_post/test_secrets/aaa-signed.pem new file mode 100644 index 00000000000..2202efae57d --- /dev/null +++ b/tests/gold_tests/slow_post/test_secrets/aaa-signed.pem @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIICnDCCAYQCAQEwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAwwGYWFhLWNhMCAX +DTIwMDgyNTAxNDAzNloYDzIxMjAwODAxMDE0MDM2WjAVMRMwEQYDVQQDDAphYWEt +c2lnbmVkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4Uim3ZOB1IfL +WxpSjQ60dq2j7oVi5fW8idDg3zZOxBv2NlTmCa4uFtwW+Jhv4CSed/7ggoPvtuxH +vTy4w2rwxFpM29sInRjQdJJ/gftIIkaEqZ5cqleGBsaG5CLDFSPejJ2+rSY0FWg2 +/F9GxljV6BNgO0ukv3AjeIGpRdZF3mJIozb3fU3/XOrgDfCt6IH9ZBPHhRA1Dzku +BtBkStDgXVYr3bzfhmVb9tMKZRJjLUPjKfa24ninFKXl/2S6/RHSRcLWde3U4Iiz +mfepXiIFi2j49UGiDzCq3sKCvMsAahwwx8fRFEMMwV6oWJM0rgzoz8YEPBC2oRO1 +FCIkHOYHEQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQAYTvaL8Ii4UW5Z6mQEFxlf +4yEJlsXlVsjNDcMCU57gN1Hgkswsg5ePvNUkqcj7DyqoSwq9uhKP5ApzVdzhJlVH +WXsMIp95Tl0pC7L2JisMivOMn2y20wVGge1jXGexq4GOl08Ow/7rCM3xVok/TrIX +L2f6u+wjJG0YanlDO6l4+aU9+kqpVI7awnOWEshLOHUO4cQdnlF7jkYaEdCp2Ke7 +peycIow1oSTOcIKsqzcHRT/gYOqk3IkCTw0fIpCb5FhcCz7ZYppl5eeJSw0JHNyA +AOqNfFWJypTWhX8cGI6BKmWQMdR0z6j2pegG/2JOSl4ozFsU2JchzTizI2WxRSzm +-----END CERTIFICATE----- From 6f0674e14e0316fcb64ce1009891fefe1aa69832 Mon Sep 17 00:00:00 2001 From: keesspoelstra Date: Tue, 14 Feb 2023 23:07:46 +0800 Subject: [PATCH 07/16] Http2 to origin fix (#4) * Fix for connection level window mismatch + force connection level window update * Fix for HPACK corruption on closed streams Todo: check stream level error returns, these should still decode the headers to prevent HPACK corruption for following header frames --------- Co-authored-by: Kees Spoelstra --- proxy/http2/Http2CommonSession.cc | 3 +++ proxy/http2/Http2ConnectionState.cc | 21 ++++++++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/proxy/http2/Http2CommonSession.cc b/proxy/http2/Http2CommonSession.cc index 0484134d203..ad038300e1b 100644 --- a/proxy/http2/Http2CommonSession.cc +++ b/proxy/http2/Http2CommonSession.cc @@ -369,6 +369,7 @@ Http2CommonSession::do_process_frame_read(int event, VIO *vio, bool inside_frame // Return if there was an error if (err > Http2ErrorCode::HTTP2_ERROR_NO_ERROR || do_start_frame_read(err) < 0) { // send an error if specified. Otherwise, just go away + this->connection_state.restart_receiving(nullptr); if (err > Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { if (!this->connection_state.is_state_closed()) { this->connection_state.send_goaway_frame(this->connection_state.get_latest_stream_id_in(), err); @@ -387,6 +388,7 @@ Http2CommonSession::do_process_frame_read(int event, VIO *vio, bool inside_frame if (this->_should_do_something_else()) { if (this->_reenable_event == nullptr) { + this->connection_state.restart_receiving(nullptr); vio->disable(); this->_reenable_event = this->get_mutex()->thread_holding->schedule_in(this->get_proxy_session(), HRTIME_MSECONDS(1), HTTP2_SESSION_EVENT_REENABLE, vio); @@ -397,6 +399,7 @@ Http2CommonSession::do_process_frame_read(int event, VIO *vio, bool inside_frame // If the client hasn't shut us down, reenable if (!this->get_proxy_session()->is_peer_closed()) { + this->connection_state.restart_receiving(nullptr); vio->reenable(); } return 0; diff --git a/proxy/http2/Http2ConnectionState.cc b/proxy/http2/Http2ConnectionState.cc index 7e3cd6bd488..aec4094b612 100644 --- a/proxy/http2/Http2ConnectionState.cc +++ b/proxy/http2/Http2ConnectionState.cc @@ -89,6 +89,9 @@ Http2ConnectionState::rcv_data_frame(const Http2Frame &frame) Http2StreamDebug(this->session, id, "Received DATA frame"); + // Update connection window size, before any stream specific handling + this->decrement_local_rwnd(payload_length); + if (this->get_zombie_event()) { Warning("Data frame for zombied session %" PRId64, this->session->get_connection_id()); } @@ -174,7 +177,8 @@ Http2ConnectionState::rcv_data_frame(const Http2Frame &frame) } // Check whether Window Size is acceptable - if (!this->_local_rwnd_is_shrinking && this->get_local_rwnd() < payload_length) { + // compare to 0 because we already decreased the connection rwnd with payload_length + if (!this->_local_rwnd_is_shrinking && this->get_local_rwnd() < 0) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_FLOW_CONTROL_ERROR, "recv data this->local_rwnd < payload_length"); } @@ -183,8 +187,7 @@ Http2ConnectionState::rcv_data_frame(const Http2Frame &frame) "recv data stream->local_rwnd < payload_length"); } - // Update Window size - this->decrement_local_rwnd(payload_length); + // Update stream window size stream->decrement_local_rwnd(payload_length); if (is_debug_tag_set("http2_con")) { @@ -316,9 +319,17 @@ Http2ConnectionState::rcv_headers_frame(const Http2Frame &frame) } } - // Ignoring HEADERS frame on a closed stream. The HdrHeap has gone away and it will core. + // HEADERS frame on a closed stream. The HdrHeap has gone away and it will core. if (stream->get_state() == Http2StreamState::HTTP2_STREAM_STATE_CLOSED) { - return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); + Http2StreamDebug(session, stream_id, "Replaced closed stream"); + free_stream_after_decoding = true; + stream = THREAD_ALLOC_INIT(http2StreamAllocator, this_ethread(), session->get_proxy_session(), stream_id, + peer_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE), true, false); + if (!stream) { + // This happening is possibly catastrophic, the HPACK tables can be out of sync + // Maybe this is a connection level error? + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); + } } Http2HeadersParameter params; From 85c3826d16c8a3ca93cedb60a307da8e494255fa Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Thu, 2 Mar 2023 17:01:57 +0000 Subject: [PATCH 08/16] proxy.config.exec_thread.autoconfig -> ''.enabled This is a changed needed to the new HTTP/2 to origin tests for the records.yaml update. --- tests/gold_tests/h2/h2origin.test.py | 2 +- tests/gold_tests/h2/h2origin_single_thread.test.py | 2 +- tests/gold_tests/h2/http2_rst_stream.test.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/gold_tests/h2/h2origin.test.py b/tests/gold_tests/h2/h2origin.test.py index 23ebbac887f..cedbc7d00b5 100644 --- a/tests/gold_tests/h2/h2origin.test.py +++ b/tests/gold_tests/h2/h2origin.test.py @@ -37,7 +37,7 @@ 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'http', - 'proxy.config.exec_thread.autoconfig': 0, + 'proxy.config.exec_thread.autoconfig.enabled': 0, # Allow for more parallelism 'proxy.config.exec_thread.limit': 4, 'proxy.config.ssl.client.alpn_protocols': 'h2,http/1.1', diff --git a/tests/gold_tests/h2/h2origin_single_thread.test.py b/tests/gold_tests/h2/h2origin_single_thread.test.py index 7e10ba06cbe..4a6dc26ff5e 100644 --- a/tests/gold_tests/h2/h2origin_single_thread.test.py +++ b/tests/gold_tests/h2/h2origin_single_thread.test.py @@ -37,7 +37,7 @@ 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'http', - 'proxy.config.exec_thread.autoconfig': 0, + 'proxy.config.exec_thread.autoconfig.enabled': 0, # Limiting ourselves to 1 thread to exercise origin reuse 'proxy.config.exec_thread.limit': 1, 'proxy.config.ssl.client.alpn_protocols': 'h2,http/1.1', diff --git a/tests/gold_tests/h2/http2_rst_stream.test.py b/tests/gold_tests/h2/http2_rst_stream.test.py index c4e634fc1f0..f205ee5ac6d 100644 --- a/tests/gold_tests/h2/http2_rst_stream.test.py +++ b/tests/gold_tests/h2/http2_rst_stream.test.py @@ -40,7 +40,7 @@ 'proxy.config.ssl.client.verify.server.policy': 'PERMISSIVE', 'proxy.config.diags.debug.enabled': 3, 'proxy.config.diags.debug.tags': 'http', - 'proxy.config.exec_thread.autoconfig': 0, + 'proxy.config.exec_thread.autoconfig.enabled': 0, 'proxy.config.exec_thread.limit': 4, 'proxy.config.ssl.client.alpn_protocols': 'h2,http/1.1', 'proxy.config.http.server_session_sharing.pool': 'thread', @@ -99,7 +99,7 @@ 'proxy.config.ssl.client.verify.server.policy': 'PERMISSIVE', 'proxy.config.diags.debug.enabled': 3, 'proxy.config.diags.debug.tags': 'http', - 'proxy.config.exec_thread.autoconfig': 0, + 'proxy.config.exec_thread.autoconfig.enabled': 0, 'proxy.config.exec_thread.limit': 4, 'proxy.config.ssl.client.alpn_protocols': 'h2,http/1.1', 'proxy.config.http.server_session_sharing.pool': 'thread', @@ -158,7 +158,7 @@ 'proxy.config.ssl.client.verify.server.policy': 'PERMISSIVE', 'proxy.config.diags.debug.enabled': 3, 'proxy.config.diags.debug.tags': 'http', - 'proxy.config.exec_thread.autoconfig': 0, + 'proxy.config.exec_thread.autoconfig.enabled': 0, 'proxy.config.exec_thread.limit': 1, 'proxy.config.ssl.client.alpn_protocols': 'h2,http/1.1', 'proxy.config.http.server_session_sharing.pool': 'thread', From 87a9ab7613b3e1bd77250670711632c4b98b625c Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Mon, 6 Mar 2023 17:49:57 +0000 Subject: [PATCH 09/16] Revert a restart_receiving call causing POST issues The post-continue.test.py AuTest demonstrated that the restart_receiving call was causing the first 64KB of data from a POST to be lost. --- proxy/http2/Http2CommonSession.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/proxy/http2/Http2CommonSession.cc b/proxy/http2/Http2CommonSession.cc index ad038300e1b..88626d47cfc 100644 --- a/proxy/http2/Http2CommonSession.cc +++ b/proxy/http2/Http2CommonSession.cc @@ -399,7 +399,6 @@ Http2CommonSession::do_process_frame_read(int event, VIO *vio, bool inside_frame // If the client hasn't shut us down, reenable if (!this->get_proxy_session()->is_peer_closed()) { - this->connection_state.restart_receiving(nullptr); vio->reenable(); } return 0; From 2503739d708ebb4602dd2e83e69b0e4573848413 Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Fri, 10 Mar 2023 23:23:25 +0000 Subject: [PATCH 10/16] Expand tunnel_handler to expect VC_EVENT_INACTIVITY_TIMEOUT The HttpSM::tunnel_handler expects the tunnel to be going away and asserts that the event it receives is HTTP_TUNNEL_EVENT_DONE. However, the tunnel can be going away due to timeout as well. Adding VC_EVENT_INACTIVITY_TIMEOUT as a possible event in that function. --- proxy/http/HttpSM.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index b481c7b02d0..e7b8219f6d9 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -3146,7 +3146,7 @@ HttpSM::tunnel_handler(int event, void *data) { STATE_ENTER(&HttpSM::tunnel_handler, event); - ink_assert(event == HTTP_TUNNEL_EVENT_DONE); + ink_assert(event == HTTP_TUNNEL_EVENT_DONE || event == VC_EVENT_INACTIVITY_TIMEOUT); // The tunnel calls this when it is done terminate_sm = true; From d33f6659d828ece6a2355ce61a413bbb56ba5019 Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Thu, 16 Mar 2023 14:47:36 +0000 Subject: [PATCH 11/16] remove the buffer reader from the consumer's vc --- proxy/http/HttpSM.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index e7b8219f6d9..33a1a8d603b 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -3498,6 +3498,11 @@ HttpSM::tunnel_handler_100_continue_ua(int event, HttpTunnelConsumer *c) // real response header is received ua_entry->in_tunnel = false; c->write_success = true; + + // remove the buffer reader from the consumer's vc + if (c->vc != nullptr) { + c->vc->do_io_write(); + } } return 0; From 3dd00955947e4ac7bfe5a3eab6fad88245345953 Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Tue, 21 Mar 2023 15:02:09 +0000 Subject: [PATCH 12/16] Add back in Http2::min_avg_window_update check --- proxy/http2/Http2ConnectionState.cc | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/proxy/http2/Http2ConnectionState.cc b/proxy/http2/Http2ConnectionState.cc index aec4094b612..e9a932541f7 100644 --- a/proxy/http2/Http2ConnectionState.cc +++ b/proxy/http2/Http2ConnectionState.cc @@ -2648,13 +2648,12 @@ Http2ConnectionState::increment_peer_rwnd(size_t amount) this->_recent_rwnd_increment[this->_recent_rwnd_increment_index] = amount; ++this->_recent_rwnd_increment_index; this->_recent_rwnd_increment_index %= this->_recent_rwnd_increment.size(); - // SKH Causing problems with gRPC processing. Python example resulted in amount 8 - // double sum = std::accumulate(this->_recent_rwnd_increment.begin(), this->_recent_rwnd_increment.end(), 0.0); - // double avg = sum / this->_recent_rwnd_increment.size(); - // if (avg < Http2::min_avg_window_update) { - // HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_INSUFFICIENT_AVG_WINDOW_UPDATE, this_ethread()); - // return Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM; - //} + double sum = std::accumulate(this->_recent_rwnd_increment.begin(), this->_recent_rwnd_increment.end(), 0.0); + double avg = sum / this->_recent_rwnd_increment.size(); + if (avg < Http2::min_avg_window_update) { + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_INSUFFICIENT_AVG_WINDOW_UPDATE, this_ethread()); + return Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM; + } return Http2ErrorCode::HTTP2_ERROR_NO_ERROR; } From ba0b41af32d42ac12f95d9f188beea82418f03bf Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Wed, 22 Mar 2023 20:24:56 +0000 Subject: [PATCH 13/16] Naming and test updates This removes some incorrectly added _ prefixes from public variables and renames some Http2ConnectionState ndone manipulation functions to make them more readable. It also reverts some test changes. --- proxy/http/ConnectingEntry.cc | 62 +++++++++---------- proxy/http/ConnectingEntry.h | 6 +- proxy/http/HttpSM.cc | 26 ++++---- proxy/http2/Http2ConnectionState.cc | 19 ++---- proxy/http2/Http2ConnectionState.h | 1 - proxy/http2/Http2Stream.h | 12 ++-- proxy/http2/unit_tests/test_HTTP2.cc | 4 +- tests/gold_tests/h2/gold/nghttp_0_stdout.gold | 2 + tests/gold_tests/h2/http2.test.py | 6 +- 9 files changed, 62 insertions(+), 76 deletions(-) diff --git a/proxy/http/ConnectingEntry.cc b/proxy/http/ConnectingEntry.cc index 255aa90f412..bbbaba5bfdb 100644 --- a/proxy/http/ConnectingEntry.cc +++ b/proxy/http/ConnectingEntry.cc @@ -40,8 +40,8 @@ ConnectingEntry::state_http_server_open(int event, void *data) switch (event) { case NET_EVENT_OPEN: { - _netvc = static_cast(data); - UnixNetVConnection *vc = static_cast(_netvc); + netvc = static_cast(data); + UnixNetVConnection *vc = static_cast(netvc); ink_release_assert(_pending_action == nullptr || _pending_action->continuation == vc->get_action()->continuation); _pending_action = nullptr; Debug("http_connect", "ConnectingEntrysetting handler for connection handshake"); @@ -49,11 +49,11 @@ ConnectingEntry::state_http_server_open(int event, void *data) // The buffer we create will be handed over to the eventually created server session _netvc_read_buffer = new_MIOBuffer(HTTP_SERVER_RESP_HDR_BUFFER_INDEX); _netvc_reader = _netvc_read_buffer->alloc_reader(); - _netvc->do_io_write(this, 1, _netvc_reader); - ink_release_assert(!_connect_sms.empty()); - if (!_connect_sms.empty()) { - HttpSM *prime_connect_sm = *(_connect_sms.begin()); - _netvc->set_inactivity_timeout(prime_connect_sm->get_server_connect_timeout()); + netvc->do_io_write(this, 1, _netvc_reader); + ink_release_assert(!connect_sms.empty()); + if (!connect_sms.empty()) { + HttpSM *prime_connect_sm = *(connect_sms.begin()); + netvc->set_inactivity_timeout(prime_connect_sm->get_server_connect_timeout()); } ink_release_assert(_pending_action == nullptr); return 0; @@ -61,27 +61,27 @@ ConnectingEntry::state_http_server_open(int event, void *data) case VC_EVENT_READ_COMPLETE: case VC_EVENT_WRITE_READY: case VC_EVENT_WRITE_COMPLETE: { - Debug("http_connect", "Kick off %zd state machines waiting for origin", _connect_sms.size()); + Debug("http_connect", "Kick off %zd state machines waiting for origin", connect_sms.size()); this->remove_entry(); - _netvc->do_io_write(nullptr, 0, nullptr); - if (!_connect_sms.empty()) { - auto prime_iter = _connect_sms.rbegin(); - ink_release_assert(prime_iter != _connect_sms.rend()); - PoolableSession *new_session = (*prime_iter)->create_server_session(_netvc, _netvc_read_buffer, _netvc_reader); - _netvc = nullptr; + netvc->do_io_write(nullptr, 0, nullptr); + if (!connect_sms.empty()) { + auto prime_iter = connect_sms.rbegin(); + ink_release_assert(prime_iter != connect_sms.rend()); + PoolableSession *new_session = (*prime_iter)->create_server_session(netvc, _netvc_read_buffer, _netvc_reader); + netvc = nullptr; _netvc_read_buffer = nullptr; // Did we end up with a multiplexing session? int count = 0; if (new_session->is_multiplexing()) { // Hand off to all queued up ConnectSM's. - while (!_connect_sms.empty()) { + while (!connect_sms.empty()) { Debug("http_connect", "ConnectingEntry Pass along CONNECT_EVENT_TXN %d", count++); - auto entry = _connect_sms.begin(); + auto entry = connect_sms.begin(); SCOPED_MUTEX_LOCK(lock, (*entry)->mutex, this_ethread()); (*entry)->handleEvent(CONNECT_EVENT_TXN, new_session); - _connect_sms.erase(entry); + connect_sms.erase(entry); } } else { // Hand off to one and tell all of the others to connect directly @@ -89,14 +89,14 @@ ConnectingEntry::state_http_server_open(int event, void *data) { SCOPED_MUTEX_LOCK(lock, (*prime_iter)->mutex, this_ethread()); (*prime_iter)->handleEvent(CONNECT_EVENT_TXN, new_session); - _connect_sms.erase((++prime_iter).base()); + connect_sms.erase((++prime_iter).base()); } - while (!_connect_sms.empty()) { - auto entry = _connect_sms.begin(); + while (!connect_sms.empty()) { + auto entry = connect_sms.begin(); Debug("http_connect", "ConnectingEntry Pass along CONNECT_EVENT_DIRECT %d", count++); SCOPED_MUTEX_LOCK(lock, (*entry)->mutex, this_ethread()); (*entry)->handleEvent(CONNECT_EVENT_DIRECT, nullptr); - _connect_sms.erase(entry); + connect_sms.erase(entry); } } } else { @@ -111,22 +111,22 @@ ConnectingEntry::state_http_server_open(int event, void *data) case VC_EVENT_ACTIVE_TIMEOUT: case VC_EVENT_ERROR: case NET_EVENT_OPEN_FAILED: { - Debug("http_connect", "Stop %zd state machines waiting for failed origin", _connect_sms.size()); + Debug("http_connect", "Stop %zd state machines waiting for failed origin", connect_sms.size()); this->remove_entry(); int vc_provided_cert = 0; int lerrno = EIO; - if (_netvc != nullptr) { - vc_provided_cert = _netvc->provided_cert(); - lerrno = _netvc->lerrno == 0 ? lerrno : _netvc->lerrno; - _netvc->do_io_close(); + if (netvc != nullptr) { + vc_provided_cert = netvc->provided_cert(); + lerrno = netvc->lerrno == 0 ? lerrno : netvc->lerrno; + netvc->do_io_close(); } - while (!_connect_sms.empty()) { - auto entry = _connect_sms.begin(); + while (!connect_sms.empty()) { + auto entry = connect_sms.begin(); SCOPED_MUTEX_LOCK(lock, (*entry)->mutex, this_ethread()); (*entry)->t_state.set_connect_fail(lerrno); (*entry)->server_connection_provided_cert = vc_provided_cert; (*entry)->handleEvent(event, data); - _connect_sms.erase(entry); + connect_sms.erase(entry); } // ConnectingEntry should remove itself from the tables and delete itself delete this; @@ -146,8 +146,8 @@ void ConnectingEntry::remove_entry() { EThread *ethread = this_ethread(); - auto ip_iter = ethread->connecting_pool->m_ip_pool.find(this->_ipaddr); - while (ip_iter != ethread->connecting_pool->m_ip_pool.end() && this->_ipaddr == ip_iter->first) { + auto ip_iter = ethread->connecting_pool->m_ip_pool.find(this->ipaddr); + while (ip_iter != ethread->connecting_pool->m_ip_pool.end() && this->ipaddr == ip_iter->first) { if (ip_iter->second == this) { ethread->connecting_pool->m_ip_pool.erase(ip_iter); break; diff --git a/proxy/http/ConnectingEntry.h b/proxy/http/ConnectingEntry.h index 3427295a4b8..f612fa4258d 100644 --- a/proxy/http/ConnectingEntry.h +++ b/proxy/http/ConnectingEntry.h @@ -43,10 +43,10 @@ class ConnectingEntry : public Continuation public: std::string sni; std::string cert_name; - IpEndpoint _ipaddr; + IpEndpoint ipaddr; std::string hostname; - std::set _connect_sms; - NetVConnection *_netvc = nullptr; + std::set connect_sms; + NetVConnection *netvc = nullptr; private: MIOBuffer *_netvc_read_buffer = nullptr; diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 33a1a8d603b..da74312a0e5 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -2297,13 +2297,13 @@ HttpSM::cancel_pending_server_connection() ConnectingEntry *connecting_entry = ip_iter->second; // Found a match // Look for our sm in the queue - auto entry = connecting_entry->_connect_sms.find(this); - if (entry != connecting_entry->_connect_sms.end()) { - connecting_entry->_connect_sms.erase(entry); - if (connecting_entry->_connect_sms.empty()) { - if (connecting_entry->_netvc) { - connecting_entry->_netvc->do_io_write(nullptr, 0, nullptr); - connecting_entry->_netvc->do_io_close(); + auto entry = connecting_entry->connect_sms.find(this); + if (entry != connecting_entry->connect_sms.end()) { + connecting_entry->connect_sms.erase(entry); + if (connecting_entry->connect_sms.empty()) { + if (connecting_entry->netvc) { + connecting_entry->netvc->do_io_write(nullptr, 0, nullptr); + connecting_entry->netvc->do_io_close(); } ethread->connecting_pool->m_ip_pool.erase(ip_iter); delete connecting_entry; @@ -2347,12 +2347,12 @@ HttpSM::add_to_existing_request() while (!retval && ip_iter != ethread->connecting_pool->m_ip_pool.end() && ip_iter->first == ip) { // Check that entry matches sni, hostname, and cert if (proposed_hostname == ip_iter->second->hostname && proposed_sni == ip_iter->second->sni && - proposed_cert == ip_iter->second->cert_name && ip_iter->second->_connect_sms.size() < 50) { + proposed_cert == ip_iter->second->cert_name && ip_iter->second->connect_sms.size() < 50) { // Pre-emptively set a server connect failure that will be cleared once a WRITE_READY is received from origin or // bytes are received back this->t_state.set_connect_fail(EIO); - ip_iter->second->_connect_sms.insert(this); - Debug("http_connect", "Add entry to connection queue. size=%" PRId64, ip_iter->second->_connect_sms.size()); + ip_iter->second->connect_sms.insert(this); + Debug("http_connect", "Add entry to connection queue. size=%" PRId64, ip_iter->second->connect_sms.size()); retval = true; break; } @@ -5677,13 +5677,13 @@ HttpSM::do_http_server_open(bool raw, bool only_direct) new_entry = new ConnectingEntry(); new_entry->mutex = this->mutex; new_entry->handler = (ContinuationHandler)&ConnectingEntry::state_http_server_open; - new_entry->_ipaddr.assign(&t_state.current.server->dst_addr.sa); + new_entry->ipaddr.assign(&t_state.current.server->dst_addr.sa); new_entry->hostname = t_state.current.server->name; new_entry->sni = this->get_outbound_sni(); new_entry->cert_name = this->get_outbound_cert(); this->t_state.set_connect_fail(EIO); - new_entry->_connect_sms.insert(this); - ethread->connecting_pool->m_ip_pool.insert(std::make_pair(new_entry->_ipaddr, new_entry)); + new_entry->connect_sms.insert(this); + ethread->connecting_pool->m_ip_pool.insert(std::make_pair(new_entry->ipaddr, new_entry)); } } diff --git a/proxy/http2/Http2ConnectionState.cc b/proxy/http2/Http2ConnectionState.cc index e9a932541f7..ab14a52c151 100644 --- a/proxy/http2/Http2ConnectionState.cc +++ b/proxy/http2/Http2ConnectionState.cc @@ -161,7 +161,7 @@ Http2ConnectionState::rcv_data_frame(const Http2Frame &frame) // Pure END_STREAM if (payload_length == 0) { - if (stream->read_enabled()) { + if (stream->is_read_enabled()) { stream->signal_read_event(VC_EVENT_READ_COMPLETE); } return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); @@ -224,16 +224,16 @@ Http2ConnectionState::rcv_data_frame(const Http2Frame &frame) return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_INTERNAL_ERROR, "Write mismatch"); } myreader->consume(num_written); - stream->read_update(num_written); + stream->update_read_length(num_written); } myreader->writer()->dealloc_reader(myreader); if (frame.header().flags & HTTP2_FLAGS_DATA_END_STREAM) { // TODO: set total written size to read_vio.nbytes - stream->read_done(); + stream->set_read_done(); } - if (stream->read_enabled()) { + if (stream->is_read_enabled()) { if (frame.header().flags & HTTP2_FLAGS_DATA_END_STREAM) { if (this->get_peer_stream_count() > 1 && this->get_local_rwnd() == 0) { // This final DATA frame for this stream consumed all the bytes for the @@ -1697,17 +1697,6 @@ Http2ConnectionState::find_stream(Http2StreamId id) const return nullptr; } -void -Http2ConnectionState::start_streams() -{ - Http2Stream *s = stream_list.head; - while (s) { - Http2Stream *next = static_cast(s->link.next); - s->reenable_write(); - s = next; - } -} - void Http2ConnectionState::restart_streams() { diff --git a/proxy/http2/Http2ConnectionState.h b/proxy/http2/Http2ConnectionState.h index fe62136ed92..44c0cbe8f71 100644 --- a/proxy/http2/Http2ConnectionState.h +++ b/proxy/http2/Http2ConnectionState.h @@ -126,7 +126,6 @@ class Http2ConnectionState : public Continuation void set_stream_id(Http2Stream *stream); Http2Stream *find_stream(Http2StreamId id) const; void restart_streams(); - void start_streams(); bool delete_stream(Http2Stream *stream); void release_stream(); void cleanup_streams(); diff --git a/proxy/http2/Http2Stream.h b/proxy/http2/Http2Stream.h index 87a0d95059e..01b93210f4d 100644 --- a/proxy/http2/Http2Stream.h +++ b/proxy/http2/Http2Stream.h @@ -137,8 +137,8 @@ class Http2Stream : public ProxyTransaction return &_send_header; } - void read_update(int count); - void read_done(); + void update_read_length(int count); + void set_read_done(); void clear_io_events(); @@ -169,7 +169,7 @@ class Http2Stream : public ProxyTransaction void reset_send_headers(); MIOBuffer *read_vio_writer() const; int64_t read_vio_read_avail(); - bool read_enabled() const; + bool is_read_enabled() const; ////////////////// // Variables @@ -409,7 +409,7 @@ Http2Stream::read_vio_writer() const } inline bool -Http2Stream::read_enabled() const +Http2Stream::is_read_enabled() const { return !this->read_vio.is_disabled(); } @@ -422,13 +422,13 @@ Http2Stream::_clear_timers() } inline void -Http2Stream::read_update(int count) +Http2Stream::update_read_length(int count) { read_vio.ndone += count; } inline void -Http2Stream::read_done() +Http2Stream::set_read_done() { read_vio.nbytes = read_vio.ndone; } diff --git a/proxy/http2/unit_tests/test_HTTP2.cc b/proxy/http2/unit_tests/test_HTTP2.cc index cb4dcf2e8a2..999252cf287 100644 --- a/proxy/http2/unit_tests/test_HTTP2.cc +++ b/proxy/http2/unit_tests/test_HTTP2.cc @@ -99,7 +99,7 @@ TEST_CASE("Convert HTTPHdr", "[HTTP2]") http2_convert_header_from_2_to_1_1(&hdr_2); // dump - char buf[128] = {0}; + char buf[1024] = {0}; int bufindex = 0; int dumpoffset = 0; @@ -154,7 +154,7 @@ TEST_CASE("Convert HTTPHdr", "[HTTP2]") http2_convert_header_from_2_to_1_1(&hdr_2); // dump - char buf[128] = {0}; + char buf[1024] = {0}; int bufindex = 0; int dumpoffset = 0; diff --git a/tests/gold_tests/h2/gold/nghttp_0_stdout.gold b/tests/gold_tests/h2/gold/nghttp_0_stdout.gold index f19a43516d3..e8e9acabd41 100644 --- a/tests/gold_tests/h2/gold/nghttp_0_stdout.gold +++ b/tests/gold_tests/h2/gold/nghttp_0_stdout.gold @@ -12,3 +12,5 @@ `` [``] recv (stream_id=1) :status: 200 `` +``; END_STREAM +`` diff --git a/tests/gold_tests/h2/http2.test.py b/tests/gold_tests/h2/http2.test.py index 3d203970228..11e6fb109b7 100644 --- a/tests/gold_tests/h2/http2.test.py +++ b/tests/gold_tests/h2/http2.test.py @@ -26,7 +26,7 @@ Test.SkipUnless( Condition.HasCurlFeature('http2') ) -#Test.ContinueOnFail = True +Test.ContinueOnFail = True # ---- # Setup Origin Server @@ -192,7 +192,6 @@ post_body, ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = "gold/post_chunked.gold" -tr.TimeOut = 60 tr.StillRunningAfter = server # Test Case 7: Post with big chunked body @@ -203,7 +202,6 @@ ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = "gold/post_chunked.gold" -tr.TimeOut = 60 tr.StillRunningAfter = server # Test Case 8: Huge response header @@ -213,7 +211,6 @@ tr.Processes.Default.Streams.stdout = "gold/http2_8_stdout.gold" # Different versions of curl will have different cases for HTTP/2 field names. tr.Processes.Default.Streams.stderr = Testers.GoldFile("gold/http2_8_stderr.gold", case_insensitive=True) -tr.TimeOut = 60 tr.StillRunningAfter = server # Test Case 9: Header Only Response - e.g. 204 @@ -223,5 +220,4 @@ tr.Processes.Default.Streams.stdout = "gold/http2_9_stdout.gold" # Different versions of curl will have different cases for HTTP/2 field names. tr.Processes.Default.Streams.stderr = Testers.GoldFile("gold/http2_9_stderr.gold", case_insensitive=True) -tr.TimeOut = 60 tr.StillRunningAfter = server From 02e00f6278f19f6ebc5f0e208ad29ce08238c4ef Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Thu, 23 Mar 2023 22:52:05 +0000 Subject: [PATCH 14/16] Enable stream error rate check on outbound streams. --- proxy/http2/Http2CommonSession.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/proxy/http2/Http2CommonSession.cc b/proxy/http2/Http2CommonSession.cc index 88626d47cfc..be81a886592 100644 --- a/proxy/http2/Http2CommonSession.cc +++ b/proxy/http2/Http2CommonSession.cc @@ -355,8 +355,7 @@ Http2CommonSession::do_process_frame_read(int event, VIO *vio, bool inside_frame } Http2ErrorCode err = Http2ErrorCode::HTTP2_ERROR_NO_ERROR; - if (this->connection_state.get_stream_error_rate() > std::min(1.0, Http2::stream_error_rate_threshold * 2.0) && - !this->is_outbound()) { + if (this->connection_state.get_stream_error_rate() > std::min(1.0, Http2::stream_error_rate_threshold * 2.0)) { ip_port_text_buffer ipb; const char *peer_ip = ats_ip_ntop(this->get_proxy_session()->get_remote_addr(), ipb, sizeof(ipb)); SiteThrottledWarning("HTTP/2 session error peer_ip=%s session_id=%" PRId64 From f2199893a3d177d5e1c3ee82890c852b854b6e0f Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Fri, 24 Mar 2023 16:19:57 +0000 Subject: [PATCH 15/16] When converting h2 to h1, add Host as first header --- proxy/hdrs/VersionConverter.cc | 23 ++++++++++++++------- proxy/http2/unit_tests/test_HTTP2.cc | 31 +++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/proxy/hdrs/VersionConverter.cc b/proxy/hdrs/VersionConverter.cc index cd5efb98cac..b239f9aa346 100644 --- a/proxy/hdrs/VersionConverter.cc +++ b/proxy/hdrs/VersionConverter.cc @@ -190,16 +190,25 @@ VersionConverter::_convert_req_from_2_to_1(HTTPHdr &header) const if (MIMEField *field = header.field_find(PSEUDO_HEADER_AUTHORITY.data(), PSEUDO_HEADER_AUTHORITY.size()); field != nullptr && field->value_is_valid(is_control_BIT | is_ws_BIT)) { int authority_len; - // Set the host header field + const char *authority = field->value_get(&authority_len); + header.m_http->u.req.m_url_impl->set_host(header.m_heap, authority, authority_len, true); + MIMEField *host = header.field_find(MIME_FIELD_HOST, MIME_LEN_HOST); if (host == nullptr) { - host = header.field_create(MIME_FIELD_HOST, MIME_LEN_HOST); - header.field_attach(host); + // Add a Host header field. [RFC 7230] 5.4 says that if a client sends a + // Host header field, it SHOULD be the first header in the header section + // of a request. We accomplish that by simply renaming the :authority + // header as Host. + header.field_detach(field); + field->name_set(header.m_heap, header.m_mime, MIME_FIELD_HOST, MIME_LEN_HOST); + header.field_attach(field); + } else { + // There already is a Host header field. Simply set the value of the Host + // field to the current value of :authority and delete the :authority + // field. + host->value_set(header.m_heap, header.m_mime, authority, authority_len); + header.field_delete(field); } - const char *authority = field->value_get(&authority_len); - header.m_http->u.req.m_url_impl->set_host(header.m_heap, authority, authority_len, true); - host->value_set(header.m_heap, header.m_mime, authority, authority_len); - header.field_delete(field); } else { return PARSE_RESULT_ERROR; } diff --git a/proxy/http2/unit_tests/test_HTTP2.cc b/proxy/http2/unit_tests/test_HTTP2.cc index 999252cf287..8c828b8a12e 100644 --- a/proxy/http2/unit_tests/test_HTTP2.cc +++ b/proxy/http2/unit_tests/test_HTTP2.cc @@ -90,7 +90,7 @@ TEST_CASE("Convert HTTPHdr", "[HTTP2]") CHECK(v == "/index.html"); } - // convert to HTTP/1.1 + // convert back to HTTP/1.1 HTTPHdr hdr_2; ts::PostScript hdr_2_defer([&]() -> void { hdr_2.destroy(); }); hdr_2.create(HTTP_TYPE_REQUEST); @@ -106,6 +106,35 @@ TEST_CASE("Convert HTTPHdr", "[HTTP2]") hdr_2.print(buf, sizeof(buf), &bufindex, &dumpoffset); // check + CHECK_THAT(buf, Catch::StartsWith("GET https://trafficserver.apache.org/index.html HTTP/1.1\r\n" + "Host: trafficserver.apache.org\r\n" + "User-Agent: foobar\r\n" + "\r\n")); + + // Verify that conversion from HTTP/2 to HTTP/1.1 works correctly when the + // HTTP/2 request contains a Host header. + HTTPHdr hdr_2_with_host; + ts::PostScript hdr_2_with_host_defer([&]() -> void { hdr_2_with_host.destroy(); }); + hdr_2_with_host.create(HTTP_TYPE_REQUEST); + hdr_2_with_host.copy(&hdr_1); + + MIMEField *host = hdr_2_with_host.field_create(MIME_FIELD_HOST, MIME_LEN_HOST); + hdr_2_with_host.field_attach(host); + std::string_view host_value = "bogus.host.com"; + host->value_set(hdr_2_with_host.m_heap, hdr_2_with_host.m_mime, host_value.data(), host_value.size()); + + http2_convert_header_from_2_to_1_1(&hdr_2_with_host); + + // dump + memset(buf, 0, sizeof(buf)); + bufindex = 0; + dumpoffset = 0; + + hdr_2_with_host.print(buf, sizeof(buf), &bufindex, &dumpoffset); + + // check: Note that the Host will now be at the end of the Headers since we + // added it above and it will remain there, albeit with the updated value + // from the :authority header. CHECK_THAT(buf, Catch::StartsWith("GET https://trafficserver.apache.org/index.html HTTP/1.1\r\n" "User-Agent: foobar\r\n" "Host: trafficserver.apache.org\r\n" From 7c65be82299d491da076c452f6601eee3ab9e324 Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Mon, 27 Mar 2023 17:12:35 +0000 Subject: [PATCH 16/16] Remove window check from update_write_request If we short circuit our update_write_request based upon HTTP/2 windows, then we can get into a situation where we are not flushing DATA frames to the client, but are internally accounting for them. Eventually we stop trying to flush because our window is depleted, which results in us never flushing. This manifested itself as DATA frames getting stuck in our VIO buffers. --- proxy/http2/Http2ConnectionState.cc | 4 ++-- proxy/http2/Http2Stream.cc | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/proxy/http2/Http2ConnectionState.cc b/proxy/http2/Http2ConnectionState.cc index ab14a52c151..a8bf54002ce 100644 --- a/proxy/http2/Http2ConnectionState.cc +++ b/proxy/http2/Http2ConnectionState.cc @@ -87,7 +87,7 @@ Http2ConnectionState::rcv_data_frame(const Http2Frame &frame) uint8_t pad_length = 0; const uint32_t payload_length = frame.header().length; - Http2StreamDebug(this->session, id, "Received DATA frame"); + Http2StreamDebug(this->session, id, "Received DATA frame, flags: %d", frame.header().flags); // Update connection window size, before any stream specific handling this->decrement_local_rwnd(payload_length); @@ -2092,7 +2092,7 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len _peer_rwnd, stream->get_peer_rwnd(), payload_length, flags); Http2DataFrame data(stream->get_id(), flags, resp_reader, payload_length); - this->session->xmit(data); + this->session->xmit(data, flags & HTTP2_FLAGS_DATA_END_STREAM); if (flags & HTTP2_FLAGS_DATA_END_STREAM) { Http2StreamDebug(session, stream->get_id(), "END_STREAM"); diff --git a/proxy/http2/Http2Stream.cc b/proxy/http2/Http2Stream.cc index 23fbe3bd635..e9c66d46764 100644 --- a/proxy/http2/Http2Stream.cc +++ b/proxy/http2/Http2Stream.cc @@ -729,9 +729,8 @@ Http2Stream::update_write_request(bool call_update) SCOPED_MUTEX_LOCK(lock, write_vio.mutex, this_ethread()); IOBufferReader *vio_reader = write_vio.get_reader(); - if (write_vio.ntodo() > 0 && (!vio_reader->is_read_avail_more_than(0) || - // If there is no window left, just give up now too until we receive a WINDOW_UPDATE. - std::min(_peer_rwnd, this->get_connection_state().get_peer_rwnd()) == 0)) { + + if (write_vio.ntodo() > 0 && (!vio_reader->is_read_avail_more_than(0))) { Http2StreamDebug("update_write_request give up without doing anything ntodo=%" PRId64 " is_read_avail=%d client_window=%" PRId64 " session_window=%" PRId64, write_vio.ntodo(), vio_reader->is_read_avail_more_than(0), _peer_rwnd,