diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index 114fd3886a6..d6355ba4a72 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -3859,6 +3859,13 @@ Client-Related Configuration Enables (``1``) or disables (``0``) TLSv1_3 in the ATS client context. If not specified, enabled by default +.. ts:cv:: CONFIG proxy.config.ssl.client.alpn_protocols STRING "" + + Set the alpn string that ATS will send to origin during new connections. By default no ALPN string will be set. + To enable HTTP/2 communication to the origin, set this to "h2,http1.1". + + :overridable: + .. ts:cv:: CONFIG proxy.config.ssl.async.handshake.enabled INT 0 Enables the use of OpenSSL async job during the TLS handshake. Traffic @@ -3964,7 +3971,17 @@ HTTP/2 Configuration .. ts:cv:: CONFIG proxy.config.http2.initial_window_size_in INT 65535 :reloadable: - The initial window size for inbound connections. + The initial window size for inbound connection streams. + +.. ts:cv:: CONFIG proxy.config.http2.session_initial_window_size_in INT 0 + :reloadable: + + The initial window size for inbound connection session. HTTP/2 provides both + a per stream window and a session wide window. Each data byte exchanged decrements + the window of the associated stream and the session window. To allow for multiple + active streams, the session window should be larger than the stream window. + |TS| verifies that the session initial window is always at least as large as the + stream initial window. .. ts:cv:: CONFIG proxy.config.http2.max_frame_size INT 16384 :reloadable: diff --git a/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst b/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst index 37cdcb06f77..4835960c64b 100644 --- a/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst @@ -168,6 +168,21 @@ HTTP/2 Represents the current number of HTTP/2 active connections from client to the |TS|. +.. ts:stat:: global proxy.process.http2.total_server_connections integer + :type: counter + + Represents the total number of HTTP/2 connections from |TS| to the origin. + +.. ts:stat:: global proxy.process.http2.current_server_connections integer + :type: gauge + + Represents the current number of HTTP/2 connections from |TS| to the origin. + +.. ts:stat:: global proxy.process.http2.current_active_server_connections integer + :type: gauge + + Represents the current number of HTTP/2 active connections from |TS| to the origin. + .. ts:stat:: global proxy.process.http2.connection_errors integer :type: counter diff --git a/doc/admin-guide/monitoring/statistics/core/http-transaction.en.rst b/doc/admin-guide/monitoring/statistics/core/http-transaction.en.rst index 07a6e60a0d8..9dd730c2be4 100644 --- a/doc/admin-guide/monitoring/statistics/core/http-transaction.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/http-transaction.en.rst @@ -165,6 +165,16 @@ HTTP/2 Represents the current number of HTTP/2 streams from client to the |TS|. +.. ts:stat:: global proxy.process.http2.total_server_streams integer + :type: counter + + Represents the total number of HTTP/2 streams from |TS| to the origin. + +.. ts:stat:: global proxy.process.http2.current_server_streams integer + :type: gauge + + Represents the current number of HTTP/2 streams from |TS| to the origin. + .. ts:stat:: global proxy.process.http2.total_transactions_time integer :type: counter :units: seconds diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index 9321dfe1794..8d3cc177711 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -884,6 +884,7 @@ typedef enum { TS_CONFIG_HTTP_CONNECT_DEAD_POLICY, TS_CONFIG_PLUGIN_VC_DEFAULT_BUFFER_INDEX, TS_CONFIG_PLUGIN_VC_DEFAULT_BUFFER_WATER_MARK, + TS_CONFIG_SSL_CLIENT_ALPN_PROTOCOLS, TS_CONFIG_LAST_ENTRY } TSOverridableConfigKey; diff --git a/include/tscore/ink_defs.h b/include/tscore/ink_defs.h index 35141368371..dc40bc812d3 100644 --- a/include/tscore/ink_defs.h +++ b/include/tscore/ink_defs.h @@ -110,6 +110,8 @@ countof(const T (&)[N]) #define MAP_SHARED_MAP_NORESERVE (MAP_SHARED) #endif +#define MAX_ALPN_STRING 30 + /* Variables */ extern int debug_level; diff --git a/iocore/eventsystem/I_EThread.h b/iocore/eventsystem/I_EThread.h index 49d74d896a3..3b15794713d 100644 --- a/iocore/eventsystem/I_EThread.h +++ b/iocore/eventsystem/I_EThread.h @@ -45,6 +45,7 @@ struct EventIO; class ServerSessionPool; class Event; class Continuation; +class ConnectingPool; enum ThreadType { REGULAR = 0, @@ -349,6 +350,7 @@ class EThread : public Thread Event *start_event = nullptr; ServerSessionPool *server_session_pool = nullptr; + ConnectingPool *connecting_pool = nullptr; /** Default handler used until it is overridden. diff --git a/iocore/eventsystem/I_Thread.h b/iocore/eventsystem/I_Thread.h index 8037d2a10c4..a7e3e0315fe 100644 --- a/iocore/eventsystem/I_Thread.h +++ b/iocore/eventsystem/I_Thread.h @@ -121,6 +121,7 @@ class Thread ProxyAllocator quicNetVCAllocator; ProxyAllocator http1ClientSessionAllocator; ProxyAllocator http2ClientSessionAllocator; + ProxyAllocator http2ServerSessionAllocator; ProxyAllocator http2StreamAllocator; ProxyAllocator httpSMAllocator; ProxyAllocator quicClientSessionAllocator; diff --git a/iocore/net/ALPNSupport.cc b/iocore/net/ALPNSupport.cc index 69b8b1c24d7..3d5832889e8 100644 --- a/iocore/net/ALPNSupport.cc +++ b/iocore/net/ALPNSupport.cc @@ -145,3 +145,36 @@ ALPNSupport::registerNextProtocolSet(SSLNextProtocolSet *s, const SessionProtoco this->npnSet = s; npnSet->create_npn_advertisement(protoenabled, &npn, &npnsz); } + +bool +ALPNSupport::process_alpn_protocols(const std::string_view protocols, unsigned char *client_alpn_protocols, int &alpn_array_len) +{ + // Count up the number of separators + int start = 0; + size_t offset = protocols.find(',', start); + int index = 0; + bool retval = true; + while (offset != protocols.npos) { + if ((index + 1 + static_cast(offset) - start) > alpn_array_len) { + retval = false; + break; + } + client_alpn_protocols[index++] = offset - start; + memcpy(client_alpn_protocols + index, protocols.data() + start, offset - start); + index += offset - start; + start = offset + 1; + offset = protocols.find(',', start); + } + // Copy in the last string + offset = protocols.length(); + if ((index + 1 + static_cast(offset) - start) > alpn_array_len) { + retval = false; + alpn_array_len = 0; + } else { + client_alpn_protocols[index++] = offset - start; + memcpy(client_alpn_protocols + index, protocols.data() + start, offset - start); + index += offset - start; + alpn_array_len = index; + } + return retval; +} diff --git a/iocore/net/I_NetVConnection.h b/iocore/net/I_NetVConnection.h index 6ad2a0189dc..57fffb2fd50 100644 --- a/iocore/net/I_NetVConnection.h +++ b/iocore/net/I_NetVConnection.h @@ -225,6 +225,9 @@ struct NetVCOptions { bool tls_upstream = false; + unsigned char alpn_protocols_array[MAX_ALPN_STRING]; + int alpn_protocols_array_size = 0; + /// Reset all values to defaults. /** diff --git a/iocore/net/P_ALPNSupport.h b/iocore/net/P_ALPNSupport.h index fe398001eaf..c97aa226d79 100644 --- a/iocore/net/P_ALPNSupport.h +++ b/iocore/net/P_ALPNSupport.h @@ -45,6 +45,7 @@ class ALPNSupport void enableProtocol(int idx); void clear(); bool setSelectedProtocol(const unsigned char *proto, unsigned int len); + static bool process_alpn_protocols(const std::string_view protocols, unsigned char *alpn_array, int &alpn_array_len); int advertise_next_protocol(SSL *ssl, const unsigned char **out, unsigned *outlen); int select_next_protocol(SSL *ssl, const unsigned char **out, unsigned char *outlen, const unsigned char *in, unsigned inlen); diff --git a/iocore/net/P_SSLConfig.h b/iocore/net/P_SSLConfig.h index 262df7c9c58..56a7da2784d 100644 --- a/iocore/net/P_SSLConfig.h +++ b/iocore/net/P_SSLConfig.h @@ -101,6 +101,9 @@ struct SSLConfigParams : public ConfigInfo { long ssl_ctx_options; long ssl_client_ctx_options; + unsigned char alpn_protocols_array[MAX_ALPN_STRING]; + int alpn_protocols_array_size = 0; + char *server_tls13_cipher_suites; char *client_tls13_cipher_suites; char *server_groups_list; diff --git a/iocore/net/SSLConfig.cc b/iocore/net/SSLConfig.cc index cecbb4592ed..a9e454bd8a3 100644 --- a/iocore/net/SSLConfig.cc +++ b/iocore/net/SSLConfig.cc @@ -255,6 +255,16 @@ SSLConfigParams::initialize() } #endif + // Read in the protocol string for ALPN to origin + char *clientALPNProtocols; + REC_ReadConfigStringAlloc(clientALPNProtocols, "proxy.config.ssl.client.alpn_protocols"); + + // Assume the protocols are comma delimited + if (clientALPNProtocols) { + this->alpn_protocols_array_size = MAX_ALPN_STRING; + ALPNSupport::process_alpn_protocols(clientALPNProtocols, this->alpn_protocols_array, this->alpn_protocols_array_size); + } + #ifdef SSL_OP_CIPHER_SERVER_PREFERENCE REC_ReadConfigInteger(option, "proxy.config.ssl.server.honor_cipher_order"); if (option) { diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc index da1b473f1b7..c40ce39ae8b 100644 --- a/iocore/net/SSLNetVConnection.cc +++ b/iocore/net/SSLNetVConnection.cc @@ -1142,6 +1142,16 @@ SSLNetVConnection::sslStartHandShake(int event, int &err) return EVENT_ERROR; } + // If it is negative, we are conscious not setting alpn (e.g. for private server sessions) + if (options.alpn_protocols_array_size >= 0) { + if (options.alpn_protocols_array_size > 0) { + SSL_set_alpn_protos(this->ssl, options.alpn_protocols_array, options.alpn_protocols_array_size); + } else if (params->alpn_protocols_array_size > 0) { + // Set the ALPN protocols we are requesting. + SSL_set_alpn_protos(this->ssl, params->alpn_protocols_array, params->alpn_protocols_array_size); + } + } + SSL_set_verify(this->ssl, SSL_VERIFY_PEER, verify_callback); // SNI @@ -1502,6 +1512,16 @@ SSLNetVConnection::sslClientHandShakeEvent(int &err) X509_free(cert); } } + { + const unsigned char *proto; + unsigned int len = 0; + // Make note of the negotiated protocol + SSL_get0_alpn_selected(ssl, &proto, &len); + if (len == 0) { + SSL_get0_next_proto_negotiated(ssl, &proto, &len); + } + this->set_negotiated_protocol_id({reinterpret_cast(proto), static_cast(len)}); + } // if the handshake is complete and write is enabled reschedule the write if (closed == 0 && write.enabled) { diff --git a/iocore/net/UnixNetVConnection.cc b/iocore/net/UnixNetVConnection.cc index 0bb3ecf5816..d193d92ecea 100644 --- a/iocore/net/UnixNetVConnection.cc +++ b/iocore/net/UnixNetVConnection.cc @@ -361,6 +361,7 @@ write_to_net_io(NetHandler *nh, UnixNetVConnection *vc, EThread *thread) { NetState *s = &vc->write; ProxyMutex *mutex = thread->mutex.get(); + Continuation *c = vc->write.vio.cont; MUTEX_TRY_LOCK(lock, s->vio.mutex, thread); @@ -445,6 +446,9 @@ write_to_net_io(NetHandler *nh, UnixNetVConnection *vc, EThread *thread) if (towrite != ntodo && buf.writer()->write_avail()) { if (write_signal_and_update(VC_EVENT_WRITE_READY, vc) != EVENT_CONT) { return; + } else if (c != s->vio.cont) { /* The write vio was updated in the handler */ + write_reschedule(nh, vc); + return; } ntodo = s->vio.ntodo(); diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index 60233fedc82..382b55a2321 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -1128,6 +1128,8 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.ssl.CA.cert.filename", RECD_STRING, nullptr, RECU_RESTART_TS, RR_NULL, RECC_STR, "^[^[:space:]]*$", RECA_NULL} , + {RECT_CONFIG, "proxy.config.ssl.client.alpn_protocols", RECD_STRING, nullptr, RECU_RESTART_TS, RR_NULL, RECC_STR, "^[^[:space:]]*$", RECA_NULL} + , {RECT_CONFIG, "proxy.config.ssl.CA.cert.path", RECD_STRING, TS_BUILD_SYSCONFDIR, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , {RECT_CONFIG, "proxy.config.ssl.client.verify.server.policy", RECD_STRING, "ENFORCED", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} @@ -1318,6 +1320,8 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.http2.initial_window_size_in", RECD_INT, "65535", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , + {RECT_CONFIG, "proxy.config.http2.session_initial_window_size_in", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} + , {RECT_CONFIG, "proxy.config.http2.max_frame_size", RECD_INT, "16384", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , {RECT_CONFIG, "proxy.config.http2.header_table_size", RECD_INT, "4096", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} @@ -1328,6 +1332,8 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.http2.no_activity_timeout_in", RECD_INT, "120", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , + {RECT_CONFIG, "proxy.config.http2.no_activity_timeout_out", RECD_INT, "120", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} + , {RECT_CONFIG, "proxy.config.http2.active_timeout_in", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} , {RECT_CONFIG, "proxy.config.http2.push_diary_size", RECD_INT, "256", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} diff --git a/plugins/lua/ts_lua_http_config.c b/plugins/lua/ts_lua_http_config.c index c17eb8d321e..feeb958ac1d 100644 --- a/plugins/lua/ts_lua_http_config.c +++ b/plugins/lua/ts_lua_http_config.c @@ -138,6 +138,7 @@ typedef enum { TS_LUA_CONFIG_HTTP_HOST_RESOLUTION_PREFERENCE = TS_CONFIG_HTTP_HOST_RESOLUTION_PREFERENCE, TS_LUA_CONFIG_PLUGIN_VC_DEFAULT_BUFFER_INDEX = TS_CONFIG_PLUGIN_VC_DEFAULT_BUFFER_INDEX, TS_LUA_CONFIG_PLUGIN_VC_DEFAULT_BUFFER_WATER_MARK = TS_CONFIG_PLUGIN_VC_DEFAULT_BUFFER_WATER_MARK, + TS_LUA_CONFIG_SSL_CLIENT_ALPN_PROTOCOLS = TS_CONFIG_SSL_CLIENT_ALPN_PROTOCOLS, TS_LUA_CONFIG_LAST_ENTRY = TS_CONFIG_LAST_ENTRY, } TSLuaOverridableConfigKey; @@ -263,6 +264,7 @@ ts_lua_var_item ts_lua_http_config_vars[] = { TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME), TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_CA_CERT_FILENAME), TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_HTTP_HOST_RESOLUTION_PREFERENCE), + TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_ALPN_PROTOCOLS), TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_HTTP_SERVER_MIN_KEEP_ALIVE_CONNS), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_PER_SERVER_CONNECTION_MAX), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_PER_SERVER_CONNECTION_MATCH), diff --git a/proxy/PoolableSession.h b/proxy/PoolableSession.h index d98f93c2540..114479fb373 100644 --- a/proxy/PoolableSession.h +++ b/proxy/PoolableSession.h @@ -85,6 +85,7 @@ class PoolableSession : public ProxySession bool is_private() const; virtual void set_netvc(NetVConnection *newvc); + virtual bool is_multiplexing() const; // Used to determine whether the session is for parent proxy // it is session to origin server @@ -237,3 +238,9 @@ PoolableSession::attach_hostname(const char *hostname) CryptoContext().hash_immediate(hostname_hash, (unsigned char *)hostname, strlen(hostname)); } } + +inline bool +PoolableSession::is_multiplexing() const +{ + return false; +} diff --git a/proxy/ProxySession.cc b/proxy/ProxySession.cc index 332b468ce9c..d11dae04d66 100644 --- a/proxy/ProxySession.cc +++ b/proxy/ProxySession.cc @@ -26,6 +26,8 @@ #include "ProxySession.h" #include "P_SSLNetVConnection.h" +std::map> ProtocolSessionCreateMap; + ProxySession::ProxySession() : VConnection(nullptr) {} ProxySession::ProxySession(NetVConnection *vc) : VConnection(nullptr), _vc(vc) {} @@ -312,3 +314,21 @@ ProxySession::support_sni() const { return _vc ? _vc->support_sni() : false; } + +PoolableSession * +ProxySession::protocol_creation(NetVConnection *netvc) +{ + // Figure out what protocol was negotiated + int proto_index = SessionProtocolNameRegistry::INVALID; + SSLNetVConnection *sslnetvc = dynamic_cast(netvc); + if (sslnetvc) { + proto_index = sslnetvc->get_negotiated_protocol_id(); + } + // No ALPN occurred. Assume it was HTTP/1.x and hope for the best + if (proto_index == SessionProtocolNameRegistry::INVALID) { + proto_index = TS_ALPN_PROTOCOL_INDEX_HTTP_1_1; + } + auto iter = ProtocolSessionCreateMap.find(proto_index); + ink_release_assert(iter != ProtocolSessionCreateMap.end()); + return iter->second(); +} diff --git a/proxy/ProxySession.h b/proxy/ProxySession.h index 4162813db4d..74060050a7d 100644 --- a/proxy/ProxySession.h +++ b/proxy/ProxySession.h @@ -162,6 +162,8 @@ class ProxySession : public VConnection, public PluginUserArgs return nullptr; } + static PoolableSession *protocol_creation(NetVConnection *netvc); + //////////////////// // Members diff --git a/proxy/ProxyTransaction.cc b/proxy/ProxyTransaction.cc index cb80b3cc2cc..a4d5cb6d83a 100644 --- a/proxy/ProxyTransaction.cc +++ b/proxy/ProxyTransaction.cc @@ -235,6 +235,34 @@ ProxyTransaction::get_version(HTTPHdr &hdr) const return hdr.version_get(); } +bool +ProxyTransaction::is_read_closed() const +{ + return false; +} + +bool +ProxyTransaction::expect_send_trailer() const +{ + return false; +} + +void +ProxyTransaction::set_expect_send_trailer() +{ +} + +bool +ProxyTransaction::expect_receive_trailer() const +{ + return false; +} + +void +ProxyTransaction::set_expect_receive_trailer() +{ +} + bool ProxyTransaction::allow_half_open() const { diff --git a/proxy/ProxyTransaction.h b/proxy/ProxyTransaction.h index 261af6829bd..cf6c3a2fa7c 100644 --- a/proxy/ProxyTransaction.h +++ b/proxy/ProxyTransaction.h @@ -49,6 +49,11 @@ class ProxyTransaction : public VConnection virtual void set_inactivity_timeout(ink_hrtime timeout_in); virtual void cancel_inactivity_timeout(); virtual void cancel_active_timeout(); + virtual bool is_read_closed() const; + virtual bool expect_send_trailer() const; + virtual void set_expect_send_trailer(); + virtual bool expect_receive_trailer() const; + virtual void set_expect_receive_trailer(); // Implement VConnection interface. VIO *do_io_read(Continuation *c, int64_t nbytes = INT64_MAX, MIOBuffer *buf = nullptr) override; diff --git a/proxy/hdrs/HdrToken.cc b/proxy/hdrs/HdrToken.cc index a4e161c4527..8702f863f1a 100644 --- a/proxy/hdrs/HdrToken.cc +++ b/proxy/hdrs/HdrToken.cc @@ -227,7 +227,10 @@ static HdrTokenFieldInfo _hdrtoken_strs_field_initializers[] = { {"Strict-Transport-Security", MIME_SLOTID_NONE, MIME_PRESENCE_NONE, (HTIF_MULTVALS)}, {"Subject", MIME_SLOTID_NONE, MIME_PRESENCE_SUBJECT, HTIF_NONE}, {"Summary", MIME_SLOTID_NONE, MIME_PRESENCE_SUMMARY, HTIF_NONE}, - {"TE", MIME_SLOTID_TE, MIME_PRESENCE_TE, (HTIF_COMMAS | HTIF_MULTVALS | HTIF_HOPBYHOP)}, + // Need to figure out why this cannot be handled as hop by hop. If it is hop-by-hop + // the information is not propagated for gRPC + //{"TE", MIME_SLOTID_TE, MIME_PRESENCE_TE, (HTIF_COMMAS | HTIF_MULTVALS | HTIF_HOPBYHOP)}, + {"TE", MIME_SLOTID_TE, MIME_PRESENCE_TE, (HTIF_COMMAS | HTIF_MULTVALS)}, {"Transfer-Encoding", MIME_SLOTID_TRANSFER_ENCODING, MIME_PRESENCE_TRANSFER_ENCODING, (HTIF_COMMAS | HTIF_MULTVALS | HTIF_HOPBYHOP)}, {"Upgrade", MIME_SLOTID_NONE, MIME_PRESENCE_UPGRADE, (HTIF_COMMAS | HTIF_MULTVALS | HTIF_HOPBYHOP)}, diff --git a/proxy/http/Http1ServerSession.cc b/proxy/http/Http1ServerSession.cc index e0a9c82926f..5d64a372a4d 100644 --- a/proxy/http/Http1ServerSession.cc +++ b/proxy/http/Http1ServerSession.cc @@ -256,3 +256,7 @@ Http1ServerSession::new_transaction() trans.set_reader(this->get_remote_reader()); return &trans; } + +std::function Create_h1_server_session = []() -> PoolableSession * { + return httpServerSessionAllocator.alloc(); +}; diff --git a/proxy/http/HttpConfig.h b/proxy/http/HttpConfig.h index 20b907907cd..b5a4bf6f85f 100644 --- a/proxy/http/HttpConfig.h +++ b/proxy/http/HttpConfig.h @@ -720,6 +720,7 @@ struct OverridableHttpConfigParams { char *ssl_client_cert_filename = nullptr; char *ssl_client_private_key_filename = nullptr; char *ssl_client_ca_cert_filename = nullptr; + char *ssl_client_alpn_protocols = nullptr; // Host Resolution order HostResData host_res_data; diff --git a/proxy/http/HttpProxyServerMain.cc b/proxy/http/HttpProxyServerMain.cc index 1a38a8bd259..597c8318749 100644 --- a/proxy/http/HttpProxyServerMain.cc +++ b/proxy/http/HttpProxyServerMain.cc @@ -49,6 +49,9 @@ HttpSessionAccept *plugin_http_accept = nullptr; HttpSessionAccept *plugin_http_transparent_accept = nullptr; +extern std::function Create_h1_server_session; +extern std::function Create_h2_server_session; +extern std::map> ProtocolSessionCreateMap; static SLL ssl_plugin_acceptors; static Ptr ssl_plugin_mutex; @@ -216,6 +219,9 @@ MakeHttpProxyAcceptor(HttpProxyAcceptor &acceptor, HttpProxyPort &port, unsigned if (port.m_session_protocol_preference.intersects(HTTP2_PROTOCOL_SET)) { probe->registerEndpoint(ProtocolProbeSessionAccept::PROTO_HTTP2, new Http2SessionAccept(accept_opt)); } + ProtocolSessionCreateMap.insert({TS_ALPN_PROTOCOL_INDEX_HTTP_1_0, Create_h1_server_session}); + ProtocolSessionCreateMap.insert({TS_ALPN_PROTOCOL_INDEX_HTTP_1_1, Create_h1_server_session}); + ProtocolSessionCreateMap.insert({TS_ALPN_PROTOCOL_INDEX_HTTP_2_0, Create_h2_server_session}); if (port.isSSL()) { SSLNextProtocolAccept *ssl = new SSLNextProtocolAccept(probe, port.m_transparent_passthrough); diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 81ed57ec6bc..f88745f7e28 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -28,6 +28,7 @@ #include "HttpTransactHeaders.h" #include "ProxyConfig.h" #include "Http1ServerSession.h" +#include "Http2ServerSession.h" #include "HttpDebugNames.h" #include "HttpSessionManager.h" #include "P_Cache.h" @@ -226,7 +227,6 @@ HttpVCTable::find_entry(VIO *vio) void HttpVCTable::remove_entry(HttpVCTableEntry *e) { - ink_assert(e->vc == nullptr || e->in_tunnel); e->vc = nullptr; e->eos = false; if (e->read_buffer) { @@ -241,11 +241,12 @@ HttpVCTable::remove_entry(HttpVCTableEntry *e) // for remaining I/O operations because the netvc // may have been deleted at this point and the pointer // could be stale. - e->read_vio = nullptr; - e->write_vio = nullptr; - e->vc_handler = nullptr; - e->vc_type = HTTP_UNKNOWN; - e->in_tunnel = false; + e->read_vio = nullptr; + e->write_vio = nullptr; + e->vc_read_handler = nullptr; + e->vc_write_handler = nullptr; + e->vc_type = HTTP_UNKNOWN; + e->in_tunnel = false; } // bool HttpVCTable::cleanup_entry(HttpVCEntry* e) @@ -258,18 +259,6 @@ HttpVCTable::cleanup_entry(HttpVCTableEntry *e) { ink_assert(e->vc); if (e->in_tunnel == false) { - // Update stats - switch (e->vc_type) { - case HTTP_UA_VC: - // proxy.process.http.current_client_transactions is decremented in HttpSM::destroy - break; - default: - // This covers: - // HTTP_UNKNOWN, HTTP_SERVER_VC, HTTP_TRANSFORM_VC, HTTP_CACHE_READ_VC, - // HTTP_CACHE_WRITE_VC, HTTP_RAW_SERVER_VC - break; - } - if (e->vc_type == HTTP_SERVER_VC) { HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_cleanup_entry); } @@ -289,6 +278,160 @@ HttpVCTable::cleanup_all() } } +class ConnectingEntry : public Continuation +{ +public: + std::set _connect_sms; + NetVConnection *_netvc = nullptr; + IOBufferReader *_netvc_reader = nullptr; + MIOBuffer *_netvc_read_buffer = nullptr; + std::string sni; + std::string cert_name; + IpEndpoint _ipaddr; + std::string hostname; + Action *_pending_action = nullptr; + NetVCOptions opt; + + void remove_entry(); + int state_http_server_open(int event, void *data); + static PoolableSession *create_server_session(HttpSM *root_sm, NetVConnection *netvc, MIOBuffer *netvc_read_buffer, + IOBufferReader *netvc_reader); +}; + +struct IpHelper { + size_t + operator()(IpEndpoint const &arg) const + { + return IpAddr{&arg.sa}.hash(); + } + bool + operator()(IpEndpoint const &arg1, IpEndpoint const &arg2) const + { + return ats_ip_addr_port_eq(&arg1.sa, &arg2.sa); + } +}; +using ConnectingIpPool = std::unordered_multimap; + +class ConnectingPool +{ +public: + ConnectingPool() {} + ConnectingIpPool m_ip_pool; +}; + +void +initialize_thread_for_connecting_pools(EThread *thread) +{ + if (thread->connecting_pool == nullptr) { + thread->connecting_pool = new ConnectingPool(); + } +} + +int +ConnectingEntry::state_http_server_open(int event, void *data) +{ + Debug("http_connect", "entered inside ConnectingEntry::state_http_server_open"); + + switch (event) { + case NET_EVENT_OPEN: { + _netvc = static_cast(data); + UnixNetVConnection *vc = static_cast(_netvc); + ink_release_assert(_pending_action == nullptr || _pending_action->continuation == vc->get_action()->continuation); + _pending_action = nullptr; + Debug("http_connect", "ConnectingEntrysetting handler for TCP handshake"); + // Just want to get a write-ready event so we know that the TCP handshake is complete. + // The buffer we create will be handed over to the eventually created server session + _netvc_read_buffer = new_MIOBuffer(HTTP_SERVER_RESP_HDR_BUFFER_INDEX); + _netvc_reader = _netvc_read_buffer->alloc_reader(); + _netvc->do_io_write(this, 1, _netvc_reader); + ink_release_assert(!_connect_sms.empty()); + if (!_connect_sms.empty()) { + HttpSM *prime_connect_sm = *(_connect_sms.begin()); + _netvc->set_inactivity_timeout(prime_connect_sm->get_server_connect_timeout()); + } + ink_release_assert(_pending_action == nullptr); + return 0; + } + case VC_EVENT_READ_COMPLETE: + case VC_EVENT_WRITE_READY: + case VC_EVENT_WRITE_COMPLETE: { + Debug("http_connect", "Kick off %" PRId64 " state machines waiting for origin", _connect_sms.size()); + this->remove_entry(); + _netvc->do_io_write(nullptr, 0, nullptr); + if (!_connect_sms.empty()) { + auto prime_iter = _connect_sms.rbegin(); + ink_release_assert(prime_iter != _connect_sms.rend()); + PoolableSession *new_session = (*prime_iter)->create_server_session(_netvc, _netvc_read_buffer, _netvc_reader); + _netvc = nullptr; + + // Did we end up with a multiplexing session? + int count = 0; + if (new_session->is_multiplexing()) { + // Hand off to all queued up ConnectSM's. + while (!_connect_sms.empty()) { + Debug("http_connect", "ConnectingEntry Pass along CONNECT_EVENT_TXN %d", count++); + auto entry = _connect_sms.begin(); + + SCOPED_MUTEX_LOCK(lock, (*entry)->mutex, this_ethread()); + (*entry)->handleEvent(CONNECT_EVENT_TXN, new_session); + _connect_sms.erase(entry); + } + } else { + // Hand off to one and tell all of the others to connect directly + Debug("http_connect", "ConnectingEntry send CONNECT_EVENT_TXN to first %d", count++); + { + SCOPED_MUTEX_LOCK(lock, (*prime_iter)->mutex, this_ethread()); + (*prime_iter)->handleEvent(CONNECT_EVENT_TXN, new_session); + _connect_sms.erase((++prime_iter).base()); + } + while (!_connect_sms.empty()) { + auto entry = _connect_sms.begin(); + Debug("http_connect", "ConnectingEntry Pass along CONNECT_EVENT_DIRECT %d", count++); + SCOPED_MUTEX_LOCK(lock, (*entry)->mutex, this_ethread()); + (*entry)->handleEvent(CONNECT_EVENT_DIRECT, nullptr); + _connect_sms.erase(entry); + } + } + } else { + ink_release_assert(!"There should be some sms on the connect_entry"); + } + delete this; + + // ConnectingEntry should remove itself from the tables and delete itself + return 0; + } + case VC_EVENT_INACTIVITY_TIMEOUT: + case VC_EVENT_ACTIVE_TIMEOUT: + case VC_EVENT_ERROR: + case NET_EVENT_OPEN_FAILED: { + ink_release_assert(_netvc != nullptr); + Debug("http_connect", "Stop %" PRId64 " state machines waiting for failed origin", _connect_sms.size()); + this->remove_entry(); + int vc_provided_cert = _netvc->provided_cert(); + int lerrno = _netvc->lerrno; + _netvc->do_io_close(); + while (!_connect_sms.empty()) { + auto entry = _connect_sms.begin(); + SCOPED_MUTEX_LOCK(lock, (*entry)->mutex, this_ethread()); + (*entry)->t_state.set_connect_fail(lerrno); + (*entry)->server_connection_provided_cert = vc_provided_cert; + (*entry)->handleEvent(event, data); + _connect_sms.erase(entry); + } + // ConnectingEntry should remove itself from the tables and delete itself + delete this; + + return 0; + } + default: + Error("[ConnectingEntry::state_http_server_open] Unknown event: %d", event); + ink_release_assert(0); + return 0; + } + + return 0; +} + #define SMDebug(tag, ...) SpecificDebug(debug_on, tag, __VA_ARGS__) #define REMEMBER(e, r) \ @@ -387,6 +530,8 @@ HttpSM::init(bool from_early_data) magic = HTTP_SM_MAGIC_ALIVE; + server_txn = nullptr; + // Unique state machine identifier sm_id = next_sm_id++; t_state.state_machine_id = sm_id; @@ -603,7 +748,7 @@ HttpSM::attach_client_session(ProxyTransaction *client_vc) hooks_set = client_vc->has_hooks(); // Setup for parsing the header - ua_entry->vc_handler = &HttpSM::state_read_client_request_header; + ua_entry->vc_read_handler = &HttpSM::state_read_client_request_header; t_state.hdr_info.client_request.destroy(); t_state.hdr_info.client_request.create(HTTP_TYPE_REQUEST); @@ -617,8 +762,7 @@ HttpSM::attach_client_session(ProxyTransaction *client_vc) // this hook maybe asynchronous, we need to disable IO on // client but set the continuation to be the state machine // so if we get an timeout events the sm handles them - // hold onto enabling read until setup_client_read_request_header - ua_entry->read_vio = client_vc->do_io_read(this, 0, nullptr); + ua_entry->read_vio = client_vc->do_io_read(this, 0, ua_txn->get_remote_reader()->mbuf); ua_entry->write_vio = client_vc->do_io_write(this, 0, nullptr); ///////////////////////// @@ -645,7 +789,7 @@ HttpSM::attach_client_session(ProxyTransaction *client_vc) void HttpSM::setup_client_read_request_header() { - ink_assert(ua_entry->vc_handler == &HttpSM::state_read_client_request_header); + ink_assert(ua_entry->vc_read_handler == &HttpSM::state_read_client_request_header); ua_entry->read_vio = ua_txn->do_io_read(this, INT64_MAX, ua_txn->get_remote_reader()->mbuf); // The header may already be in the buffer if this @@ -805,7 +949,9 @@ HttpSM::state_read_client_request_header(int event, void *data) ua_raw_buffer_reader = nullptr; } http_parser_clear(&http_parser); - ua_entry->vc_handler = &HttpSM::state_watch_for_client_abort; + ua_entry->vc_read_handler = &HttpSM::state_watch_for_client_abort; + ua_entry->vc_write_handler = &HttpSM::state_watch_for_client_abort; + ua_txn->cancel_inactivity_timeout(); milestones[TS_MILESTONE_UA_READ_HEADER_DONE] = Thread::get_hrtime(); } @@ -992,20 +1138,25 @@ HttpSM::state_watch_for_client_abort(int event, void *data) * client. */ case VC_EVENT_EOS: { - // We got an early EOS. If the tunnal has cache writer, don't kill it for background fill. - NetVConnection *netvc = ua_txn->get_netvc(); - if (ua_txn->allow_half_open() || tunnel.has_consumer_besides_client()) { - if (netvc) { - netvc->do_io_shutdown(IO_SHUTDOWN_READ); + // We got an early EOS. + if (!terminate_sm) { // Not done already + NetVConnection *netvc = ua_txn->get_netvc(); + if (ua_txn->allow_half_open() || tunnel.has_consumer_besides_client()) { + if (netvc) { + netvc->do_io_shutdown(IO_SHUTDOWN_READ); + } + } else { + ua_txn->do_io_close(); + vc_table.cleanup_entry(ua_entry); + ink_release_assert(vc_table.find_entry(ua_txn) == nullptr); + ua_entry = nullptr; + tunnel.kill_tunnel(); + terminate_sm = true; // Just die already, the requester is gone + set_ua_abort(HttpTransact::ABORTED, event); + } + if (ua_entry) { + ua_entry->eos = true; } - ua_entry->eos = true; - } else { - ua_txn->do_io_close(); - vc_table.cleanup_entry(ua_entry); - ua_entry = nullptr; - tunnel.kill_tunnel(); - terminate_sm = true; // Just die already, the requester is gone - set_ua_abort(HttpTransact::ABORTED, event); } break; } @@ -1076,7 +1227,7 @@ HttpSM::setup_push_read_response_header() ink_assert(t_state.method == HTTP_WKSIDX_PUSH); // Set the handler to read the pushed response hdr - ua_entry->vc_handler = &HttpSM::state_read_push_response_header; + ua_entry->vc_read_handler = &HttpSM::state_read_push_response_header; // We record both the total payload size as // client_request_body_bytes and the bytes for the individual @@ -1220,14 +1371,6 @@ HttpSM::state_raw_http_server_open(int event, void *data) pending_action = nullptr; switch (event) { - case EVENT_INTERVAL: - // If we get EVENT_INTERNAL it means that we moved the transaction - // to a different thread in do_http_server_open. Since we didn't - // do any of the actual work in do_http_server_open, we have to - // go back and do it now. - do_http_server_open(true); - return 0; - case NET_EVENT_OPEN: // Record the VC in our table @@ -1544,7 +1687,7 @@ plugins required to work with sni_routing. api_timer = -Thread::get_hrtime_updated(); HTTP_SM_SET_DEFAULT_HANDLER(&HttpSM::state_api_callout); ink_release_assert(pending_action.is_empty()); - pending_action = mutex->thread_holding->schedule_in(this, HRTIME_MSECONDS(10)); + pending_action = this_ethread()->schedule_in(this, HRTIME_MSECONDS(10)); return -1; } @@ -1620,6 +1763,10 @@ plugins required to work with sni_routing. } break; + // Eat the EOS while we are waiting for any locks to complete the transaction + case VC_EVENT_EOS: + return 0; + default: ink_assert(false); terminate_sm = true; @@ -1813,32 +1960,29 @@ HttpSM::handle_api_return() } PoolableSession * -HttpSM::create_server_session(NetVConnection *netvc) +HttpSM::create_server_session(NetVConnection *netvc, MIOBuffer *netvc_read_buffer, IOBufferReader *netvc_reader) { - HttpTransact::State &s = this->t_state; - PoolableSession *retval = httpServerSessionAllocator.alloc(); + PoolableSession *retval = ProxySession::protocol_creation(netvc); - retval->sharing_pool = static_cast(s.http_config_param->server_session_sharing_pool); - retval->sharing_match = static_cast(s.txn_conf->server_session_sharing_match); - MIOBuffer *netvc_read_buffer = new_MIOBuffer(HTTP_SERVER_RESP_HDR_BUFFER_INDEX); - IOBufferReader *netvc_reader = netvc_read_buffer->alloc_reader(); + retval->sharing_pool = static_cast(t_state.http_config_param->server_session_sharing_pool); + retval->sharing_match = static_cast(t_state.txn_conf->server_session_sharing_match); retval->new_connection(netvc, netvc_read_buffer, netvc_reader); + retval->attach_hostname(t_state.current.server->name); - retval->attach_hostname(s.current.server->name); - - ATS_PROBE1(new_origin_server_connection, s.current.server->name); + ATS_PROBE1(new_origin_server_connection, t_state.current.server->name); retval->set_active(); if (netvc) { - ats_ip_copy(&s.server_info.src_addr, netvc->get_local_addr()); + ats_ip_copy(&t_state.server_info.src_addr, netvc->get_local_addr()); } // If origin_max_connections or origin_min_keep_alive_connections is set then we are metering // the max and or min number of connections per host. Transfer responsibility for this to the // session object. - if (s.outbound_conn_track_state.is_active()) { - Debug("http_connect", "[%" PRId64 "] max number of outbound connections: %d", this->sm_id, s.txn_conf->outbound_conntrack.max); - retval->enable_outbound_connection_tracking(s.outbound_conn_track_state.drop()); + if (t_state.outbound_conn_track_state.is_active()) { + Debug("http_connect", "[%" PRId64 "] max number of outbound connections: %d", this->sm_id, + t_state.txn_conf->outbound_conntrack.max); + retval->enable_outbound_connection_tracking(t_state.outbound_conn_track_state.drop()); } return retval; } @@ -1846,14 +1990,26 @@ HttpSM::create_server_session(NetVConnection *netvc) bool HttpSM::create_server_txn(PoolableSession *new_session) { + ink_assert(new_session != nullptr); bool retval = false; - server_txn = new_session->new_transaction(); - if (server_txn != nullptr) { + + server_txn = new_session->new_transaction(); + if (server_txn) { + retval = true; server_txn->attach_transaction(this); + if (t_state.current.request_to == HttpTransact::PARENT_PROXY) { + new_session->to_parent_proxy = true; + HTTP_INCREMENT_DYN_STAT(http_current_parent_proxy_connections_stat); + HTTP_INCREMENT_DYN_STAT(http_total_parent_proxy_connections_stat); + } else { + new_session->to_parent_proxy = false; + } server_txn->do_io_write(this, 0, nullptr); attach_server_session(); - retval = true; } + _netvc = nullptr; + _netvc_read_buffer = nullptr; + _netvc_reader = nullptr; return retval; } @@ -1876,80 +2032,76 @@ HttpSM::state_http_server_open(int event, void *data) switch (event) { case NET_EVENT_OPEN: { - NetVConnection *netvc = static_cast(data); - UnixNetVConnection *vc = static_cast(data); - PoolableSession *new_session = this->create_server_session(netvc); - if (t_state.current.request_to == HttpTransact::PARENT_PROXY) { - new_session->to_parent_proxy = true; - HTTP_INCREMENT_DYN_STAT(http_current_parent_proxy_connections_stat); - HTTP_INCREMENT_DYN_STAT(http_total_parent_proxy_connections_stat); - } else { - new_session->to_parent_proxy = false; - } - this->create_server_txn(new_session); - // Since the UnixNetVConnection::action_ or SocksEntry::action_ may be returned from netProcessor.connect_re, and the - // SocksEntry::action_ will be copied into UnixNetVConnection::action_ before call back NET_EVENT_OPEN from SocksEntry::free(), - // so we just compare the Continuation between pending_action and VC's action_. + // SocksEntry::action_ will be copied into UnixNetVConnection::action_ before call back NET_EVENT_OPEN from + // SocksEntry::free(), so we just compare the Continuation between pending_action and VC's action_. + _netvc = static_cast(data); + _netvc_read_buffer = new_MIOBuffer(HTTP_SERVER_RESP_HDR_BUFFER_INDEX); + _netvc_reader = _netvc_read_buffer->alloc_reader(); + UnixNetVConnection *vc = static_cast(_netvc); ink_release_assert(pending_action.is_empty() || pending_action.get_continuation() == vc->get_action()->continuation); pending_action = nullptr; if (this->plugin_tunnel_type == HTTP_NO_PLUGIN_TUNNEL) { - SMDebug("http", "[%" PRId64 "] setting handler for TCP handshake", sm_id); + SMDebug("http_connect", "[%" PRId64 "] setting handler for TCP handshake timeout %" PRId64, this->sm_id, + this->get_server_connect_timeout()); // Just want to get a write-ready event so we know that the TCP handshake is complete. - server_entry->vc_handler = &HttpSM::state_http_server_open; - - int64_t nbytes = 1; - if (t_state.txn_conf->proxy_protocol_out >= 0) { - nbytes = do_outbound_proxy_protocol(server_txn->get_remote_reader()->mbuf, vc, ua_txn->get_netvc(), - t_state.txn_conf->proxy_protocol_out); - } - - server_entry->write_vio = server_txn->do_io_write(this, nbytes, server_txn->get_remote_reader()); + // The buffer we create will be handed over to the eventually created server session + _netvc->do_io_write(this, 1, _netvc_reader); + _netvc->set_inactivity_timeout(this->get_server_connect_timeout()); // Pre-emptively set a server connect failure that will be cleared once a WRITE_READY is received from origin or // bytes are received back t_state.set_connect_fail(EIO); } else { // in the case of an intercept plugin don't to the connect timeout change - SMDebug("http", "[%" PRId64 "] not setting handler for TCP handshake", sm_id); + Debug("http_connect", "[%" PRId64 "] not setting handler for TCP handshake", this->sm_id); + this->create_server_txn(this->create_server_session(_netvc, _netvc_read_buffer, _netvc_reader)); handle_http_server_open(); } - + ink_assert(pending_action.is_empty()); return 0; } + case CONNECT_EVENT_DIRECT: + // Try it again, but direct this time + do_http_server_open(false, true); + break; + case CONNECT_EVENT_TXN: + Debug("http", "[%" PRId64 "] TCP Handshake complete via CONNECT_EVENT_TXN", sm_id); + if (this->create_server_txn(static_cast(data))) { + write_outbound_proxy_protocol(); + handle_http_server_open(); + } else { // Failed to create transaction. Maybe too many active transactions already + // Try again (probably need a bounding counter here) + do_http_server_open(false); + } + return 0; case VC_EVENT_READ_COMPLETE: case VC_EVENT_WRITE_READY: case VC_EVENT_WRITE_COMPLETE: // Update the time out to the regular connection timeout. SMDebug("http_ss", "[%" PRId64 "] TCP Handshake complete", sm_id); - server_entry->vc_handler = &HttpSM::state_send_server_request_header; - - // Reset the timeout to the non-connect timeout - server_txn->set_inactivity_timeout(get_server_inactivity_timeout()); + this->create_server_txn(this->create_server_session(_netvc, _netvc_read_buffer, _netvc_reader)); + write_outbound_proxy_protocol(); t_state.current.server->clear_connect_fail(); handle_http_server_open(); return 0; - case EVENT_INTERVAL: // Delayed call from another thread - if (server_txn == nullptr) { - do_http_server_open(); - } - break; case VC_EVENT_INACTIVITY_TIMEOUT: case VC_EVENT_ACTIVE_TIMEOUT: t_state.set_connect_fail(ETIMEDOUT); /* fallthrough */ case VC_EVENT_ERROR: case NET_EVENT_OPEN_FAILED: { - if (server_txn) { - NetVConnection *vc = server_txn->get_netvc(); - if (vc) { - t_state.set_connect_fail(vc->lerrno); - server_connection_provided_cert = vc->provided_cert(); - } - } - t_state.current.state = HttpTransact::CONNECTION_ERROR; t_state.outbound_conn_track_state.clear(); + if (_netvc != nullptr) { + if (event == VC_EVENT_ERROR || event == NET_EVENT_OPEN_FAILED) { + t_state.set_connect_fail(_netvc->lerrno); + } + this->server_connection_provided_cert = _netvc->provided_cert(); + _netvc->do_io_write(nullptr, 0, nullptr); + _netvc->do_io_close(); + _netvc = nullptr; + } /* If we get this error in transparent mode, then we simply can't bind to the 4-tuple to make the connection. There's no hope of retries succeeding in the near future. The best option is to just shut down the connection without further comment. The @@ -1995,10 +2147,7 @@ HttpSM::state_read_server_response_header(int event, void *data) // a WRITE event appear after receiving EOS from the server connection if (server_entry->eos) { return 0; - } else if (data == server_entry->write_vio) { - return this->state_send_server_request_header(event, data); } - ink_assert(server_entry->eos != true); ink_assert(server_entry->read_vio == (VIO *)data); ink_assert(t_state.current.server->state == HttpTransact::STATE_UNDEFINED); @@ -2065,6 +2214,12 @@ HttpSM::state_read_server_response_header(int event, void *data) http_parser_clear(&http_parser); milestones[TS_MILESTONE_SERVER_READ_HEADER_DONE] = Thread::get_hrtime(); + // Any other events to the end + if (server_entry->vc_type == HTTP_SERVER_VC) { + server_entry->vc_read_handler = &HttpSM::tunnel_handler; + server_entry->vc_write_handler = &HttpSM::tunnel_handler; + } + // If there is a post body in transit, give up on it if (tunnel.is_tunnel_alive()) { tunnel.abort_tunnel(); @@ -2162,9 +2317,6 @@ int HttpSM::state_send_server_request_header(int event, void *data) { ink_assert(server_entry != nullptr); - if (server_entry->read_vio == data) { - return this->state_read_server_response_header(event, data); - } ink_assert(server_entry->eos == false); ink_assert(server_entry->write_vio == (VIO *)data); STATE_ENTER(&HttpSM::state_send_server_request_header, event); @@ -2193,6 +2345,10 @@ HttpSM::state_send_server_request_header(int event, void *data) do_setup_post_tunnel(HTTP_SERVER_VC); } } + // Any other events to these read response + if (server_entry->vc_type == HTTP_SERVER_VC) { + server_entry->vc_read_handler = &HttpSM::state_read_server_response_header; + } } break; @@ -2243,6 +2399,106 @@ HttpSM::state_send_server_request_header(int event, void *data) return 0; } +bool +HttpSM::origin_multiplexed() const +{ + return (t_state.host_db_info.app.http_data.http_version == HTTP_2_0 || + t_state.host_db_info.app.http_data.http_version == HTTP_INVALID); +} + +void +ConnectingEntry::remove_entry() +{ + EThread *ethread = this_ethread(); + auto ip_iter = ethread->connecting_pool->m_ip_pool.find(this->_ipaddr); + while (ip_iter != ethread->connecting_pool->m_ip_pool.end() && this->_ipaddr == ip_iter->first) { + if (ip_iter->second == this) { + ethread->connecting_pool->m_ip_pool.erase(ip_iter); + break; + } + ++ip_iter; + } +} + +void +HttpSM::cancel_pending_server_connection() +{ + EThread *ethread = this_ethread(); + if (nullptr == ethread->connecting_pool) { + return; // No pending requests + } + if (t_state.current.server) { + IpEndpoint ip; + ip.assign(&this->t_state.current.server->dst_addr.sa); + auto ip_iter = ethread->connecting_pool->m_ip_pool.find(ip); + while (ip_iter != ethread->connecting_pool->m_ip_pool.end() && ip_iter->first == ip) { + ConnectingEntry *connecting_entry = ip_iter->second; + // Found a match + // Look for our sm in the queue + auto entry = connecting_entry->_connect_sms.find(this); + if (entry != connecting_entry->_connect_sms.end()) { + connecting_entry->_connect_sms.erase(entry); + if (connecting_entry->_connect_sms.empty()) { + if (connecting_entry->_netvc) { + connecting_entry->_netvc->do_io_write(nullptr, 0, nullptr); + connecting_entry->_netvc->do_io_close(); + } + ethread->connecting_pool->m_ip_pool.erase(ip_iter); + delete connecting_entry; + break; + } else { + // Leave the shared entry remaining alone + } + } + ++ip_iter; + } + } +} + +// Returns true if there was a matching entry that we +// queued this request on +bool +HttpSM::add_to_existing_request() +{ + HttpTransact::State &s = this->t_state; + bool retval = false; + EThread *ethread = this_ethread(); + + if (this->plugin_tunnel_type != HTTP_NO_PLUGIN_TUNNEL) { + return false; + } + + if (nullptr == ethread->connecting_pool) { + initialize_thread_for_connecting_pools(ethread); + } + auto my_nh = ((UnixNetVConnection *)(this)->ua_txn->get_netvc())->nh; + ink_release_assert(my_nh == nullptr /* PluginVC */ || my_nh == get_NetHandler(this_ethread())); + + HTTP_SM_SET_DEFAULT_HANDLER(&HttpSM::state_http_server_open); + + IpEndpoint ip; + ip.assign(&s.current.server->dst_addr.sa); + auto ip_iter = ethread->connecting_pool->m_ip_pool.find(ip); + std::string_view proposed_sni = this->get_outbound_sni(); + std::string_view proposed_cert = this->get_outbound_cert(); + std::string_view proposed_hostname = this->t_state.current.server->name; + while (!retval && ip_iter != ethread->connecting_pool->m_ip_pool.end() && ip_iter->first == ip) { + // Check that entry matches sni, hostname, and cert + if (proposed_hostname == ip_iter->second->hostname && proposed_sni == ip_iter->second->sni && + proposed_cert == ip_iter->second->cert_name && ip_iter->second->_connect_sms.size() < 50) { + // Pre-emptively set a server connect failure that will be cleared once a WRITE_READY is received from origin or + // bytes are received back + this->t_state.set_connect_fail(EIO); + ip_iter->second->_connect_sms.insert(this); + Debug("http_connect", "Add entry to connection queue. size=%" PRId64, ip_iter->second->_connect_sms.size()); + retval = true; + break; + } + ++ip_iter; + } + return retval; +} + void HttpSM::process_srv_info(HostDBInfo *r) { @@ -2382,11 +2638,6 @@ int HttpSM::state_hostdb_lookup(int event, void *data) { STATE_ENTER(&HttpSM::state_hostdb_lookup, event); - // ink_assert (m_origin_server_vc == 0); - // REQ_FLAVOR_SCHEDULED_UPDATE can be transformed into - // REQ_FLAVOR_REVPROXY - ink_assert(t_state.req_flavor == HttpTransact::REQ_FLAVOR_SCHEDULED_UPDATE || - t_state.req_flavor == HttpTransact::REQ_FLAVOR_REVPROXY || ua_entry->vc != nullptr); switch (event) { case EVENT_HOST_DB_LOOKUP: @@ -2417,7 +2668,6 @@ HttpSM::state_hostdb_lookup(int event, void *data) default: ink_assert(!"Unexpected event"); } - return 0; } @@ -2744,7 +2994,7 @@ HttpSM::main_handler(int event, void *data) } if (vc_entry) { - jump_point = vc_entry->vc_handler; + jump_point = (static_cast(data) == vc_entry->read_vio) ? vc_entry->vc_read_handler : vc_entry->vc_write_handler; ink_assert(jump_point != (HttpSMHandler) nullptr); ink_assert(vc_entry->vc != (VConnection *)nullptr); (this->*jump_point)(event, data); @@ -2923,6 +3173,50 @@ HttpSM::tunnel_handler_post(int event, void *data) return 0; } +int +HttpSM::tunnel_handler_trailer(int event, void *data) +{ + STATE_ENTER(&HttpSM::tunnel_handler_trailer, event); + + switch (event) { + case HTTP_TUNNEL_EVENT_DONE: // Response tunnel done. + break; + + default: + // If the response tunnel did not succeed, just clean up as in the default case + return tunnel_handler(event, data); + } + + ink_assert(event == HTTP_TUNNEL_EVENT_DONE); + + // Set up a new tunnel to transport the trailing header to the UA + HTTP_SM_SET_DEFAULT_HANDLER(&HttpSM::tunnel_handler); + + MIOBuffer *trailer_buffer = new_MIOBuffer(HTTP_HEADER_BUFFER_SIZE_INDEX); + IOBufferReader *buf_start = trailer_buffer->alloc_reader(); + + size_t nbytes = INT64_MAX; + int start_bytes = trailer_buffer->write(server_txn->get_remote_reader(), server_txn->get_remote_reader()->read_avail()); + server_txn->get_remote_reader()->consume(start_bytes); + // The server has already sent all it has + if (server_txn->is_read_closed()) { + nbytes = start_bytes; + } + // Signal the ua_txn to get ready for a trailer + ua_txn->set_expect_send_trailer(); + tunnel.reset(); + HttpTunnelProducer *p = tunnel.add_producer(server_entry->vc, nbytes, buf_start, &HttpSM::tunnel_handler_trailer_server, + HT_HTTP_SERVER, "http server trailer"); + tunnel.add_consumer(ua_entry->vc, server_entry->vc, &HttpSM::tunnel_handler_trailer_ua, HT_HTTP_CLIENT, "user agent trailer"); + + ua_entry->in_tunnel = true; + server_entry->in_tunnel = true; + + tunnel.tunnel_run(p); + + return 0; +} + int HttpSM::tunnel_handler_cache_fill(int event, void *data) { @@ -2933,12 +3227,31 @@ HttpSM::tunnel_handler_cache_fill(int event, void *data) ink_release_assert(cache_sm.cache_write_vc); - tunnel.deallocate_buffers(); - this->postbuf_clear(); - tunnel.reset(); + int64_t alloc_index = find_server_buffer_size(); + MIOBuffer *buf = new_MIOBuffer(alloc_index); + IOBufferReader *buf_start = buf->alloc_reader(); - setup_server_transfer_to_cache_only(); - tunnel.tunnel_run(); + TunnelChunkingAction_t action = + (t_state.current.server && t_state.current.server->transfer_encoding == HttpTransact::CHUNKED_ENCODING) ? + TCA_DECHUNK_CONTENT : + TCA_PASSTHRU_DECHUNKED_CONTENT; + + int64_t nbytes = server_transfer_init(buf, 0); + + HTTP_SM_SET_DEFAULT_HANDLER(&HttpSM::tunnel_handler); + + server_entry->vc = server_txn; + HttpTunnelProducer *p = + tunnel.add_producer(server_entry->vc, nbytes, buf_start, &HttpSM::tunnel_handler_server, HT_HTTP_SERVER, "http server"); + + tunnel.set_producer_chunking_action(p, 0, action); + tunnel.set_producer_chunking_size(p, t_state.txn_conf->http_chunking_size); + + setup_cache_write_transfer(&cache_sm, server_entry->vc, &t_state.cache_info.object_store, 0, "cache write"); + + server_entry->in_tunnel = true; + // Kick off the new producer + tunnel.tunnel_run(p); return 0; } @@ -3193,6 +3506,13 @@ HttpSM::tunnel_handler_server(int event, HttpTunnelProducer *p) tunnel.local_finish_all(p); } } + if (server_txn->expect_receive_trailer()) { + SMDebug("http", "[%" PRId64 "] wait for that trailing header", sm_id); + // Swap out the default hander to set up the new tunnel for the trailer exchange. + HTTP_SM_SET_DEFAULT_HANDLER(&HttpSM::tunnel_handler_trailer); + tunnel.local_finish_all(p); + return 0; + } break; case HTTP_TUNNEL_EVENT_CONSUMER_DETACH: @@ -3268,6 +3588,84 @@ HttpSM::tunnel_handler_server(int event, HttpTunnelProducer *p) return 0; } +int +HttpSM::tunnel_handler_trailer_server(int event, HttpTunnelProducer *p) +{ + STATE_ENTER(&HttpSM::tunnel_handler_trailer_server, event); + + switch (event) { + case VC_EVENT_INACTIVITY_TIMEOUT: + case VC_EVENT_ACTIVE_TIMEOUT: + case VC_EVENT_ERROR: + t_state.squid_codes.log_code = SQUID_LOG_ERR_READ_TIMEOUT; + t_state.squid_codes.hier_code = SQUID_HIER_TIMEOUT_DIRECT; + /* fallthru */ + + case VC_EVENT_EOS: + + switch (event) { + case VC_EVENT_INACTIVITY_TIMEOUT: + t_state.current.server->state = HttpTransact::INACTIVE_TIMEOUT; + break; + case VC_EVENT_ACTIVE_TIMEOUT: + t_state.current.server->state = HttpTransact::ACTIVE_TIMEOUT; + break; + case VC_EVENT_ERROR: + t_state.current.server->state = HttpTransact::CONNECTION_ERROR; + break; + case VC_EVENT_EOS: + t_state.current.server->state = HttpTransact::TRANSACTION_COMPLETE; + break; + } + + ink_assert(p->vc_type == HT_HTTP_SERVER); + + SMDebug("http", "[%" PRId64 "] [HttpSM::tunnel_handler_server] aborting HTTP tunnel due to server truncation", sm_id); + tunnel.chain_abort_all(p); + + t_state.current.server->abort = HttpTransact::ABORTED; + t_state.client_info.keep_alive = HTTP_NO_KEEPALIVE; + t_state.current.server->keep_alive = HTTP_NO_KEEPALIVE; + t_state.squid_codes.log_code = SQUID_LOG_ERR_READ_ERROR; + break; + + case HTTP_TUNNEL_EVENT_PRECOMPLETE: + case VC_EVENT_READ_COMPLETE: + // + // The transfer completed successfully + p->read_success = true; + t_state.current.server->state = HttpTransact::TRANSACTION_COMPLETE; + t_state.current.server->abort = HttpTransact::DIDNOT_ABORT; + break; + + case HTTP_TUNNEL_EVENT_CONSUMER_DETACH: + case VC_EVENT_READ_READY: + case VC_EVENT_WRITE_READY: + case VC_EVENT_WRITE_COMPLETE: + default: + // None of these events should ever come our way + ink_assert(0); + break; + } + + // We handled the event. Now either shutdown server transaction + ink_assert(server_entry->vc == p->vc); + ink_assert(p->vc_type == HT_HTTP_SERVER); + ink_assert(p->vc == server_txn); + + // The server session has been released. Clean all pointer + // Calling remove_entry instead of server_entry because we don't + // want to close the server VC at this point + vc_table.remove_entry(server_entry); + + p->vc->do_io_close(); + p->read_vio = nullptr; + + server_entry = nullptr; + + return 0; +} + // int HttpSM::tunnel_handler_100_continue_ua(int event, HttpTunnelConsumer* c) // // Used for tunneling the 100 continue response. The tunnel @@ -3289,6 +3687,7 @@ HttpSM::tunnel_handler_100_continue_ua(int event, HttpTunnelConsumer *c) case VC_EVENT_ACTIVE_TIMEOUT: case VC_EVENT_ERROR: set_ua_abort(HttpTransact::ABORTED, event); + vc_table.remove_entry(ua_entry); c->vc->do_io_close(); break; case VC_EVENT_WRITE_COMPLETE: @@ -3387,13 +3786,13 @@ HttpSM::tunnel_handler_ua(int event, HttpTunnelConsumer *c) HTTP_INCREMENT_DYN_STAT(http_background_fill_current_count_stat); HTTP_INCREMENT_DYN_STAT(http_background_fill_total_count_stat); - ink_assert(server_entry->vc == server_txn); ink_assert(c->is_downstream_from(server_txn)); server_txn->set_active_timeout(HRTIME_SECONDS(t_state.txn_conf->background_fill_active_timeout)); } // Even with the background fill, the client side should go down c->write_vio = nullptr; + vc_table.remove_entry(ua_entry); c->vc->do_io_close(EHTTP_ERROR); c->alive = false; @@ -3462,8 +3861,9 @@ HttpSM::tunnel_handler_ua(int event, HttpTunnelConsumer *c) break; } - ink_assert(ua_entry->vc == c->vc); - if (close_connection) { + if (event == VC_EVENT_WRITE_COMPLETE && server_txn && server_txn->expect_receive_trailer()) { + // Don't shutdown if we are still expecting a trailer + } else if (close_connection) { // If the client could be pipelining or is doing a POST, we need to // set the ua_txn into half close mode @@ -3475,6 +3875,7 @@ HttpSM::tunnel_handler_ua(int event, HttpTunnelConsumer *c) } vc_table.remove_entry(this->ua_entry); + ink_release_assert(vc_table.find_entry(ua_txn) == nullptr); ua_txn->do_io_close(); } else { ink_assert(ua_txn->get_remote_reader() != nullptr); @@ -3485,6 +3886,66 @@ HttpSM::tunnel_handler_ua(int event, HttpTunnelConsumer *c) return 0; } +int +HttpSM::tunnel_handler_trailer_ua(int event, HttpTunnelConsumer *c) +{ + HttpTunnelProducer *p = nullptr; + HttpTunnelConsumer *selfc = nullptr; + + STATE_ENTER(&HttpSM::tunnel_handler_trailer_ua, event); + ink_assert(c->vc == ua_txn); + milestones[TS_MILESTONE_UA_CLOSE] = Thread::get_hrtime(); + + switch (event) { + case VC_EVENT_EOS: + ua_entry->eos = true; + + // FALL-THROUGH + case VC_EVENT_INACTIVITY_TIMEOUT: + case VC_EVENT_ACTIVE_TIMEOUT: + case VC_EVENT_ERROR: + + // The user agent died or aborted. Check to + // see if we should setup a background fill + set_ua_abort(HttpTransact::ABORTED, event); + + // Should not be processing trailer headers in the background fill case + ink_assert(!is_bg_fill_necessary(c)); + p = c->producer; + tunnel.chain_abort_all(c->producer); + selfc = p->self_consumer; + if (selfc) { + // This is the case where there is a transformation between ua and os + p = selfc->producer; + // if producer is the cache or OS, close the producer. + // Otherwise in case of large docs, producer iobuffer gets filled up, + // waiting for a consumer to consume data and the connection is never closed. + if (p->alive && ((p->vc_type == HT_CACHE_READ) || (p->vc_type == HT_HTTP_SERVER))) { + tunnel.chain_abort_all(p); + } + } + break; + + case VC_EVENT_WRITE_COMPLETE: + c->write_success = true; + t_state.client_info.abort = HttpTransact::DIDNOT_ABORT; + break; + case VC_EVENT_WRITE_READY: + case VC_EVENT_READ_READY: + case VC_EVENT_READ_COMPLETE: + default: + // None of these events should ever come our way + ink_assert(0); + break; + } + + ink_assert(ua_entry->vc == c->vc); + vc_table.remove_entry(this->ua_entry); + ua_txn->do_io_close(); + ink_release_assert(vc_table.find_entry(ua_txn) == nullptr); + return 0; +} + int HttpSM::tunnel_handler_ua_push(int event, HttpTunnelProducer *p) { @@ -3687,8 +4148,9 @@ HttpSM::tunnel_handler_post_ua(int event, HttpTunnelProducer *p) ua_txn->cancel_inactivity_timeout(); // Initiate another read to catch aborts - ua_entry->vc_handler = &HttpSM::state_watch_for_client_abort; - ua_entry->read_vio = p->vc->do_io_read(this, INT64_MAX, ua_txn->get_remote_reader()->mbuf); + ua_entry->vc_read_handler = &HttpSM::state_watch_for_client_abort; + ua_entry->vc_write_handler = &HttpSM::state_watch_for_client_abort; + ua_entry->read_vio = p->vc->do_io_read(this, INT64_MAX, ua_txn->get_remote_reader()->mbuf); } break; default: @@ -3737,6 +4199,18 @@ HttpSM::tunnel_handler_post_server(int event, HttpTunnelConsumer *c) server_request_body_bytes = c->bytes_written; } + // We may be reading from a transform. In that case, we + // want to close the transform + if (c->producer->vc_type == HT_TRANSFORM) { + if (c->producer->handler_state == HTTP_SM_TRANSFORM_OPEN) { + ink_assert(c->producer->vc == post_transform_info.vc); + c->producer->vc->do_io_close(); + c->producer->alive = false; + c->producer->self_consumer->alive = false; + ink_release_assert(vc_table.find_entry(c->producer->vc) == nullptr); + } + } + switch (event) { case VC_EVENT_EOS: case VC_EVENT_ERROR: @@ -3803,7 +4277,8 @@ HttpSM::tunnel_handler_post_server(int event, HttpTunnelConsumer *c) // Before shutting down, initiate another read // on the user agent in order to get timeouts // coming to the state machine and not the tunnel - ua_entry->vc_handler = &HttpSM::state_watch_for_client_abort; + ua_entry->vc_read_handler = &HttpSM::state_watch_for_client_abort; + ua_entry->vc_write_handler = &HttpSM::state_watch_for_client_abort; // YTS Team, yamsat Plugin // When event is VC_EVENT_ERROR,and when redirection is enabled @@ -4966,7 +5441,7 @@ HttpSM::get_outbound_sni() const // ////////////////////////////////////////////////////////////////////////// void -HttpSM::do_http_server_open(bool raw) +HttpSM::do_http_server_open(bool raw, bool only_direct) { int ip_family = t_state.current.server->dst_addr.sa.sa_family; auto fam_name = ats_ip_family_name(ip_family); @@ -5111,6 +5586,7 @@ HttpSM::do_http_server_open(bool raw) (t_state.txn_conf->keep_alive_post_out == 1 || t_state.hdr_info.request_content_length <= 0) && !is_private() && ua_txn != nullptr) { HSMresult_t shared_result; + SMDebug("http_ss", "Try to acquire_session for %s", t_state.current.server->name); shared_result = httpSessionManager.acquire_session(this, // state machine &t_state.current.server->dst_addr.sa, // ip + port t_state.current.server->name, // hostname @@ -5190,6 +5666,18 @@ HttpSM::do_http_server_open(bool raw) ink_release_assert(ua_txn == nullptr); } } + + bool multiplexed_origin = !only_direct && !raw && this->origin_multiplexed() && !is_private(); + if (multiplexed_origin) { + Debug("http_ss", "Check for existing connect request"); + if (this->add_to_existing_request()) { + Debug("http_ss", "Queue behind existing request"); + // We are queued up behind an existing connect request + // Go away and wait. + return; + } + } + // Check to see if we have reached the max number of connections. // Atomically read the current number of connections and check to see // if we have gone above the max allowed. @@ -5313,7 +5801,39 @@ HttpSM::do_http_server_open(bool raw) opt.set_ssl_client_cert_name(t_state.txn_conf->ssl_client_cert_filename); opt.ssl_client_private_key_name = t_state.txn_conf->ssl_client_private_key_filename; opt.ssl_client_ca_cert_name = t_state.txn_conf->ssl_client_ca_cert_filename; - + if (is_private()) { + // If the connection to origin is private, don't try the negotiate the higher overhead H2 + opt.alpn_protocols_array_size = -1; + Debug("connect", "Clear alpn for private session"); + } else if (t_state.txn_conf->ssl_client_alpn_protocols != nullptr) { + opt.alpn_protocols_array_size = MAX_ALPN_STRING; + Debug("connect", "Override alpn %s", t_state.txn_conf->ssl_client_alpn_protocols); + ALPNSupport::process_alpn_protocols(t_state.txn_conf->ssl_client_alpn_protocols, opt.alpn_protocols_array, + opt.alpn_protocols_array_size); + } + + ConnectingEntry *new_entry = nullptr; + if (multiplexed_origin) { + EThread *ethread = this_ethread(); + if (nullptr != ethread->connecting_pool) { + Debug("http_ss", "Queue multiplexed request"); + new_entry = new ConnectingEntry(); + new_entry->mutex = this->mutex; + new_entry->handler = (ContinuationHandler)&ConnectingEntry::state_http_server_open; + new_entry->_ipaddr.assign(&t_state.current.server->dst_addr.sa); + new_entry->hostname = t_state.current.server->name; + new_entry->sni = this->get_outbound_sni(); + new_entry->cert_name = this->get_outbound_cert(); + this->t_state.set_connect_fail(EIO); + new_entry->_connect_sms.insert(this); + ethread->connecting_pool->m_ip_pool.insert(std::make_pair(new_entry->_ipaddr, new_entry)); + } + } + + Continuation *cont = new_entry; + if (!cont) { + cont = this; + } if (tls_upstream) { SMDebug("http", "calling sslNetProcessor.connect_re"); @@ -5334,12 +5854,12 @@ HttpSM::do_http_server_open(bool raw) opt.set_ssl_servername(t_state.server_info.name); } - pending_action = sslNetProcessor.connect_re(this, // state machine + pending_action = sslNetProcessor.connect_re(cont, // state machine or ConnectingEntry &t_state.current.server->dst_addr.sa, // addr + port &opt); } else { SMDebug("http", "calling netProcessor.connect_re"); - pending_action = netProcessor.connect_re(this, // state machine + pending_action = netProcessor.connect_re(cont, // state machine or ConnectingEntry &t_state.current.server->dst_addr.sa, // addr + port &opt); } @@ -5471,7 +5991,6 @@ HttpSM::mark_host_failure(HostDBInfo *info, time_t time_down) .extend(1) .write('\0') .data()); - if (url_str) { t_state.arena.str_free(url_str); } @@ -5674,16 +6193,17 @@ HttpSM::handle_http_server_open() vc->apply_options(); } } - } - server_txn->set_inactivity_timeout(get_server_inactivity_timeout()); + server_txn->set_inactivity_timeout(get_server_inactivity_timeout()); - int method = t_state.hdr_info.server_request.method_get_wksidx(); - if (method != HTTP_WKSIDX_TRACE && - (t_state.hdr_info.request_content_length > 0 || t_state.client_info.transfer_encoding == HttpTransact::CHUNKED_ENCODING) && - do_post_transform_open()) { - do_setup_post_tunnel(HTTP_TRANSFORM_VC); // Seems like we should be sending the request along this way too - } else if (server_txn != nullptr) { - setup_server_send_request_api(); + int method = t_state.hdr_info.server_request.method_get_wksidx(); + if (method != HTTP_WKSIDX_TRACE && + server_txn->has_request_body(t_state.hdr_info.response_content_length, + t_state.server_info.transfer_encoding == HttpTransact::CHUNKED_ENCODING) && + do_post_transform_open()) { + do_setup_post_tunnel(HTTP_TRANSFORM_VC); + } else { + setup_server_send_request_api(); + } } } @@ -5727,8 +6247,9 @@ HttpSM::handle_server_setup_error(int event, void *data) HttpTunnelProducer *ua_producer = c->producer; ink_assert(ua_entry->vc == ua_producer->vc); - ua_entry->vc_handler = &HttpSM::state_watch_for_client_abort; - ua_entry->read_vio = ua_producer->vc->do_io_read(this, INT64_MAX, c->producer->read_buffer); + ua_entry->vc_read_handler = &HttpSM::state_watch_for_client_abort; + ua_entry->vc_write_handler = &HttpSM::state_watch_for_client_abort; + ua_entry->read_vio = ua_producer->vc->do_io_read(this, INT64_MAX, c->producer->read_buffer); ua_producer->vc->do_io_shutdown(IO_SHUTDOWN_READ); ua_producer->alive = false; @@ -5889,6 +6410,7 @@ HttpSM::do_setup_post_tunnel(HttpVC_t to_vc_type) bool chunked = t_state.client_info.transfer_encoding == HttpTransact::CHUNKED_ENCODING || t_state.hdr_info.request_content_length == HTTP_UNDEFINED_CL; bool post_redirect = false; + int64_t post_bytes; HttpTunnelProducer *p = nullptr; // YTS Team, yamsat Plugin @@ -5902,8 +6424,8 @@ HttpSM::do_setup_post_tunnel(HttpVC_t to_vc_type) IOBufferReader *postdata_producer_reader = postdata_producer_buffer->alloc_reader(); postdata_producer_buffer->write(this->_postbuf.postdata_copy_buffer_start); - int64_t post_bytes = postdata_producer_reader->read_avail(); - transfered_bytes = post_bytes; + post_bytes = postdata_producer_reader->read_avail(); + transfered_bytes = post_bytes; p = tunnel.add_producer(HTTP_TUNNEL_STATIC_PRODUCER, post_bytes, postdata_producer_reader, (HttpProducerHandler) nullptr, HT_STATIC, "redirect static agent post"); } else { @@ -5920,7 +6442,7 @@ HttpSM::do_setup_post_tunnel(HttpVC_t to_vc_type) } MIOBuffer *post_buffer = new_MIOBuffer(alloc_index); IOBufferReader *buf_start = post_buffer->alloc_reader(); - int64_t post_bytes = chunked ? INT64_MAX : t_state.hdr_info.request_content_length; + post_bytes = chunked ? INT64_MAX : t_state.hdr_info.request_content_length; if (enable_redirection) { this->_postbuf.init(post_buffer->clone_reader(buf_start)); @@ -5943,6 +6465,10 @@ HttpSM::do_setup_post_tunnel(HttpVC_t to_vc_type) client_request_body_bytes = num_body_bytes; } ua_txn->get_remote_reader()->consume(num_body_bytes); + // The user agent has already sent all it has + if (ua_txn->is_read_closed()) { + post_bytes = client_request_body_bytes; + } p = tunnel.add_producer(ua_entry->vc, post_bytes - transfered_bytes, buf_start, &HttpSM::tunnel_handler_post_ua, HT_HTTP_CLIENT, "user agent post"); } @@ -5976,14 +6502,22 @@ HttpSM::do_setup_post_tunnel(HttpVC_t to_vc_type) break; } - // The user agent may support chunked (HTTP/1.1) or not (HTTP/2) - // In either case, the server will support chunked (HTTP/1.1) + // The user agent and origin may support chunked (HTTP/1.1) or not (HTTP/2) if (chunked) { if (ua_txn->is_chunked_encoding_supported()) { - tunnel.set_producer_chunking_action(p, 0, TCA_PASSTHRU_CHUNKED_CONTENT); + if (server_txn->is_chunked_encoding_supported()) { + tunnel.set_producer_chunking_action(p, 0, TCA_PASSTHRU_CHUNKED_CONTENT); + } else { + tunnel.set_producer_chunking_action(p, 0, TCA_DECHUNK_CONTENT); + tunnel.set_producer_chunking_size(p, 0); + } } else { - tunnel.set_producer_chunking_action(p, 0, TCA_CHUNK_CONTENT); - tunnel.set_producer_chunking_size(p, 0); + if (server_txn->is_chunked_encoding_supported()) { + tunnel.set_producer_chunking_action(p, 0, TCA_CHUNK_CONTENT); + tunnel.set_producer_chunking_size(p, 0); + } else { + tunnel.set_producer_chunking_action(p, 0, TCA_PASSTHRU_DECHUNKED_CONTENT); + } } } @@ -6150,10 +6684,21 @@ HttpSM::write_header_into_buffer(HTTPHdr *h, MIOBuffer *b) return dumpoffset; } +void +HttpSM::write_outbound_proxy_protocol() +{ + int64_t nbytes = 1; + if (t_state.txn_conf->proxy_protocol_out >= 0) { + nbytes = do_outbound_proxy_protocol(server_txn->get_remote_reader()->mbuf, server_txn->get_netvc(), ua_txn->get_netvc(), + t_state.txn_conf->proxy_protocol_out); + } + server_entry->write_vio = server_txn->do_io_write(this, nbytes, server_txn->get_remote_reader()); +} + void HttpSM::attach_server_session() { - hsm_release_assert(server_entry == nullptr); + hsm_release_assert(this->server_entry == nullptr); // In the h1 only origin version, the transact_count was updated after making this assignment. // The SSN-TXN-COUNT option in header rewrite relies on this fact, so we decrement here so the // plugin API interface is consistent as we move to more protocols to origin @@ -6185,10 +6730,10 @@ HttpSM::attach_server_session() server_txn->increment_transactions_stat(); // Record the VC in our table - server_entry = vc_table.new_entry(); - server_entry->vc = server_txn; - server_entry->vc_type = HTTP_SERVER_VC; - server_entry->vc_handler = &HttpSM::state_send_server_request_header; + server_entry = vc_table.new_entry(); + server_entry->vc = server_txn; + server_entry->vc_type = HTTP_SERVER_VC; + server_entry->vc_write_handler = &HttpSM::state_send_server_request_header; UnixNetVConnection *server_vc = static_cast(server_txn->get_netvc()); @@ -6232,12 +6777,14 @@ HttpSM::attach_server_session() // Do we need Transfer_Encoding? if (ua_txn->has_request_body(t_state.hdr_info.request_content_length, t_state.client_info.transfer_encoding == HttpTransact::CHUNKED_ENCODING)) { - // See if we need to insert a chunked header - if (!t_state.hdr_info.server_request.presence(MIME_PRESENCE_CONTENT_LENGTH)) { - // Stuff in a TE setting so we treat this as chunked, sort of. - t_state.server_info.transfer_encoding = HttpTransact::CHUNKED_ENCODING; - t_state.hdr_info.server_request.value_append(MIME_FIELD_TRANSFER_ENCODING, MIME_LEN_TRANSFER_ENCODING, HTTP_VALUE_CHUNKED, - HTTP_LEN_CHUNKED, true); + if (server_txn->is_chunked_encoding_supported()) { + // See if we need to insert a chunked header + if (!t_state.hdr_info.server_request.presence(MIME_PRESENCE_CONTENT_LENGTH)) { + // Stuff in a TE setting so we treat this as chunked, sort of. + t_state.server_info.transfer_encoding = HttpTransact::CHUNKED_ENCODING; + t_state.hdr_info.server_request.value_append(MIME_FIELD_TRANSFER_ENCODING, MIME_LEN_TRANSFER_ENCODING, HTTP_VALUE_CHUNKED, + HTTP_LEN_CHUNKED, true); + } } } @@ -6266,8 +6813,8 @@ HttpSM::setup_server_send_request() hsm_release_assert(server_entry->vc == server_txn); // Send the request header - server_entry->vc_handler = &HttpSM::state_send_server_request_header; - server_entry->write_buffer = new_MIOBuffer(HTTP_HEADER_BUFFER_SIZE_INDEX); + server_entry->vc_write_handler = &HttpSM::state_send_server_request_header; + server_entry->write_buffer = new_MIOBuffer(HTTP_HEADER_BUFFER_SIZE_INDEX); if (t_state.api_server_request_body_set) { msg_len = t_state.internal_msg_buffer_size; @@ -6313,8 +6860,8 @@ HttpSM::setup_server_read_response_header() // Now that we've got the ability to read from the // server, setup to read the response header - server_entry->vc_handler = &HttpSM::state_read_server_response_header; - server_entry->vc = server_txn; + server_entry->vc_read_handler = &HttpSM::state_read_server_response_header; + server_entry->vc = server_txn; t_state.current.state = HttpTransact::STATE_UNDEFINED; t_state.current.server->state = HttpTransact::STATE_UNDEFINED; @@ -6327,10 +6874,6 @@ HttpSM::setup_server_read_response_header() server_response_hdr_bytes = 0; milestones[TS_MILESTONE_SERVER_READ_HEADER_DONE] = 0; - // We already done the READ when we setup the connection to - // read the request header - ink_assert(server_entry->read_vio); - // The tunnel from OS to UA is now setup. Ready to read the response server_entry->read_vio = server_txn->do_io_read(this, INT64_MAX, server_txn->get_remote_reader()->mbuf); @@ -6769,36 +7312,6 @@ HttpSM::setup_transfer_from_transform_to_cache_only() return p; } -void -HttpSM::setup_server_transfer_to_cache_only() -{ - TunnelChunkingAction_t action; - int64_t alloc_index; - int64_t nbytes; - - alloc_index = find_server_buffer_size(); - MIOBuffer *buf = new_MIOBuffer(alloc_index); - IOBufferReader *buf_start = buf->alloc_reader(); - - action = (t_state.current.server && t_state.current.server->transfer_encoding == HttpTransact::CHUNKED_ENCODING) ? - TCA_DECHUNK_CONTENT : - TCA_PASSTHRU_DECHUNKED_CONTENT; - - nbytes = server_transfer_init(buf, 0); - - HTTP_SM_SET_DEFAULT_HANDLER(&HttpSM::tunnel_handler); - - HttpTunnelProducer *p = - tunnel.add_producer(server_entry->vc, nbytes, buf_start, &HttpSM::tunnel_handler_server, HT_HTTP_SERVER, "http server"); - - tunnel.set_producer_chunking_action(p, 0, action); - tunnel.set_producer_chunking_size(p, t_state.txn_conf->http_chunking_size); - - setup_cache_write_transfer(&cache_sm, server_entry->vc, &t_state.cache_info.object_store, 0, "cache write"); - - server_entry->in_tunnel = true; -} - HttpTunnelProducer * HttpSM::setup_server_transfer() { @@ -6864,28 +7377,6 @@ HttpSM::setup_server_transfer() this->setup_plugin_agents(p, client_response_hdr_bytes); - // If the incoming server response is chunked and the client does not - // expect a chunked response, then dechunk it. Otherwise, if the - // incoming response is not chunked and the client expects a chunked - // response, then chunk it. - /* - // this block is moved up so that we know if we need to remove - // Content-Length field from response header before writing the - // response header into buffer bz50730 - TunnelChunkingAction_t action; - if (t_state.client_info.receive_chunked_response == false) { - if (t_state.current.server->transfer_encoding == - HttpTransact::CHUNKED_ENCODING) - action = TCA_DECHUNK_CONTENT; - else action = TCA_PASSTHRU_DECHUNKED_CONTENT; - } - else { - if (t_state.current.server->transfer_encoding != - HttpTransact::CHUNKED_ENCODING) - action = TCA_CHUNK_CONTENT; - else action = TCA_PASSTHRU_CHUNKED_CONTENT; - } - */ tunnel.set_producer_chunking_action(p, client_response_hdr_bytes, action); tunnel.set_producer_chunking_size(p, t_state.txn_conf->http_chunking_size); return p; @@ -7076,12 +7567,17 @@ HttpSM::kill_this() transform_cache_sm.end_both(); vc_table.cleanup_all(); - // tunnel.deallocate_buffers(); - // Why don't we just kill the tunnel? Might still be - // active if the state machine is going down hard, - // and we should clean it up. + // Clean up the tunnel resources. Take + // it down if it is still active tunnel.kill_tunnel(); + if (_netvc) { + _netvc->do_io_close(); + free_MIOBuffer(_netvc_read_buffer); + } else if (server_txn == nullptr) { + this->cancel_pending_server_connection(); + } + // It possible that a plugin added transform hook // but the hook never executed due to a client abort // In that case, we need to manually close all the @@ -7157,6 +7653,8 @@ HttpSM::kill_this() } if (ua_txn) { ua_txn->transaction_done(); + } else { + ink_release_assert(!"Cannot transact done"); } // In the async state, the plugin could have been @@ -7502,7 +8000,6 @@ HttpSM::set_next_state() case HttpTransact::SM_ACTION_DNS_LOOKUP: { sockaddr const *addr; - if (t_state.api_server_addr_set) { /* If the API has set the server address before the OS DNS lookup * then we can skip the lookup @@ -7588,8 +8085,7 @@ HttpSM::set_next_state() } ink_assert(t_state.dns_info.looking_up != HttpTransact::UNDEFINED_LOOKUP); - do_hostdb_lookup(); - break; + return do_hostdb_lookup(); } case HttpTransact::SM_ACTION_DNS_REVERSE_LOOKUP: { @@ -8189,6 +8685,9 @@ HttpSM::get_http_schedule(int event, void * /* data ATS_UNUSED */) return 0; } +/* + * Used from an InkAPI + */ bool HttpSM::set_server_session_private(bool private_session) { @@ -8199,8 +8698,8 @@ HttpSM::set_server_session_private(bool private_session) return false; } -inline bool -HttpSM::is_private() +bool +HttpSM::is_private() const { bool res = false; if (will_be_private_ss) { diff --git a/proxy/http/HttpSM.h b/proxy/http/HttpSM.h index ac7c74512eb..67af4a32a5a 100644 --- a/proxy/http/HttpSM.h +++ b/proxy/http/HttpSM.h @@ -48,6 +48,9 @@ #define HTTP_API_CONTINUE (INK_API_EVENT_EVENTS_START + 0) #define HTTP_API_ERROR (INK_API_EVENT_EVENTS_START + 1) +#define CONNECT_EVENT_TXN (HTTP_NET_CONNECTION_EVENT_EVENTS_START) + 0 +#define CONNECT_EVENT_DIRECT (HTTP_NET_CONNECTION_EVENT_EVENTS_START) + 1 + // The default size for http header buffers when we don't // need to include extra space for the document static size_t const HTTP_HEADER_BUFFER_SIZE_INDEX = CLIENT_CONNECTION_FIRST_READ_BUFFER_SIZE_INDEX; @@ -59,7 +62,7 @@ static size_t const HTTP_HEADER_BUFFER_SIZE_INDEX = CLIENT_CONNECTION_FIRST_READ // the larger buffer size static size_t const HTTP_SERVER_RESP_HDR_BUFFER_INDEX = BUFFER_SIZE_INDEX_8K; -class Http1ServerSession; +class PoolableSession; class AuthHttpAdapter; class HttpSM; @@ -90,7 +93,8 @@ struct HttpVCTableEntry { MIOBuffer *write_buffer; VIO *read_vio; VIO *write_vio; - HttpSMHandler vc_handler; + HttpSMHandler vc_read_handler; + HttpSMHandler vc_write_handler; HttpVC_t vc_type; HttpSM *sm; bool eos; @@ -269,21 +273,24 @@ class HttpSM : public Continuation, public PluginUserArgs // holding the lock for the server session void attach_server_session(); - PoolableSession *create_server_session(NetVConnection *netvc); + PoolableSession *create_server_session(NetVConnection *netvc, MIOBuffer *netvc_read_buffer, IOBufferReader *netvc_reader); bool create_server_txn(PoolableSession *new_session); HTTPVersion get_server_version(HTTPHdr &hdr) const; + // Write out the proxy_protocol information on a new outbound connection + void write_outbound_proxy_protocol(); + ProxyTransaction * - get_ua_txn() + get_server_txn() { - return ua_txn; + return server_txn; } ProxyTransaction * - get_server_txn() + get_ua_txn() { - return server_txn; + return ua_txn; } // Called by transact. Updates are fire and forget @@ -316,6 +323,8 @@ class HttpSM : public Continuation, public PluginUserArgs // A NULL 'r' argument indicates the hostdb lookup failed void process_hostdb_info(HostDBInfo *r); void process_srv_info(HostDBInfo *r); + bool origin_multiplexed() const; + bool add_to_existing_request(); // Called by transact. Synchronous. VConnection *do_transform_open(); @@ -349,7 +358,7 @@ class HttpSM : public Continuation, public PluginUserArgs void txn_hook_add(TSHttpHookID id, INKContInternal *cont); APIHook *txn_hook_get(TSHttpHookID id); - bool is_private(); + bool is_private() const; bool is_redirect_required(); /// Get the protocol stack for the inbound (client, user agent) connection. @@ -454,6 +463,7 @@ class HttpSM : public Continuation, public PluginUserArgs int tunnel_handler(int event, void *data); int tunnel_handler_push(int event, void *data); int tunnel_handler_post(int event, void *data); + int tunnel_handler_trailer(int event, void *data); // YTS Team, yamsat Plugin int tunnel_handler_for_partial_post(int event, void *data); @@ -502,6 +512,8 @@ class HttpSM : public Continuation, public PluginUserArgs int tunnel_handler_cache_read(int event, HttpTunnelProducer *p); int tunnel_handler_post_ua(int event, HttpTunnelProducer *c); int tunnel_handler_post_server(int event, HttpTunnelConsumer *c); + int tunnel_handler_trailer_ua(int event, HttpTunnelConsumer *c); + int tunnel_handler_trailer_server(int event, HttpTunnelProducer *c); int tunnel_handler_ssl_producer(int event, HttpTunnelProducer *p); int tunnel_handler_ssl_consumer(int event, HttpTunnelConsumer *p); int tunnel_handler_transform_write(int event, HttpTunnelConsumer *c); @@ -511,7 +523,7 @@ class HttpSM : public Continuation, public PluginUserArgs void do_hostdb_lookup(); void do_hostdb_reverse_lookup(); void do_cache_lookup_and_read(); - void do_http_server_open(bool raw = false); + void do_http_server_open(bool raw = false, bool only_direct = false); void send_origin_throttled_response(); void do_setup_post_tunnel(HttpVC_t to_vc_type); void do_cache_prepare_write(); @@ -545,7 +557,6 @@ class HttpSM : public Continuation, public PluginUserArgs void setup_server_send_request(); void setup_server_send_request_api(); HttpTunnelProducer *setup_server_transfer(); - void setup_server_transfer_to_cache_only(); HttpTunnelProducer *setup_cache_read_transfer(); void setup_internal_transfer(HttpSMHandler handler); void setup_error_transfer(); @@ -698,10 +709,15 @@ class HttpSM : public Continuation, public PluginUserArgs void rewind_state_machine(); private: + void cancel_pending_server_connection(); + PostDataBuffers _postbuf; int _client_connection_id = -1, _client_transaction_id = -1; int _client_transaction_priority_weight = -1, _client_transaction_priority_dependence = -1; - bool _from_early_data = false; + bool _from_early_data = false; + NetVConnection *_netvc = nullptr; + IOBufferReader *_netvc_reader = nullptr; + MIOBuffer *_netvc_read_buffer = nullptr; }; // Function to get the cache_sm object - YTS Team, yamsat diff --git a/proxy/http/HttpSessionManager.cc b/proxy/http/HttpSessionManager.cc index d20a1e9e7fe..09aeb8d73e0 100644 --- a/proxy/http/HttpSessionManager.cc +++ b/proxy/http/HttpSessionManager.cc @@ -165,7 +165,9 @@ ServerSessionPool::acquireSession(sockaddr const *addr, CryptoHash const &hostna } if (zret == HSM_DONE) { to_return = first; - this->removeSession(to_return); + if (!to_return->is_multiplexing()) { + this->removeSession(to_return); + } } else if (first != m_fqdn_pool.end()) { Debug("http_ss", "Failed find entry due to name mismatch %s", sm->t_state.current.server->name); } @@ -190,7 +192,9 @@ ServerSessionPool::acquireSession(sockaddr const *addr, CryptoHash const &hostna } if (zret == HSM_DONE) { to_return = first; - this->removeSession(to_return); + if (!to_return->is_multiplexing()) { + this->removeSession(to_return); + } } } return zret; @@ -447,7 +451,10 @@ HttpSessionManager::_acquire_session(sockaddr const *ip, CryptoHash const &hostn } else { Debug("http_ss", "[%" PRId64 "] [acquire session] failed to get transaction on session from shared pool", to_return->connection_id()); - to_return->do_io_close(); + // Don't close the H2 origin. Otherwise you get use-after free with the activity timeout cop + if (!to_return->is_multiplexing()) { + to_return->do_io_close(); + } retval = HSM_RETRY; } } diff --git a/proxy/http/HttpSessionManager.h b/proxy/http/HttpSessionManager.h index 8ce9a89baab..6645122cc8e 100644 --- a/proxy/http/HttpSessionManager.h +++ b/proxy/http/HttpSessionManager.h @@ -67,6 +67,8 @@ class ServerSessionPool : public Continuation static bool validate_host_sni(HttpSM *sm, NetVConnection *netvc); static bool validate_sni(HttpSM *sm, NetVConnection *netvc); static bool validate_cert(HttpSM *sm, NetVConnection *netvc); + void removeSession(PoolableSession *ssn); + void addSession(PoolableSession *ssn); int count() const { @@ -74,9 +76,6 @@ class ServerSessionPool : public Continuation } private: - void removeSession(PoolableSession *ssn); - void addSession(PoolableSession *ssn); - using IPTable = IntrusiveHashMap; using FQDNTable = IntrusiveHashMap; diff --git a/proxy/http/HttpTunnel.cc b/proxy/http/HttpTunnel.cc index 3f7a3b53e47..a9601b69808 100644 --- a/proxy/http/HttpTunnel.cc +++ b/proxy/http/HttpTunnel.cc @@ -715,6 +715,7 @@ HttpTunnel::chain(HttpTunnelConsumer *c, HttpTunnelProducer *p) void HttpTunnel::tunnel_run(HttpTunnelProducer *p_arg) { + ++reentrancy_count; Debug("http_tunnel", "tunnel_run started, p_arg is %s", p_arg ? "provided" : "NULL"); if (p_arg) { producer_run(p_arg); @@ -730,6 +731,7 @@ HttpTunnel::tunnel_run(HttpTunnelProducer *p_arg) } } } + --reentrancy_count; // It is possible that there was nothing to do // due to a all transfers being zero length @@ -974,11 +976,14 @@ HttpTunnel::producer_run(HttpTunnelProducer *p) p->handler_state = HTTP_SM_POST_SUCCESS; } } + Debug("http_tunnel", "Start write vio %ld bytes", c_write); // Start the writes now that we know we will consume all the initial data c->write_vio = c->vc->do_io_write(this, c_write, c->buffer_reader); ink_assert(c_write > 0); if (c->write_vio == nullptr) { consumer_handler(VC_EVENT_ERROR, c); + } else if (c->write_vio->ntodo() == 0 && c->alive) { + consumer_handler(VC_EVENT_WRITE_COMPLETE, c); } } } @@ -998,9 +1003,17 @@ HttpTunnel::producer_run(HttpTunnelProducer *p) if (read_start_pos > 0) { p->read_vio = ((CacheVC *)p->vc)->do_io_pread(this, producer_n, p->read_buffer, read_start_pos); } else { + Debug("http_tunnel", "Start read vio %ld bytes", producer_n); p->read_vio = p->vc->do_io_read(this, producer_n, p->read_buffer); } } + } else { + // If the producer is not alive (precomplete) make sure to kick the consumers + for (c = p->consumer_list.head; c; c = c->link.next) { + if (c->alive && c->write_vio) { + c->write_vio->reenable(); + } + } } // Now that the tunnel has started, we must remove producer's reader so @@ -1126,14 +1139,6 @@ HttpTunnel::producer_handler(int event, HttpTunnelProducer *p) // Handle chunking/dechunking/chunked-passthrough if necessary. if (p->do_chunking) { event = producer_handler_dechunked(event, p); - - // If we were in PRECOMPLETE when this function was called - // and we are doing chunking, then we just wrote the last - // chunk in the the function call above. We are done with the - // tunnel. - if (event == HTTP_TUNNEL_EVENT_PRECOMPLETE) { - event = VC_EVENT_EOS; - } } else if (p->do_dechunking || p->do_chunked_passthru) { event = producer_handler_chunked(event, p); } else { @@ -1172,6 +1177,7 @@ HttpTunnel::producer_handler(int event, HttpTunnelProducer *p) // Data read from producer, reenable consumers for (c = p->consumer_list.head; c; c = c->link.next) { if (c->alive && c->write_vio) { + Debug("http_redirect", "Read ready alive"); c->write_vio->reenable(); } } @@ -1181,6 +1187,8 @@ HttpTunnel::producer_handler(int event, HttpTunnelProducer *p) // If the write completes on the stack (as it can for http2), then // consumer could have called back by this point. Must treat this as // a regular read complete (falling through to the following cases). + p->bytes_read = p->init_bytes_done; + [[fallthrough]]; case VC_EVENT_READ_COMPLETE: case VC_EVENT_EOS: @@ -1194,7 +1202,6 @@ HttpTunnel::producer_handler(int event, HttpTunnelProducer *p) // the message length being a property of the encoding) // In that case, we won't have done a do_io so there // will not be vio - p->bytes_read = 0; } // callback the SM to notify of completion @@ -1209,9 +1216,12 @@ HttpTunnel::producer_handler(int event, HttpTunnelProducer *p) sm_callback = true; p->update_state_if_not_set(HTTP_SM_POST_SUCCESS); - // Data read from producer, reenable consumers + // Kick off the consumers if appropriate for (c = p->consumer_list.head; c; c = c->link.next) { if (c->alive && c->write_vio) { + if (c->write_vio->nbytes == INT64_MAX) { + c->write_vio->nbytes = p->bytes_read + p->init_bytes_done - c->skip_bytes; + } c->write_vio->reenable(); } } @@ -1347,6 +1357,9 @@ HttpTunnel::consumer_handler(int event, HttpTunnelConsumer *c) case VC_EVENT_INACTIVITY_TIMEOUT: ink_assert(c->alive); ink_assert(c->buffer_reader); + if (c->write_vio) { + c->write_vio->reenable(); + } c->alive = false; c->bytes_written = c->write_vio ? c->write_vio->ndone : 0; diff --git a/proxy/http2/HTTP2.cc b/proxy/http2/HTTP2.cc index 648609cbb80..c070d5d6167 100644 --- a/proxy/http2/HTTP2.cc +++ b/proxy/http2/HTTP2.cc @@ -61,11 +61,16 @@ Http2HeaderName http2_connection_specific_headers[5] = {}; // Statistics RecRawStatBlock *http2_rsb; static const char *const HTTP2_STAT_CURRENT_CLIENT_CONNECTION_NAME = "proxy.process.http2.current_client_connections"; +static const char *const HTTP2_STAT_CURRENT_SERVER_CONNECTION_NAME = "proxy.process.http2.current_server_connections"; static const char *const HTTP2_STAT_CURRENT_ACTIVE_CLIENT_CONNECTION_NAME = "proxy.process.http2.current_active_client_connections"; +static const char *const HTTP2_STAT_CURRENT_ACTIVE_SERVER_CONNECTION_NAME = "proxy.process.http2.current_active_server_connections"; static const char *const HTTP2_STAT_CURRENT_CLIENT_STREAM_NAME = "proxy.process.http2.current_client_streams"; +static const char *const HTTP2_STAT_CURRENT_SERVER_STREAM_NAME = "proxy.process.http2.current_server_streams"; static const char *const HTTP2_STAT_TOTAL_CLIENT_STREAM_NAME = "proxy.process.http2.total_client_streams"; +static const char *const HTTP2_STAT_TOTAL_SERVER_STREAM_NAME = "proxy.process.http2.total_server_streams"; static const char *const HTTP2_STAT_TOTAL_TRANSACTIONS_TIME_NAME = "proxy.process.http2.total_transactions_time"; static const char *const HTTP2_STAT_TOTAL_CLIENT_CONNECTION_NAME = "proxy.process.http2.total_client_connections"; +static const char *const HTTP2_STAT_TOTAL_SERVER_CONNECTION_NAME = "proxy.process.http2.total_server_connections"; static const char *const HTTP2_STAT_CONNECTION_ERRORS_NAME = "proxy.process.http2.connection_errors"; static const char *const HTTP2_STAT_STREAM_ERRORS_NAME = "proxy.process.http2.stream_errors"; static const char *const HTTP2_STAT_SESSION_DIE_DEFAULT_NAME = "proxy.process.http2.session_die_default"; @@ -442,10 +447,16 @@ http2_convert_header_from_2_to_1_1(HTTPHdr *headers) if (MIMEField *field = headers->field_find(HTTP2_VALUE_AUTHORITY, HTTP2_LEN_AUTHORITY); field != nullptr && field->value_is_valid()) { int authority_len; - const char *authority = field->value_get(&authority_len); + // Set the host header field + MIMEField *host = headers->field_find(MIME_FIELD_HOST, MIME_LEN_HOST); + if (host == nullptr) { + host = headers->field_create(MIME_FIELD_HOST, MIME_LEN_HOST); + headers->field_attach(host); + } + const char *authority = field->value_get(&authority_len); url_host_set(headers->m_heap, headers->m_http->u.req.m_url_impl, authority, authority_len, true); - + host->value_set(headers->m_heap, headers->m_mime, authority, authority_len); headers->field_delete(field); } else { return PARSE_RESULT_ERROR; @@ -595,6 +606,9 @@ http2_convert_header_from_1_1_to_2(HTTPHdr *headers) } else { field->value_set(headers->m_heap, headers->m_mime, value, value_len); } + // Remove the host header field, redundant to the authority field + // For istio/envoy, having both was causing 404 responses + headers->field_delete(MIME_FIELD_HOST, MIME_LEN_HOST); } else { ink_abort("initialize HTTP/2 pseudo-headers"); return PARSE_RESULT_ERROR; @@ -604,13 +618,27 @@ http2_convert_header_from_1_1_to_2(HTTPHdr *headers) if (MIMEField *field = headers->field_find(HTTP2_VALUE_PATH, HTTP2_LEN_PATH); field != nullptr) { int value_len; const char *value = headers->path_get(&value_len); + int param_len; + const char *param = headers->params_get(¶m_len); + int query_len; + const char *query = headers->query_get(&query_len); + int path_len = value_len + 1; - ts::LocalBuffer buf(value_len + 1); + ts::LocalBuffer buf(value_len + 1 + 1 + 1 + query_len + param_len); char *path = buf.data(); path[0] = '/'; memcpy(path + 1, value, value_len); - - field->value_set(headers->m_heap, headers->m_mime, path, value_len + 1); + if (param_len > 0) { + path[path_len] = ';'; + memcpy(path + path_len + 1, param, param_len); + path_len += 1 + param_len; + } + if (query_len > 0) { + path[path_len] = '?'; + memcpy(path + path_len + 1, query, query_len); + path_len += 1 + query_len; + } + field->value_set(headers->m_heap, headers->m_mime, path, path_len); } else { ink_abort("initialize HTTP/2 pseudo-headers"); return PARSE_RESULT_ERROR; @@ -678,12 +706,11 @@ http2_encode_header_blocks(HTTPHdr *in, uint8_t *out, uint32_t out_len, uint32_t */ Http2ErrorCode http2_decode_header_blocks(HTTPHdr *hdr, const uint8_t *buf_start, const uint32_t buf_len, uint32_t *len_read, HpackHandle &handle, - bool &trailing_header, uint32_t maximum_table_size) + bool is_trailing_header, uint32_t maximum_table_size, bool is_outbound) { const MIMEField *field; const char *value; int len; - bool is_trailing_header = trailing_header; int64_t result = hpack_decode_header_block(handle, hdr, buf_start, buf_len, Http2::max_header_list_size, maximum_table_size); if (result < 0) { @@ -700,7 +727,7 @@ http2_decode_header_blocks(HTTPHdr *hdr, const uint8_t *buf_start, const uint32_ } MIMEFieldIter iter; - unsigned int expected_pseudo_header_count = 4; + unsigned int expected_pseudo_header_count = is_outbound ? 1 : 4; unsigned int pseudo_header_count = 0; if (is_trailing_header) { @@ -728,7 +755,7 @@ http2_decode_header_blocks(HTTPHdr *hdr, const uint8_t *buf_start, const uint32_ if (hdr->field_find(MIME_FIELD_CONNECTION, MIME_LEN_CONNECTION) != nullptr || hdr->field_find(MIME_FIELD_KEEP_ALIVE, MIME_LEN_KEEP_ALIVE) != nullptr || hdr->field_find(MIME_FIELD_PROXY_CONNECTION, MIME_LEN_PROXY_CONNECTION) != nullptr || - hdr->field_find(MIME_FIELD_TRANSFER_ENCODING, MIME_LEN_TRANSFER_ENCODING) != nullptr || + // hdr->field_find(MIME_FIELD_TRANSFER_ENCODING, MIME_LEN_TRANSFER_ENCODING) != nullptr || hdr->field_find(MIME_FIELD_UPGRADE, MIME_LEN_UPGRADE) != nullptr) { return Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR; } @@ -742,13 +769,6 @@ http2_decode_header_blocks(HTTPHdr *hdr, const uint8_t *buf_start, const uint32_ } } - // turn on that we have a trailer header - const char trailer_name[] = "trailer"; - field = hdr->field_find(trailer_name, sizeof(trailer_name) - 1); - if (field) { - trailing_header = true; - } - // when The TE header field is received, it MUST NOT contain any // value other than "trailers". field = hdr->field_find(MIME_FIELD_TE, MIME_LEN_TE); @@ -761,18 +781,29 @@ http2_decode_header_blocks(HTTPHdr *hdr, const uint8_t *buf_start, const uint32_ if (!is_trailing_header) { // Check pseudo headers - if (hdr->fields_count() >= 4) { - if (hdr->field_find(HTTP2_VALUE_SCHEME, HTTP2_LEN_SCHEME) == nullptr || - hdr->field_find(HTTP2_VALUE_METHOD, HTTP2_LEN_METHOD) == nullptr || - hdr->field_find(HTTP2_VALUE_PATH, HTTP2_LEN_PATH) == nullptr || - hdr->field_find(HTTP2_VALUE_AUTHORITY, HTTP2_LEN_AUTHORITY) == nullptr || - hdr->field_find(HTTP2_VALUE_STATUS, HTTP2_LEN_STATUS) != nullptr) { - // Decoded header field is invalid + if (is_outbound) { + if (hdr->fields_count() >= 1) { + if (hdr->field_find(HTTP2_VALUE_STATUS, HTTP2_LEN_STATUS) == nullptr) { + return Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR; + } + } else { + // Pseudo headers is insufficient return Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR; } } else { - // Pseudo headers is insufficient - return Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR; + if (hdr->fields_count() >= 4) { + if (hdr->field_find(HTTP2_VALUE_SCHEME, HTTP2_LEN_SCHEME) == nullptr || + hdr->field_find(HTTP2_VALUE_METHOD, HTTP2_LEN_METHOD) == nullptr || + hdr->field_find(HTTP2_VALUE_PATH, HTTP2_LEN_PATH) == nullptr || + hdr->field_find(HTTP2_VALUE_AUTHORITY, HTTP2_LEN_AUTHORITY) == nullptr || + hdr->field_find(HTTP2_VALUE_STATUS, HTTP2_LEN_STATUS) != nullptr) { + // Decoded header field is invalid + return Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR; + } + } else { + // Pseudo headers is insufficient + return Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR; + } } } @@ -786,11 +817,13 @@ uint32_t Http2::max_active_streams_in = 0; bool Http2::throttling = false; uint32_t Http2::stream_priority_enabled = 0; uint32_t Http2::initial_window_size = 65535; +uint32_t Http2::session_initial_window_size = 65535; uint32_t Http2::max_frame_size = 16384; uint32_t Http2::header_table_size = 4096; uint32_t Http2::max_header_list_size = 4294967295; uint32_t Http2::accept_no_activity_timeout = 120; uint32_t Http2::no_activity_timeout_in = 120; +uint32_t Http2::no_activity_timeout_out = 120; uint32_t Http2::active_timeout_in = 0; uint32_t Http2::push_diary_size = 256; uint32_t Http2::zombie_timeout_in = 0; @@ -816,11 +849,19 @@ Http2::init() REC_EstablishStaticConfigInt32U(max_active_streams_in, "proxy.config.http2.max_active_streams_in"); REC_EstablishStaticConfigInt32U(stream_priority_enabled, "proxy.config.http2.stream_priority_enabled"); REC_EstablishStaticConfigInt32U(initial_window_size, "proxy.config.http2.initial_window_size_in"); + REC_EstablishStaticConfigInt32U(session_initial_window_size, "proxy.config.http2.session_initial_window_size_in"); + + // The session window must be at least as big as the stream window + if (session_initial_window_size < initial_window_size) { + session_initial_window_size = initial_window_size; + } + REC_EstablishStaticConfigInt32U(max_frame_size, "proxy.config.http2.max_frame_size"); REC_EstablishStaticConfigInt32U(header_table_size, "proxy.config.http2.header_table_size"); REC_EstablishStaticConfigInt32U(max_header_list_size, "proxy.config.http2.max_header_list_size"); REC_EstablishStaticConfigInt32U(accept_no_activity_timeout, "proxy.config.http2.accept_no_activity_timeout"); REC_EstablishStaticConfigInt32U(no_activity_timeout_in, "proxy.config.http2.no_activity_timeout_in"); + REC_EstablishStaticConfigInt32U(no_activity_timeout_out, "proxy.config.http2.no_activity_timeout_out"); REC_EstablishStaticConfigInt32U(active_timeout_in, "proxy.config.http2.active_timeout_in"); REC_EstablishStaticConfigInt32U(push_diary_size, "proxy.config.http2.push_diary_size"); REC_EstablishStaticConfigInt32U(zombie_timeout_in, "proxy.config.http2.zombie_debug_timeout_in"); @@ -857,18 +898,31 @@ Http2::init() RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_CURRENT_CLIENT_CONNECTION_NAME, RECD_INT, RECP_NON_PERSISTENT, static_cast(HTTP2_STAT_CURRENT_CLIENT_SESSION_COUNT), RecRawStatSyncSum); HTTP2_CLEAR_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_SESSION_COUNT); + RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_CURRENT_SERVER_CONNECTION_NAME, RECD_INT, RECP_NON_PERSISTENT, + static_cast(HTTP2_STAT_CURRENT_SERVER_SESSION_COUNT), RecRawStatSyncSum); + HTTP2_CLEAR_DYN_STAT(HTTP2_STAT_CURRENT_SERVER_SESSION_COUNT); RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_CURRENT_ACTIVE_CLIENT_CONNECTION_NAME, RECD_INT, RECP_NON_PERSISTENT, static_cast(HTTP2_STAT_CURRENT_ACTIVE_CLIENT_CONNECTION_COUNT), RecRawStatSyncSum); HTTP2_CLEAR_DYN_STAT(HTTP2_STAT_CURRENT_ACTIVE_CLIENT_CONNECTION_COUNT); + RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_CURRENT_ACTIVE_SERVER_CONNECTION_NAME, RECD_INT, RECP_NON_PERSISTENT, + static_cast(HTTP2_STAT_CURRENT_ACTIVE_SERVER_CONNECTION_COUNT), RecRawStatSyncSum); + HTTP2_CLEAR_DYN_STAT(HTTP2_STAT_CURRENT_ACTIVE_SERVER_CONNECTION_COUNT); RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_CURRENT_CLIENT_STREAM_NAME, RECD_INT, RECP_NON_PERSISTENT, static_cast(HTTP2_STAT_CURRENT_CLIENT_STREAM_COUNT), RecRawStatSyncSum); HTTP2_CLEAR_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_STREAM_COUNT); + RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_CURRENT_SERVER_STREAM_NAME, RECD_INT, RECP_NON_PERSISTENT, + static_cast(HTTP2_STAT_CURRENT_SERVER_STREAM_COUNT), RecRawStatSyncSum); + HTTP2_CLEAR_DYN_STAT(HTTP2_STAT_CURRENT_SERVER_STREAM_COUNT); RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_TOTAL_CLIENT_STREAM_NAME, RECD_INT, RECP_PERSISTENT, static_cast(HTTP2_STAT_TOTAL_CLIENT_STREAM_COUNT), RecRawStatSyncCount); + RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_TOTAL_SERVER_STREAM_NAME, RECD_INT, RECP_PERSISTENT, + static_cast(HTTP2_STAT_TOTAL_SERVER_STREAM_COUNT), RecRawStatSyncCount); RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_TOTAL_TRANSACTIONS_TIME_NAME, RECD_INT, RECP_PERSISTENT, static_cast(HTTP2_STAT_TOTAL_TRANSACTIONS_TIME), RecRawStatSyncSum); RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_TOTAL_CLIENT_CONNECTION_NAME, RECD_INT, RECP_PERSISTENT, static_cast(HTTP2_STAT_TOTAL_CLIENT_CONNECTION_COUNT), RecRawStatSyncSum); + RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_TOTAL_SERVER_CONNECTION_NAME, RECD_INT, RECP_PERSISTENT, + static_cast(HTTP2_STAT_TOTAL_SERVER_CONNECTION_COUNT), RecRawStatSyncSum); RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_CONNECTION_ERRORS_NAME, RECD_INT, RECP_PERSISTENT, static_cast(HTTP2_STAT_CONNECTION_ERRORS_COUNT), RecRawStatSyncSum); RecRegisterRawStat(http2_rsb, RECT_PROCESS, HTTP2_STAT_STREAM_ERRORS_NAME, RECD_INT, RECP_PERSISTENT, diff --git a/proxy/http2/HTTP2.h b/proxy/http2/HTTP2.h index d2eed22ffe3..67d1bf6ece2 100644 --- a/proxy/http2/HTTP2.h +++ b/proxy/http2/HTTP2.h @@ -84,12 +84,17 @@ const uint8_t HTTP2_PRIORITY_DEFAULT_WEIGHT = 15; // Statistics enum { - HTTP2_STAT_CURRENT_CLIENT_SESSION_COUNT, // Current # of HTTP2 connections - HTTP2_STAT_CURRENT_ACTIVE_CLIENT_CONNECTION_COUNT, // Current # of active HTTP2 connections - HTTP2_STAT_CURRENT_CLIENT_STREAM_COUNT, // Current # of active HTTP2 streams + HTTP2_STAT_CURRENT_CLIENT_SESSION_COUNT, // Current # of inbound HTTP2 connections + HTTP2_STAT_CURRENT_SERVER_SESSION_COUNT, // Current # of outbound HTTP2 connections + HTTP2_STAT_CURRENT_ACTIVE_CLIENT_CONNECTION_COUNT, // Current # of active inbound HTTP2 connections + HTTP2_STAT_CURRENT_ACTIVE_SERVER_CONNECTION_COUNT, // Current # of active outbound HTTP2 connections + HTTP2_STAT_CURRENT_CLIENT_STREAM_COUNT, // Current # of active inbound HTTP2 streams + HTTP2_STAT_CURRENT_SERVER_STREAM_COUNT, // Current # of active outboundHTTP2 streams HTTP2_STAT_TOTAL_CLIENT_STREAM_COUNT, + HTTP2_STAT_TOTAL_SERVER_STREAM_COUNT, HTTP2_STAT_TOTAL_TRANSACTIONS_TIME, // Total stream time and streams - HTTP2_STAT_TOTAL_CLIENT_CONNECTION_COUNT, // Total connections running http2 + HTTP2_STAT_TOTAL_CLIENT_CONNECTION_COUNT, // Total inbound connections running http2 + HTTP2_STAT_TOTAL_SERVER_CONNECTION_COUNT, // Total outbound connections running http2 HTTP2_STAT_STREAM_ERRORS_COUNT, HTTP2_STAT_CONNECTION_ERRORS_COUNT, HTTP2_STAT_SESSION_DIE_DEFAULT, @@ -245,8 +250,8 @@ enum Http2SettingsIdentifier { HTTP2_SETTINGS_INITIAL_WINDOW_SIZE = 4, HTTP2_SETTINGS_MAX_FRAME_SIZE = 5, HTTP2_SETTINGS_MAX_HEADER_LIST_SIZE = 6, - - HTTP2_SETTINGS_MAX + HTTP2_SETTINGS_MAX, // Really just the max of the "densely numbered" core id's + HTTP2_SETTINGS_GRPC_ALLOW_TRUE_BINARY_METADATA = 0xfe03, }; // [RFC 7540] 4.1. Frame Format @@ -362,7 +367,8 @@ bool http2_parse_goaway(IOVec, Http2Goaway &); bool http2_parse_window_update(IOVec, uint32_t &); -Http2ErrorCode http2_decode_header_blocks(HTTPHdr *, const uint8_t *, const uint32_t, uint32_t *, HpackHandle &, bool &, uint32_t); +Http2ErrorCode http2_decode_header_blocks(HTTPHdr *, const uint8_t *, const uint32_t, uint32_t *, HpackHandle &, bool, uint32_t, + bool is_outbound = false); Http2ErrorCode http2_encode_header_blocks(HTTPHdr *, uint8_t *, uint32_t, uint32_t *, HpackHandle &, int32_t); @@ -385,11 +391,13 @@ class Http2 static bool throttling; static uint32_t stream_priority_enabled; static uint32_t initial_window_size; + static uint32_t session_initial_window_size; static uint32_t max_frame_size; static uint32_t header_table_size; static uint32_t max_header_list_size; static uint32_t accept_no_activity_timeout; static uint32_t no_activity_timeout_in; + static uint32_t no_activity_timeout_out; static uint32_t active_timeout_in; static uint32_t push_diary_size; static uint32_t zombie_timeout_in; diff --git a/proxy/http2/Http2ClientSession.cc b/proxy/http2/Http2ClientSession.cc index ea0c9630d67..0b0e0eb49ba 100644 --- a/proxy/http2/Http2ClientSession.cc +++ b/proxy/http2/Http2ClientSession.cc @@ -45,6 +45,10 @@ Http2ClientSession::destroy() in_destroy = true; REMEMBER(NO_EVENT, this->recursion) Http2SsnDebug("session destroy"); + if (_vc) { + _vc->do_io_close(); + _vc = nullptr; + } // Let everyone know we are going down do_api_callout(TS_HTTP_SSN_CLOSE_HOOK); } @@ -53,14 +57,10 @@ Http2ClientSession::destroy() void Http2ClientSession::free() { - if (_vc) { - _vc->do_io_close(); - _vc = nullptr; - } auto mutex_thread = this->mutex->thread_holding; if (Http2CommonSession::common_free(this)) { HTTP2_DECREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_SESSION_COUNT, mutex_thread); - THREAD_FREE(this, http2ClientSessionAllocator, this_ethread()); + THREAD_FREE(this, http2ClientSessionAllocator, mutex_thread); } } @@ -97,7 +97,6 @@ Http2ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOB _vc->set_inactivity_timeout(HRTIME_SECONDS(Http2::accept_no_activity_timeout)); this->schedule_event = nullptr; this->mutex = new_vc->mutex; - this->in_destroy = false; this->connection_state.mutex = this->mutex; @@ -112,7 +111,7 @@ Http2ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOB this->_vc->set_tcp_congestion_control(CLIENT_SIDE); this->read_buffer = iobuf ? iobuf : new_MIOBuffer(HTTP2_HEADER_BUFFER_SIZE_INDEX); - this->read_buffer->water_mark = connection_state.server_settings.get(HTTP2_SETTINGS_MAX_FRAME_SIZE); + this->read_buffer->water_mark = connection_state.local_settings.get(HTTP2_SETTINGS_MAX_FRAME_SIZE); this->_read_buffer_reader = reader ? reader : this->read_buffer->alloc_reader(); // This block size is the buffer size that we pass to SSLWriteBuffer @@ -134,17 +133,20 @@ void Http2ClientSession::do_io_close(int alerrno) { REMEMBER(NO_EVENT, this->recursion) - Http2SsnDebug("session closed"); - ink_assert(this->mutex->thread_holding == this_ethread()); - send_connection_event(&this->connection_state, HTTP2_SESSION_EVENT_FINI, this); + if (!this->connection_state.is_state_closed()) { + Http2SsnDebug("session closed"); - this->connection_state.release_stream(); + ink_assert(this->mutex->thread_holding == this_ethread()); + send_connection_event(&this->connection_state, HTTP2_SESSION_EVENT_FINI, this); - this->clear_session_active(); + this->connection_state.release_stream(); - // Clean up the write VIO in case of inactivity timeout - this->do_io_write(this, 0, nullptr); + this->clear_session_active(); + + // Clean up the write VIO in case of inactivity timeout + this->do_io_write(this, 0, nullptr); + } } int @@ -152,6 +154,7 @@ Http2ClientSession::main_event_handler(int event, void *edata) { ink_assert(this->mutex->thread_holding == this_ethread()); int retval; + bool set_closed = false; recursion++; @@ -185,7 +188,8 @@ Http2ClientSession::main_event_handler(int event, void *edata) Http2SsnDebug("Closing event %d", event); this->set_dying_event(event); this->do_io_close(); - retval = 0; + retval = 0; + set_closed = true; break; case VC_EVENT_WRITE_READY: @@ -227,7 +231,7 @@ Http2ClientSession::main_event_handler(int event, void *edata) } } - if (this->connection_state.get_shutdown_state() == HTTP2_SHUTDOWN_NOT_INITIATED) { + if (!set_closed && this->connection_state.get_shutdown_state() == HTTP2_SHUTDOWN_NOT_INITIATED) { send_connection_event(&this->connection_state, HTTP2_SESSION_EVENT_SHUTDOWN_INIT, this); } @@ -268,17 +272,17 @@ Http2ClientSession::get_transact_count() const return connection_state.get_stream_requests(); } -void -Http2ClientSession::release(ProxyTransaction *trans) -{ -} - const char * Http2ClientSession::get_protocol_string() const { return "http/2"; } +void +Http2ClientSession::release(ProxyTransaction *trans) +{ +} + int Http2ClientSession::populate_protocol(std::string_view *result, int size) const { @@ -311,6 +315,15 @@ Http2ClientSession::get_proxy_session() return this; } +void +Http2ClientSession::set_no_activity_timeout() +{ + // Only set if not previously set + if (this->_vc->get_inactivity_timeout() == 0) { + this->set_inactivity_timeout(HRTIME_SECONDS(Http2::no_activity_timeout_in)); + } +} + HTTPVersion Http2ClientSession::get_version(HTTPHdr &hdr) const { diff --git a/proxy/http2/Http2ClientSession.h b/proxy/http2/Http2ClientSession.h index cd4cf0df503..dc1336face0 100644 --- a/proxy/http2/Http2ClientSession.h +++ b/proxy/http2/Http2ClientSession.h @@ -33,8 +33,7 @@ class Http2ClientSession : public ProxySession, public Http2CommonSession { public: - using super = ProxySession; ///< Parent type. - using SessionHandler = int (Http2ClientSession::*)(int, void *); + using super = ProxySession; ///< Parent type. Http2ClientSession(); @@ -63,6 +62,8 @@ class Http2ClientSession : public ProxySession, public Http2CommonSession void increment_current_active_connections_stat() override; void decrement_current_active_connections_stat() override; + void set_no_activity_timeout() override; + ProxySession *get_proxy_session() override; // noncopyable diff --git a/proxy/http2/Http2CommonSession.cc b/proxy/http2/Http2CommonSession.cc index 55974b5b0fc..729657457d3 100644 --- a/proxy/http2/Http2CommonSession.cc +++ b/proxy/http2/Http2CommonSession.cc @@ -91,6 +91,7 @@ Http2CommonSession::common_free(ProxySession *ssn) ink_hrtime_to_msec(this->_milestones[Http2SsnMilestone::OPEN]), this->_milestones.difference_sec(Http2SsnMilestone::OPEN, Http2SsnMilestone::CLOSE)); } + // Update stats on how we died. May want to eliminate this. Was useful for // tracking down which cases we were having problems cleaning up. But for general // use probably not worth the effort @@ -147,6 +148,20 @@ Http2CommonSession::set_half_close_local_flag(bool flag) half_close_local = flag; } +void +Http2CommonSession::add_url_to_pushed_table(const char *url, int url_len) +{ + // Delay std::unordered_set allocation until when it used + if (_h2_pushed_urls == nullptr) { + this->_h2_pushed_urls = new std::unordered_set(); + this->_h2_pushed_urls->reserve(Http2::push_diary_size); + } + + if (_h2_pushed_urls->size() < Http2::push_diary_size) { + _h2_pushed_urls->emplace(url); + } +} + int64_t Http2CommonSession::xmit(const Http2TxFrame &frame, bool flush) { @@ -172,12 +187,30 @@ void Http2CommonSession::flush() { if (this->_pending_sending_data_size > 0) { + total_write_len += this->_pending_sending_data_size; this->_pending_sending_data_size = 0; this->_write_buffer_last_flush = Thread::get_hrtime(); write_reenable(); } } +void +Http2CommonSession::write_reenable() +{ + if (write_vio) { + // Grab the lock for the write_vio. Holding the lock is + // checked eventually via the reenable logic + SCOPED_MUTEX_LOCK(lock, write_vio->mutex, this_ethread()); + write_vio->reenable(); + } +} + +int64_t +Http2CommonSession::write_avail() +{ + return this->write_buffer->write_avail(); +} + int Http2CommonSession::state_read_connection_preface(int event, void *edata) { @@ -268,13 +301,13 @@ Http2CommonSession::do_start_frame_read(Http2ErrorCode &ret_error) this->_read_buffer_reader->consume(nbytes); - if (!http2_frame_header_is_valid(this->current_hdr, this->connection_state.server_settings.get(HTTP2_SETTINGS_MAX_FRAME_SIZE))) { + if (!http2_frame_header_is_valid(this->current_hdr, this->connection_state.local_settings.get(HTTP2_SETTINGS_MAX_FRAME_SIZE))) { ret_error = Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR; return -1; } // If we know up front that the payload is too long, nuke this connection. - if (this->current_hdr.length > this->connection_state.server_settings.get(HTTP2_SETTINGS_MAX_FRAME_SIZE)) { + if (this->current_hdr.length > this->connection_state.local_settings.get(HTTP2_SETTINGS_MAX_FRAME_SIZE)) { ret_error = Http2ErrorCode::HTTP2_ERROR_FRAME_SIZE_ERROR; return -1; } @@ -341,11 +374,15 @@ Http2CommonSession::do_complete_frame_read() int Http2CommonSession::do_process_frame_read(int event, VIO *vio, bool inside_frame) { + Http2SsnDebug("do_process_frame_read %" PRId64 " bytes ready", this->_read_buffer_reader->read_avail()); + if (inside_frame) { do_complete_frame_read(); } while (this->_read_buffer_reader->read_avail() >= static_cast(HTTP2_FRAME_HEADER_LEN)) { + Http2SsnDebug("do_process_frame_read try to read frame %" PRId64 " bytes ready", this->_read_buffer_reader->read_avail()); + // Cancel reading if there was an error or connection is closed if (connection_state.tx_error_code.code != static_cast(Http2ErrorCode::HTTP2_ERROR_NO_ERROR) || connection_state.is_state_closed()) { @@ -354,10 +391,11 @@ Http2CommonSession::do_process_frame_read(int event, VIO *vio, bool inside_frame } Http2ErrorCode err = Http2ErrorCode::HTTP2_ERROR_NO_ERROR; - if (this->connection_state.get_stream_error_rate() > std::min(1.0, Http2::stream_error_rate_threshold * 2.0)) { + if (this->connection_state.get_stream_error_rate() > std::min(1.0, Http2::stream_error_rate_threshold * 2.0) && + !this->is_outbound()) { ip_port_text_buffer ipb; const char *client_ip = ats_ip_ntop(this->get_proxy_session()->get_remote_addr(), ipb, sizeof(ipb)); - SiteThrottledWarning("HTTP/2 session error client_ip=%s session_id=%" PRId64 + SiteThrottledWarning("HTTP/2 session error peer_ip=%s session_id=%" PRId64 " closing a connection, because its stream error rate (%f) exceeded the threshold (%f)", client_ip, this->get_connection_id(), this->connection_state.get_stream_error_rate(), Http2::stream_error_rate_threshold); @@ -403,32 +441,17 @@ Http2CommonSession::do_process_frame_read(int event, VIO *vio, bool inside_frame bool Http2CommonSession::_should_do_something_else() { - // Do something else every 128 incoming frames if connection state isn't closed - return (this->_n_frame_read & 0x7F) == 0 && !connection_state.is_state_closed(); -} - -int64_t -Http2CommonSession::write_avail() -{ - return this->write_buffer->write_avail(); + // Do something else every 128 incoming frames + return (this->_n_frame_read & 0x7F) == 0; } void -Http2CommonSession::write_reenable() +Http2CommonSession::add_session() { - write_vio->reenable(); } -void -Http2CommonSession::add_url_to_pushed_table(const char *url, int url_len) +bool +Http2CommonSession::is_outbound() const { - // Delay std::unordered_set allocation until when it used - if (_h2_pushed_urls == nullptr) { - this->_h2_pushed_urls = new std::unordered_set(); - this->_h2_pushed_urls->reserve(Http2::push_diary_size); - } - - if (_h2_pushed_urls->size() < Http2::push_diary_size) { - _h2_pushed_urls->emplace(url); - } + return false; } diff --git a/proxy/http2/Http2CommonSession.h b/proxy/http2/Http2CommonSession.h index 0961054fc21..d60beb09b45 100644 --- a/proxy/http2/Http2CommonSession.h +++ b/proxy/http2/Http2CommonSession.h @@ -108,6 +108,10 @@ class Http2CommonSession // Variables Http2ConnectionState connection_state; + virtual void add_session(); + virtual bool is_outbound() const; + virtual void set_no_activity_timeout() = 0; + protected: // SessionHandler(s) - state of reading frame int state_read_connection_preface(int, void *); @@ -133,6 +137,8 @@ class Http2CommonSession MIOBuffer *write_buffer = nullptr; IOBufferReader *_write_buffer_reader = nullptr; + int64_t total_write_len = 0; + Http2FrameHeader current_hdr = {0, 0, 0, 0}; uint32_t _write_size_threshold = 0; uint32_t _write_time_threshold = 100; diff --git a/proxy/http2/Http2ConnectionState.cc b/proxy/http2/Http2ConnectionState.cc index 75e85889cae..15865ab89d2 100644 --- a/proxy/http2/Http2ConnectionState.cc +++ b/proxy/http2/Http2ConnectionState.cc @@ -1,6 +1,6 @@ /** @file - Http2ConnectionState. + Http2ConnectionState.cc @section license License @@ -24,6 +24,7 @@ #include "P_Net.h" #include "Http2ConnectionState.h" #include "Http2ClientSession.h" +#include "Http2ServerSession.h" #include "Http2Stream.h" #include "Http2Frame.h" #include "Http2DebugNames.h" @@ -42,12 +43,10 @@ } \ } -#define Http2ConDebug(session, fmt, ...) \ - SsnDebug(session->get_proxy_session(), "http2_con", "[%" PRId64 "] " fmt, session->get_connection_id(), ##__VA_ARGS__); +#define Http2ConDebug(session, fmt, ...) Debug("http2_con", "[%" PRId64 "] " fmt, session->get_connection_id(), ##__VA_ARGS__); -#define Http2StreamDebug(session, stream_id, fmt, ...) \ - SsnDebug(session->get_proxy_session(), "http2_con", "[%" PRId64 "] [%u] " fmt, session->get_connection_id(), stream_id, \ - ##__VA_ARGS__); +#define Http2StreamDebug(session, stream_id, fmt, ...) \ + Debug("http2_con", "[%" PRId64 "] [%u] " fmt, session->get_connection_id(), stream_id, ##__VA_ARGS__); using http2_frame_dispatch = Http2Error (*)(Http2ConnectionState &, const Http2Frame &); @@ -105,13 +104,24 @@ rcv_data_frame(Http2ConnectionState &cstate, const Http2Frame &frame) if (stream == nullptr) { if (cstate.is_valid_streamid(id)) { // This error occurs fairly often, and is probably innocuous (SM initiates the shutdown) - return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_STREAM_CLOSED, nullptr); + cstate.send_rst_stream_frame(id, Http2ErrorCode::HTTP2_ERROR_NO_ERROR); + if (cstate.session->is_outbound()) { + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); + } else { + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_STREAM_CLOSED, nullptr); + } } else { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR, "recv data stream freed with invalid id"); } } + if (stream->get_state() == Http2StreamState::HTTP2_STREAM_STATE_CLOSED || + stream->get_state() == Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_REMOTE) { + cstate.send_rst_stream_frame(id, Http2ErrorCode::HTTP2_ERROR_STREAM_CLOSED); + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); + } + // If a DATA frame is received whose stream is not in "open" or "half closed // (local)" state, // the recipient MUST respond with a stream error of type STREAM_CLOSED. @@ -147,14 +157,19 @@ rcv_data_frame(Http2ConnectionState &cstate, const Http2Frame &frame) // Pure END_STREAM if (payload_length == 0) { - stream->signal_read_event(VC_EVENT_READ_COMPLETE); + if (stream->read_enabled()) { + stream->signal_read_event(VC_EVENT_READ_COMPLETE); + } return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); } } else { - // If payload length is 0 without END_STREAM flag, do nothing - if (payload_length == 0) { - return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); - } + // Any headers that show up after we received data are by definition trailing headers + stream->set_trailing_header_is_possible(); + } + + // If payload length is 0 without END_STREAM flag, do nothing + if (payload_length == 0 && !stream->recv_end_stream) { + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); } // Check whether Window Size is acceptable @@ -172,7 +187,7 @@ rcv_data_frame(Http2ConnectionState &cstate, const Http2Frame &frame) stream->decrement_server_rwnd(payload_length); if (is_debug_tag_set("http2_con")) { - uint32_t rwnd = cstate.server_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE); + uint32_t rwnd = cstate.local_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE); Http2StreamDebug(cstate.session, id, "Received DATA frame: rwnd con=%zd/%" PRId32 " stream=%zd/%" PRId32, cstate.server_rwnd(), rwnd, stream->server_rwnd(), rwnd); } @@ -180,7 +195,7 @@ rcv_data_frame(Http2ConnectionState &cstate, const Http2Frame &frame) const uint32_t unpadded_length = payload_length - pad_length; MIOBuffer *writer = stream->read_vio_writer(); if (writer == nullptr) { - return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_INTERNAL_ERROR); + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_INTERNAL_ERROR, "no writer"); } // If we call write() multiple times, we must keep the same reader, so we can @@ -200,17 +215,24 @@ rcv_data_frame(Http2ConnectionState &cstate, const Http2Frame &frame) unsigned int num_written = writer->write(myreader, read_len); if (num_written != read_len) { myreader->writer()->dealloc_reader(myreader); - return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_INTERNAL_ERROR); + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_INTERNAL_ERROR, "Write mismatch"); } myreader->consume(num_written); + stream->read_update(num_written); } myreader->writer()->dealloc_reader(myreader); if (frame.header().flags & HTTP2_FLAGS_DATA_END_STREAM) { // TODO: set total written size to read_vio.nbytes - stream->signal_read_event(VC_EVENT_READ_COMPLETE); - } else { - stream->signal_read_event(VC_EVENT_READ_READY); + stream->read_done(); + } + + if (stream->read_enabled()) { + if (frame.header().flags & HTTP2_FLAGS_DATA_END_STREAM) { + stream->signal_read_event(VC_EVENT_READ_COMPLETE); + } else { + stream->signal_read_event(VC_EVENT_READ_READY); + } } return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); @@ -238,28 +260,47 @@ rcv_headers_frame(Http2ConnectionState &cstate, const Http2Frame &frame) "recv headers bad client id"); } - Http2Stream *stream = nullptr; - bool new_stream = false; + Http2Stream *stream = nullptr; + bool new_stream = false; + bool reset_header_after_decoding = false; + bool free_stream_after_decoding = false; if (cstate.is_valid_streamid(stream_id)) { stream = cstate.find_stream(stream_id); - if (stream == nullptr) { - return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_STREAM_CLOSED, - "recv headers cannot find existing stream_id"); - } else if (stream->get_state() == Http2StreamState::HTTP2_STREAM_STATE_CLOSED) { + if (!cstate.session->is_outbound() && (stream == nullptr || !stream->trailing_header_is_possible())) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_STREAM_CLOSED, - "recv_header to closed stream"); - } else if (!stream->has_trailing_header()) { - return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR, "stream not expecting trailer header"); + } else if (stream == nullptr || stream->get_state() == Http2StreamState::HTTP2_STREAM_STATE_CLOSED) { + if (cstate.session->is_outbound()) { + reset_header_after_decoding = true; + // return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); + // return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_STREAM_CLOSED, + // "recv_header to closed stream"); + } else { + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_STREAM_CLOSED, + "recv_header to closed stream"); + } } - } else { - // Create new stream - Http2Error error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); - stream = cstate.create_stream(stream_id, error); - new_stream = true; - if (!stream) { - return error; + } + + if (!http2_is_client_streamid(stream_id)) { + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR, + "recv headers bad client id"); + } + + if (!stream) { + if (reset_header_after_decoding) { + free_stream_after_decoding = true; + stream = THREAD_ALLOC_INIT(http2StreamAllocator, this_ethread(), cstate.session->get_proxy_session(), stream_id, + cstate.peer_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE), true, false); + } else { + // Create new stream + Http2Error error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); + stream = cstate.create_stream(stream_id, error); + new_stream = true; + if (!stream) { + return error; + } } } @@ -351,27 +392,36 @@ rcv_headers_frame(Http2ConnectionState &cstate, const Http2Frame &frame) if (frame.header().flags & HTTP2_FLAGS_HEADERS_END_HEADERS) { // NOTE: If there are END_HEADERS flag, decode stored Header Blocks. - if (!stream->change_state(HTTP2_FRAME_TYPE_HEADERS, frame.header().flags) && stream->has_trailing_header() == false) { + if (!stream->change_state(HTTP2_FRAME_TYPE_HEADERS, frame.header().flags)) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR, "recv headers end headers and not trailing header"); } - bool empty_request = false; - if (stream->has_trailing_header()) { + if (stream->trailing_header_is_possible()) { if (!(frame.header().flags & HTTP2_FLAGS_HEADERS_END_STREAM)) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR, "recv headers tailing header without endstream"); } - // If the flag has already been set before decoding header blocks, this is the trailing header. - // Set a flag to avoid initializing fetcher for now. - // Decoding header blocks is still needed to maintain a HPACK dynamic table. - // TODO: TS-3812 - empty_request = true; } - stream->mark_milestone(Http2StreamMilestone::START_DECODE_HEADERS); + if (stream->trailing_header_is_possible()) { + stream->reset_recv_headers(); + } else { + stream->mark_milestone(Http2StreamMilestone::START_DECODE_HEADERS); + } Http2ErrorCode result = - stream->decode_header_blocks(*cstate.local_hpack_handle, cstate.server_settings.get(HTTP2_SETTINGS_HEADER_TABLE_SIZE)); + stream->decode_header_blocks(*cstate.local_hpack_handle, cstate.local_settings.get(HTTP2_SETTINGS_HEADER_TABLE_SIZE)); + + // If this was an outbound connection and the state was already closed, just clear the + // headers after processing. We just processed the heaer blocks to keep the dynamic table in + // sync with peer to avoid future HPACK compression errors + if (reset_header_after_decoding) { + stream->reset_recv_headers(); + if (free_stream_after_decoding) { + THREAD_FREE(stream, http2StreamAllocator, this_ethread()); + } + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); + } if (result != Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { if (result == Http2ErrorCode::HTTP2_ERROR_COMPRESSION_ERROR) { @@ -387,15 +437,22 @@ rcv_headers_frame(Http2ConnectionState &cstate, const Http2Frame &frame) } // Set up the State Machine - if (!empty_request) { + if (!stream->is_outbound_connection() && !stream->trailing_header_is_possible()) { SCOPED_MUTEX_LOCK(stream_lock, stream->mutex, this_ethread()); stream->mark_milestone(Http2StreamMilestone::START_TXN); stream->new_transaction(frame.is_from_early_data()); // Send request header to SM stream->send_request(cstate); } else { - // Signal VC_EVENT_READ_COMPLETE because received trailing header fields with END_STREAM flag - stream->signal_read_event(VC_EVENT_READ_COMPLETE); + // If this is a trailer, first signal to the SM that the body is done + if (stream->trailing_header_is_possible()) { + stream->set_expect_receive_trailer(); + // Propagate the trailer header + stream->send_request(cstate); + } else { + // Propagate the response + stream->send_request(cstate); + } } } else { // NOTE: Expect CONTINUATION Frame. Do NOT change state of stream or decode @@ -569,7 +626,7 @@ rcv_settings_frame(Http2ConnectionState &cstate, const Http2Frame &frame) Warning("Setting frame for zombied session %" PRId64, cstate.session->get_connection_id()); } - // Update SETTIGNS frame count per minute + // Update SETTINGS frame count per minute cstate.increment_received_settings_frame_count(); // Close this connection if its SETTINGS frame count exceeds a limit if (cstate.get_received_settings_frame_count() > Http2::max_settings_frames_per_minute) { @@ -645,7 +702,7 @@ rcv_settings_frame(Http2ConnectionState &cstate, const Http2Frame &frame) cstate.update_initial_rwnd(param.value); } - cstate.client_settings.set(static_cast(param.id), param.value); + cstate.peer_settings.set(static_cast(param.id), param.value); ++n_settings; } @@ -663,8 +720,8 @@ rcv_settings_frame(Http2ConnectionState &cstate, const Http2Frame &frame) // [RFC 7540] 6.5. Once all values have been applied, the recipient MUST // immediately emit a SETTINGS frame with the ACK flag set. Http2SettingsFrame ack_frame(0, HTTP2_FLAGS_SETTINGS_ACK); + Http2StreamDebug(cstate.session, stream_id, "Send Setting ACK"); cstate.session->xmit(ack_frame); - return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); } @@ -757,6 +814,14 @@ rcv_goaway_frame(Http2ConnectionState &cstate, const Http2Frame &frame) static_cast(goaway.error_code)); cstate.rx_error_code = {ProxyErrorClass::SSN, static_cast(goaway.error_code)}; + + if (static_cast(goaway.error_code) != 0) { + ip_port_text_buffer ipb; + const char *client_ip = ats_ip_ntop(cstate.session->get_proxy_session()->get_remote_addr(), ipb, sizeof(ipb)); + Warning("HTTP/2 rcv GOAWAY error code=0x%02x %s_ip=%s session_id=%" PRId64, static_cast(goaway.error_code), + cstate.session->is_outbound() ? "server" : "client", client_ip, cstate.session->get_connection_id()); + } + cstate.session->get_proxy_session()->do_io_close(); return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); @@ -845,11 +910,11 @@ rcv_window_update_frame(Http2ConnectionState &cstate, const Http2Frame &frame) auto error = stream->increment_client_rwnd(size); if (error != Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { - return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, error); + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, error, "Bad stream rwnd"); } ssize_t wnd = std::min(cstate.client_rwnd(), stream->client_rwnd()); - if (!stream->is_closed() && stream->get_state() == Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_REMOTE && wnd > 0) { + if (wnd > 0) { SCOPED_MUTEX_LOCK(lock, stream->mutex, this_ethread()); stream->restart_sending(); } @@ -929,7 +994,7 @@ rcv_continuation_frame(Http2ConnectionState &cstate, const Http2Frame &frame) } Http2ErrorCode result = - stream->decode_header_blocks(*cstate.local_hpack_handle, cstate.server_settings.get(HTTP2_SETTINGS_HEADER_TABLE_SIZE)); + stream->decode_header_blocks(*cstate.local_hpack_handle, cstate.local_settings.get(HTTP2_SETTINGS_HEADER_TABLE_SIZE)); if (result != Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { if (result == Http2ErrorCode::HTTP2_ERROR_COMPRESSION_ERROR) { @@ -1043,7 +1108,7 @@ void Http2ConnectionState::init(Http2CommonSession *ssn) { session = ssn; - this->_server_rwnd = Http2::initial_window_size; + this->_server_rwnd = Http2::session_initial_window_size; local_hpack_handle = new HpackHandle(HTTP2_HEADER_TABLE_SIZE); remote_hpack_handle = new HpackHandle(HTTP2_HEADER_TABLE_SIZE); @@ -1061,9 +1126,10 @@ Http2ConnectionState::init(Http2CommonSession *ssn) The client connection preface is HTTP2_CONNECTION_PREFACE. The server connection preface consists of a potentially empty SETTINGS frame. - Details in [RFC 7540] 3.5. HTTP/2 Connection Preface - - TODO: send client connection preface if the connection is outbound + [RFC 7540] 3.5. HTTP/2 Connection Preface. Upon establishment of a TCP connection and + determination that HTTP/2 will be used by both peers, each endpoint MUST + send a connection preface as a final confirmation ... The server connection + preface consists of a potentially empty SETTINGS frame. */ void Http2ConnectionState::send_connection_preface() @@ -1076,8 +1142,10 @@ Http2ConnectionState::send_connection_preface() send_settings_frame(configured_settings); - if (server_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE) > HTTP2_INITIAL_WINDOW_SIZE) { - send_window_update_frame(0, server_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE) - HTTP2_INITIAL_WINDOW_SIZE); + // If the session window size is non-default, send a window update right away + + if (Http2::session_initial_window_size > HTTP2_INITIAL_WINDOW_SIZE) { + send_window_update_frame(0, Http2::session_initial_window_size - HTTP2_INITIAL_WINDOW_SIZE); } } @@ -1160,7 +1228,7 @@ Http2ConnectionState::rcv_frame(const Http2Frame *frame) if (error.cls == Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION) { if (error.msg) { Error("HTTP/2 connection error code=0x%02x client_ip=%s session_id=%" PRId64 " stream_id=%u %s", - static_cast(error.code), client_ip, session->get_connection_id(), stream_id, error.msg); + static_cast(error.code), client_ip, session->get_proxy_session()->connection_id(), stream_id, error.msg); } this->send_goaway_frame(this->latest_streamid_in, error.code); this->session->set_half_close_local_flag(true); @@ -1173,7 +1241,7 @@ Http2ConnectionState::rcv_frame(const Http2Frame *frame) } else if (error.cls == Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM) { if (error.msg) { Error("HTTP/2 stream error code=0x%02x client_ip=%s session_id=%" PRId64 " stream_id=%u %s", static_cast(error.code), - client_ip, session->get_connection_id(), stream_id, error.msg); + client_ip, session->get_proxy_session()->connection_id(), stream_id, error.msg); } this->send_rst_stream_frame(stream_id, error.code); } @@ -1258,7 +1326,6 @@ Http2ConnectionState::main_event_handler(int event, void *edata) } } } - return 0; } @@ -1278,8 +1345,114 @@ Http2ConnectionState::state_closed(int event, void *edata) return 0; } +bool +Http2ConnectionState::is_peer_concurrent_stream_ub() const +{ + return client_streams_in_count >= (peer_settings.get(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS)) * 0.9; +} + +bool +Http2ConnectionState::is_peer_concurrent_stream_lb() const +{ + return client_streams_in_count <= (peer_settings.get(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS)) / 2; +} + +void +Http2ConnectionState::set_stream_id(Http2Stream *stream) +{ + if (stream->get_transaction_id() < 0) { + Http2StreamId stream_id = (latest_streamid_in == 0) ? 3 : latest_streamid_in + 2; + stream->set_transaction_id(stream_id); + latest_streamid_in = stream_id; + } +} + +Http2Stream * +Http2ConnectionState::create_initiating_stream(bool client_streamid, Http2Error &error) +{ + // first check if we've hit the active connection limit + if (!session->get_netvc()->add_to_active_queue()) { + error = Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_NO_ERROR, + "refused to create new stream, maxed out active connections"); + return nullptr; + } + + // In half_close state, TS doesn't create new stream. Because GOAWAY frame is sent to client + if (session->get_half_close_local_flag()) { + error = Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_REFUSED_STREAM, + "refused to create new stream, because session is in half_close state"); + return nullptr; + } + + // Endpoints MUST NOT exceed the limit set by their peer. An endpoint + // that receives a HEADERS frame that causes their advertised concurrent + // stream limit to be exceeded MUST treat this as a stream error. + int check_max_concurrent_limit; + int check_count; + if (client_streamid) { + check_count = client_streams_in_count; + // If this is an outbound client stream, must check against the peer's max_concurrent + if (session->is_outbound()) { + check_max_concurrent_limit = peer_settings.get(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS); + } else { // Inbound client streamm check against our own max_connecurent limits + check_max_concurrent_limit = local_settings.get(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS); + } + } else { // Not a client stream (i.e. a push) + check_count = client_streams_out_count; + // If this is an outbound non-client stream, must check against the local max_concurrent + if (session->is_outbound()) { + check_max_concurrent_limit = local_settings.get(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS); + } else { // Inbound non-client streamm check against the peer's max_connecurent limits + check_max_concurrent_limit = peer_settings.get(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS); + } + } + ink_release_assert(check_max_concurrent_limit != 0); + // If we haven't got the peers settings yet, just hope for the best + if (check_max_concurrent_limit >= 0) { + if (session->is_outbound() && Http2ConnectionState::is_peer_concurrent_stream_ub()) { + error = Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_REFUSED_STREAM, + "recv headers creating stream beyond max_concurrent limit"); + return nullptr; + } else if (check_count >= check_max_concurrent_limit) { + error = Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_REFUSED_STREAM, + "recv headers creating stream beyond max_concurrent limit"); + return nullptr; + } + } + + Http2Stream *new_stream = THREAD_ALLOC_INIT(http2StreamAllocator, this_ethread(), session->get_proxy_session(), -1, + peer_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE), true, true); + + ink_assert(nullptr != new_stream); + !stream_list.in(new_stream); + + stream_list.enqueue(new_stream); + if (client_streamid) { + ink_assert(client_streams_in_count < UINT32_MAX); + ++client_streams_in_count; + } else { + ink_assert(client_streams_out_count < UINT32_MAX); + ++client_streams_out_count; + } + ++total_client_streams_count; + + if (zombie_event != nullptr) { + zombie_event->cancel(); + zombie_event = nullptr; + } + + new_stream->mutex = new_ProxyMutex(); + new_stream->is_first_transaction_flag = get_stream_requests() == 0; + increment_stream_requests(); + + // Clear the session timeout. Let the transaction timeouts reign + session->get_proxy_session()->cancel_inactivity_timeout(); + + return new_stream; +} + Http2Stream * -Http2ConnectionState::create_stream(Http2StreamId new_id, Http2Error &error) +Http2ConnectionState::create_stream(Http2StreamId new_id, Http2Error &error, bool initiating_connection) { // first check if we've hit the active connection limit if (!session->get_netvc()->add_to_active_queue()) { @@ -1320,22 +1493,34 @@ Http2ConnectionState::create_stream(Http2StreamId new_id, Http2Error &error) // Endpoints MUST NOT exceed the limit set by their peer. An endpoint // that receives a HEADERS frame that causes their advertised concurrent // stream limit to be exceeded MUST treat this as a stream error. + int check_max_concurrent_limit; + int check_count; if (client_streamid) { - if (client_streams_in_count >= server_settings.get(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS)) { - error = Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_REFUSED_STREAM, - "recv headers creating inbound stream beyond max_concurrent limit"); - return nullptr; - } - } else { - if (client_streams_out_count >= client_settings.get(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS)) { - error = Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_REFUSED_STREAM, - "recv headers creating outbound stream beyond max_concurrent limit"); - return nullptr; - } + check_count = client_streams_in_count; + // If this is an outbound client stream, must check against the peer's max_concurrent + if (session->is_outbound()) { + check_max_concurrent_limit = peer_settings.get(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS); + } else { // Inbound client streamm check against our own max_connecurent limits + check_max_concurrent_limit = local_settings.get(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS); + } + } else { // Not a client stream (i.e. a push) + check_count = client_streams_out_count; + // If this is an outbound non-client stream, must check against the local max_concurrent + if (session->is_outbound()) { + check_max_concurrent_limit = local_settings.get(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS); + } else { // Inbound non-client streamm check against the peer's max_connecurent limits + check_max_concurrent_limit = peer_settings.get(HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS); + } + } + // If we haven't got the peers settings yet, just hope for the best + if (check_max_concurrent_limit >= 0 && check_count >= check_max_concurrent_limit) { + error = Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_REFUSED_STREAM, + "recv headers creating stream beyond max_concurrent limit"); + return nullptr; } Http2Stream *new_stream = THREAD_ALLOC_INIT(http2StreamAllocator, this_ethread(), session->get_proxy_session(), new_id, - client_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE)); + peer_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE), initiating_connection, true); ink_assert(nullptr != new_stream); ink_assert(!stream_list.in(new_stream)); @@ -1361,6 +1546,9 @@ Http2ConnectionState::create_stream(Http2StreamId new_id, Http2Error &error) new_stream->is_first_transaction_flag = get_stream_requests() == 0; increment_stream_requests(); + // Clear the session timeout. Let the transaction timeouts reign + session->get_proxy_session()->cancel_inactivity_timeout(); + return new_stream; } @@ -1376,6 +1564,17 @@ Http2ConnectionState::find_stream(Http2StreamId id) const return nullptr; } +void +Http2ConnectionState::start_streams() +{ + Http2Stream *s = stream_list.head; + while (s) { + Http2Stream *next = static_cast(s->link.next); + s->reenable_write(); + s = next; + } +} + void Http2ConnectionState::restart_streams() { @@ -1397,16 +1596,14 @@ Http2ConnectionState::restart_streams() // Call send_response_body() for each streams while (s != end) { Http2Stream *next = static_cast(s->link.next ? s->link.next : stream_list.head); - if (!s->is_closed() && s->get_state() == Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_REMOTE && - std::min(this->client_rwnd(), s->client_rwnd()) > 0) { + if (std::min(this->client_rwnd(), s->client_rwnd()) > 0) { SCOPED_MUTEX_LOCK(lock, s->mutex, this_ethread()); s->restart_sending(); } ink_assert(s != next); s = next; } - if (!s->is_closed() && s->get_state() == Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_REMOTE && - std::min(this->client_rwnd(), s->client_rwnd()) > 0) { + if (std::min(this->client_rwnd(), s->client_rwnd()) > 0) { SCOPED_MUTEX_LOCK(lock, s->mutex, this_ethread()); s->restart_sending(); } @@ -1418,11 +1615,11 @@ Http2ConnectionState::restart_streams() void Http2ConnectionState::restart_receiving(Http2Stream *stream) { - uint32_t initial_rwnd = this->server_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE); - uint32_t min_rwnd = std::min(initial_rwnd, this->server_settings.get(HTTP2_SETTINGS_MAX_FRAME_SIZE)); + uint32_t initial_rwnd = Http2::session_initial_window_size; + uint32_t min_rwnd = std::min(initial_rwnd, this->local_settings.get(HTTP2_SETTINGS_MAX_FRAME_SIZE)); // Connection level WINDOW UPDATE - if (this->server_rwnd() < min_rwnd) { + if (this->server_rwnd() < initial_rwnd) { Http2WindowSize diff_size = initial_rwnd - this->server_rwnd(); this->increment_server_rwnd(diff_size); this->send_window_update_frame(0, diff_size); @@ -1439,6 +1636,8 @@ Http2ConnectionState::restart_receiving(Http2Stream *stream) return; } + // Update the window size for the stream + initial_rwnd = Http2::initial_window_size; Http2WindowSize diff_size = initial_rwnd - std::max(static_cast(stream->server_rwnd()), data_size); stream->increment_server_rwnd(diff_size); this->send_window_update_frame(stream->get_id(), diff_size); @@ -1510,6 +1709,9 @@ Http2ConnectionState::delete_stream(Http2Stream *stream) if (http2_is_client_streamid(stream->get_id())) { ink_assert(client_streams_in_count > 0); --client_streams_in_count; + if (!fini_received && is_peer_concurrent_stream_lb()) { + session->add_session(); + } } else { ink_assert(client_streams_out_count > 0); --client_streams_out_count; @@ -1543,10 +1745,14 @@ Http2ConnectionState::release_stream() // Can't do this because we just destroyed right here ^, // or we can use a local variable to do it. // session = nullptr; + } else if (shutdown_state == HTTP2_SHUTDOWN_IN_PROGRESS && fini_event == nullptr) { + session->do_clear_session_active(); + fini_event = this_ethread()->schedule_imm_local((Continuation *)this, HTTP2_SESSION_EVENT_FINI); } else if (session->get_proxy_session()->is_active()) { // If the number of clients is 0, HTTP2_SESSION_EVENT_FINI is not received or sent, and session is active, // then mark the connection as inactive session->do_clear_session_active(); + session->set_no_activity_timeout(); UnixNetVConnection *vc = static_cast(session->get_netvc()); if (vc && vc->active_timeout_in == 0) { // With heavy traffic, session could be destroyed. Do not touch session after this. @@ -1567,7 +1773,7 @@ Http2ConnectionState::update_initial_rwnd(Http2WindowSize new_size) // Update stream level window sizes for (Http2Stream *s = stream_list.head; s; s = static_cast(s->link.next)) { SCOPED_MUTEX_LOCK(lock, s->mutex, this_ethread()); - s->update_initial_rwnd(new_size - (client_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE) - s->client_rwnd())); + s->update_initial_rwnd(new_size - (peer_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE) - s->client_rwnd())); } } @@ -1616,7 +1822,7 @@ Http2ConnectionState::send_data_frames_depends_on_priority() dependency_tree->update(node, len); SCOPED_MUTEX_LOCK(stream_lock, stream->mutex, this_ethread()); - stream->signal_write_event(true); + stream->signal_write_event(stream->is_write_vio_done() ? VC_EVENT_WRITE_COMPLETE : VC_EVENT_WRITE_READY); } break; } @@ -1644,7 +1850,7 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len payload_length = 0; uint8_t flags = 0x00; - IOBufferReader *resp_reader = stream->response_get_data_reader(); + IOBufferReader *resp_reader = stream->send_get_data_reader(); SCOPED_MUTEX_LOCK(stream_lock, stream->mutex, this_ethread()); @@ -1687,7 +1893,8 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len return Http2SendDataFrameResult::NO_PAYLOAD; } - if (stream->is_write_vio_done()) { + if (stream->is_write_vio_done() && !resp_reader->is_read_avail_more_than(payload_length)) { + Http2StreamDebug(this->session, stream->get_id(), "End of Data Frame"); flags |= HTTP2_FLAGS_DATA_END_STREAM; } @@ -1696,8 +1903,8 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len stream->decrement_client_rwnd(payload_length); // Create frame - Http2StreamDebug(session, stream->get_id(), "Send a DATA frame - client window con: %5zd stream: %5zd payload: %5zd", - _client_rwnd, stream->client_rwnd(), payload_length); + Http2StreamDebug(session, stream->get_id(), "Send a DATA frame - client window con: %5zd stream: %5zd payload: %5zd flags: 0x%x", + _client_rwnd, stream->client_rwnd(), payload_length, flags); Http2DataFrame data(stream->get_id(), flags, resp_reader, payload_length); this->session->xmit(data, flags & HTTP2_FLAGS_DATA_END_STREAM); @@ -1726,20 +1933,36 @@ Http2ConnectionState::send_data_frames(Http2Stream *stream) return; } + if (zombie_event != nullptr) { + zombie_event->cancel(); + zombie_event = nullptr; + } + size_t len = 0; Http2SendDataFrameResult result = Http2SendDataFrameResult::NO_ERROR; - while (result == Http2SendDataFrameResult::NO_ERROR) { - result = send_a_data_frame(stream, len); + bool more_data = true; + IOBufferReader *resp_reader = stream->send_get_data_reader(); + while (more_data && result == Http2SendDataFrameResult::NO_ERROR) { + result = send_a_data_frame(stream, len); + more_data = resp_reader->is_read_avail_more_than(0); if (result == Http2SendDataFrameResult::DONE) { - // Delete a stream immediately - // TODO its should not be deleted for a several time to handling - // RST_STREAM and WINDOW_UPDATE. - // See 'closed' state written at [RFC 7540] 5.1. - Http2StreamDebug(this->session, stream->get_id(), "Shutdown stream"); - stream->initiating_close(); + if (!stream->is_outbound_connection()) { + // Delete a stream immediately + // TODO its should not be deleted for a several time to handling + // RST_STREAM and WINDOW_UPDATE. + // See 'closed' state written at [RFC 7540] 5.1. + Http2StreamDebug(this->session, stream->get_id(), "Shutdown stream"); + stream->signal_write_event(VC_EVENT_WRITE_COMPLETE); + stream->do_io_close(); + } else if (stream->is_outbound_connection() && stream->is_write_vio_done()) { + stream->signal_write_event(VC_EVENT_WRITE_COMPLETE); + } } } + if (!more_data && result != Http2SendDataFrameResult::DONE) { + stream->signal_write_event(VC_EVENT_WRITE_READY); + } return; } @@ -1753,16 +1976,26 @@ Http2ConnectionState::send_headers_frame(Http2Stream *stream) Http2StreamDebug(session, stream->get_id(), "Send HEADERS frame"); - HTTPHdr *resp_hdr = &stream->response_header; - http2_convert_header_from_1_1_to_2(resp_hdr); + // For outbound streams, set the ID if it has not yet already been set + // Need to defer setting the stream ID to avoid another later created stream + // sending out first. This may cause the peer to issue a stream or connection + // error (new stream less that the greatest we have seen so far) + this->set_stream_id(stream); + + HTTPHdr *send_hdr = stream->get_send_header(); + if (stream->expect_send_trailer()) { + // Which is a no-op conversion + } else { + http2_convert_header_from_1_1_to_2(send_hdr); + } - uint32_t buf_len = resp_hdr->length_get() * 2; // Make it double just in case + uint32_t buf_len = send_hdr->length_get() * 2; // Make it double just in case ts::LocalBuffer local_buffer(buf_len); uint8_t *buf = local_buffer.data(); stream->mark_milestone(Http2StreamMilestone::START_ENCODE_HEADERS); - Http2ErrorCode result = http2_encode_header_blocks(resp_hdr, buf, buf_len, &header_blocks_size, *(this->remote_hpack_handle), - client_settings.get(HTTP2_SETTINGS_HEADER_TABLE_SIZE)); + Http2ErrorCode result = http2_encode_header_blocks(send_hdr, buf, buf_len, &header_blocks_size, *(this->remote_hpack_handle), + peer_settings.get(HTTP2_SETTINGS_HEADER_TABLE_SIZE)); if (result != Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { return; } @@ -1771,11 +2004,24 @@ Http2ConnectionState::send_headers_frame(Http2Stream *stream) if (header_blocks_size <= static_cast(BUFFER_SIZE_FOR_INDEX(buffer_size_index[HTTP2_FRAME_TYPE_HEADERS]))) { payload_length = header_blocks_size; flags |= HTTP2_FLAGS_HEADERS_END_HEADERS; - if ((resp_hdr->presence(MIME_PRESENCE_CONTENT_LENGTH) && resp_hdr->get_content_length() == 0) || - (!resp_hdr->expect_final_response() && stream->is_write_vio_done())) { - Http2StreamDebug(session, stream->get_id(), "END_STREAM"); - flags |= HTTP2_FLAGS_HEADERS_END_STREAM; - stream->send_end_stream = true; + if (stream->is_outbound_connection()) { // Will be sending a request_header + int method = send_hdr->method_get_wksidx(); + if (!(method == HTTP_WKSIDX_POST || method == HTTP_WKSIDX_PUSH || method == HTTP_WKSIDX_PUT) && + !send_hdr->presence(MIME_PRESENCE_TRANSFER_ENCODING) && + ((send_hdr->presence(MIME_PRESENCE_CONTENT_LENGTH) && send_hdr->get_content_length() == 0) || + !send_hdr->presence(MIME_PRESENCE_CONTENT_LENGTH))) { + // TODO deal with the chunked encoding case + Http2StreamDebug(session, stream->get_id(), "request END_STREAM"); + flags |= HTTP2_FLAGS_HEADERS_END_STREAM; + stream->send_end_stream = true; + } + } else { + if ((send_hdr->presence(MIME_PRESENCE_CONTENT_LENGTH) && send_hdr->get_content_length() == 0) || + (!send_hdr->expect_final_response() && stream->is_write_vio_done())) { + Http2StreamDebug(session, stream->get_id(), "response END_STREAM"); + flags |= HTTP2_FLAGS_HEADERS_END_STREAM; + stream->send_end_stream = true; + } } stream->mark_milestone(Http2StreamMilestone::START_TX_HEADERS_FRAMES); } else { @@ -1793,6 +2039,7 @@ Http2ConnectionState::send_headers_frame(Http2Stream *stream) return; } + Http2StreamDebug(session, stream->get_id(), "Send HEADERS frame flags: 0x%x length: %d", flags, payload_length); Http2HeadersFrame headers(stream->get_id(), flags, buf, payload_length); this->session->xmit(headers); uint64_t sent = payload_length; @@ -1821,7 +2068,7 @@ Http2ConnectionState::send_push_promise_frame(Http2Stream *stream, URL &url, con int payload_length = 0; uint8_t flags = 0x00; - if (client_settings.get(HTTP2_SETTINGS_ENABLE_PUSH) == 0) { + if (peer_settings.get(HTTP2_SETTINGS_ENABLE_PUSH) == 0) { return false; } @@ -1853,7 +2100,7 @@ Http2ConnectionState::send_push_promise_frame(Http2Stream *stream, URL &url, con uint8_t *buf = local_buffer.data(); Http2ErrorCode result = http2_encode_header_blocks(&hdr, buf, buf_len, &header_blocks_size, *(this->remote_hpack_handle), - client_settings.get(HTTP2_SETTINGS_HEADER_TABLE_SIZE)); + peer_settings.get(HTTP2_SETTINGS_HEADER_TABLE_SIZE)); if (result != Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { return false; } @@ -1912,7 +2159,7 @@ Http2ConnectionState::send_push_promise_frame(Http2Stream *stream, URL &url, con } } stream->change_state(HTTP2_FRAME_TYPE_PUSH_PROMISE, HTTP2_FLAGS_PUSH_PROMISE_END_HEADERS); - stream->set_request_headers(hdr); + stream->set_recv_headers(hdr); stream->new_transaction(); stream->recv_end_stream = true; // No more data with the request stream->send_request(*this); @@ -1964,18 +2211,18 @@ Http2ConnectionState::send_settings_frame(const Http2ConnectionSettings &new_set unsigned settings_value = new_settings.get(id); // Send only difference - if (settings_value != server_settings.get(id)) { + if (settings_value != local_settings.get(id)) { Http2StreamDebug(session, stream_id, " %s : %u", Http2DebugNames::get_settings_param_name(id), settings_value); params[params_size++] = {static_cast(id), settings_value}; // Update current settings - server_settings.set(id, new_settings.get(id)); + local_settings.set(id, new_settings.get(id)); } } Http2SettingsFrame settings(stream_id, HTTP2_FRAME_NO_FLAG, params, params_size); - this->session->xmit(settings); + this->session->xmit(settings, true); } void @@ -2115,12 +2362,13 @@ Http2ConnectionState::increment_client_rwnd(size_t amount) this->_recent_rwnd_increment[this->_recent_rwnd_increment_index] = amount; ++this->_recent_rwnd_increment_index; this->_recent_rwnd_increment_index %= this->_recent_rwnd_increment.size(); - double sum = std::accumulate(this->_recent_rwnd_increment.begin(), this->_recent_rwnd_increment.end(), 0.0); - double avg = sum / this->_recent_rwnd_increment.size(); - if (avg < Http2::min_avg_window_update) { - HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_INSUFFICIENT_AVG_WINDOW_UPDATE, this_ethread()); - return Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM; - } + // SKH Causing problems with gRPC processing. Python example resulted in amount 8 + // double sum = std::accumulate(this->_recent_rwnd_increment.begin(), this->_recent_rwnd_increment.end(), 0.0); + // double avg = sum / this->_recent_rwnd_increment.size(); + // if (avg < Http2::min_avg_window_update) { + // HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_INSUFFICIENT_AVG_WINDOW_UPDATE, this_ethread()); + // return Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM; + //} return Http2ErrorCode::HTTP2_ERROR_NO_ERROR; } diff --git a/proxy/http2/Http2ConnectionState.h b/proxy/http2/Http2ConnectionState.h index caf4a957aef..de3c0d71de4 100644 --- a/proxy/http2/Http2ConnectionState.h +++ b/proxy/http2/Http2ConnectionState.h @@ -1,6 +1,6 @@ /** @file - Http2ConnectionState. + Http2ConnectionState.h @section license License @@ -33,8 +33,8 @@ #include "Http2DependencyTree.h" #include "Http2FrequencyCounter.h" -class Http2CommonSession; class Http2Frame; +class Http2CommonSession; enum class Http2SendDataFrameResult { NO_ERROR = 0, @@ -87,8 +87,8 @@ class Http2ConnectionState : public Continuation ActivityCop _cop; // Settings. - Http2ConnectionSettings server_settings; - Http2ConnectionSettings client_settings; + Http2ConnectionSettings local_settings; + Http2ConnectionSettings peer_settings; void init(Http2CommonSession *ssn); void send_connection_preface(); @@ -100,9 +100,12 @@ class Http2ConnectionState : public Continuation int state_closed(int, void *); // Stream control interfaces - Http2Stream *create_stream(Http2StreamId new_id, Http2Error &error); + Http2Stream *create_stream(Http2StreamId new_id, Http2Error &error, bool initiating_connection = false); + Http2Stream *create_initiating_stream(bool client_request, Http2Error &error); + void set_stream_id(Http2Stream *stream); Http2Stream *find_stream(Http2StreamId id) const; void restart_streams(); + void start_streams(); bool delete_stream(Http2Stream *stream); void release_stream(); void cleanup_streams(); @@ -113,6 +116,8 @@ class Http2ConnectionState : public Continuation Http2StreamId get_latest_stream_id_out() const; int get_stream_requests() const; void increment_stream_requests(); + bool is_peer_concurrent_stream_ub() const; + bool is_peer_concurrent_stream_lb() const; // Continuated header decoding Http2StreamId get_continued_stream_id() const; @@ -163,6 +168,9 @@ class Http2ConnectionState : public Continuation Http2ErrorCode increment_server_rwnd(size_t amount); Http2ErrorCode decrement_server_rwnd(size_t amount); + bool no_streams() const; + bool single_stream() const; + private: unsigned _adjust_concurrent_stream(); diff --git a/proxy/http2/Http2ServerSession.cc b/proxy/http2/Http2ServerSession.cc new file mode 100644 index 00000000000..86936ec2ace --- /dev/null +++ b/proxy/http2/Http2ServerSession.cc @@ -0,0 +1,421 @@ +/** @file + + Http2ServerSession. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "Http2ServerSession.h" +#include "HttpDebugNames.h" +#include "tscore/ink_base64.h" +#include "Http2CommonSessionInternal.h" +#include "HttpSessionManager.h" +#include "P_SSLNetVConnection.h" + +ClassAllocator http2ServerSessionAllocator("http2ServerSessionAllocator"); + +static int +send_connection_event(Continuation *cont, int event, void *edata) +{ + SCOPED_MUTEX_LOCK(lock, cont->mutex, this_ethread()); + return cont->handleEvent(event, edata); +} + +Http2ServerSession::Http2ServerSession() = default; + +void +Http2ServerSession::destroy() +{ + if (!in_destroy) { + in_destroy = true; + write_vio = nullptr; + this->remove_session(); + this->release_outbound_connection_tracking(); + REMEMBER(NO_EVENT, this->recursion) + Http2SsnDebug("session destroy"); + if (_vc) { + _vc->do_io_close(); + _vc = nullptr; + } + free(); + } +} + +void +Http2ServerSession::free() +{ + auto mutex_thread = this->mutex->thread_holding; + if (Http2CommonSession::common_free(this)) { + HTTP2_DECREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_SERVER_SESSION_COUNT, mutex_thread); + THREAD_FREE(this, http2ServerSessionAllocator, mutex_thread); + } +} + +void +Http2ServerSession::start() +{ + SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); + + SET_HANDLER(&Http2ServerSession::main_event_handler); + HTTP2_SET_SESSION_HANDLER(&Http2ServerSession::state_start_frame_read); + + VIO *read_vio = this->do_io_read(this, INT64_MAX, this->read_buffer); + write_vio = this->do_io_write(this, INT64_MAX, this->_write_buffer_reader); + + this->connection_state.init(this); + + // 3.5 HTTP/2 Connection Preface. Upon establishment of a TCP connection and + // determination that HTTP/2 will be used by both peers, each endpoint MUST + // send a connection preface as a final confirmation ... + // This is the preface string sent by the client + this->write_buffer->write(HTTP2_CONNECTION_PREFACE, HTTP2_CONNECTION_PREFACE_LEN); + total_write_len += HTTP2_CONNECTION_PREFACE_LEN; + // write_vio->nbytes = total_write_len; + write_reenable(); + Http2SsnDebug("Sent Connection Preface"); + + this->connection_state.send_connection_preface(); + + this->handleEvent(VC_EVENT_READ_READY, read_vio); +} + +void +Http2ServerSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader) +{ + ink_assert(new_vc->mutex->thread_holding == this_ethread()); + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_SERVER_SESSION_COUNT, new_vc->mutex->thread_holding); + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_TOTAL_SERVER_CONNECTION_COUNT, new_vc->mutex->thread_holding); + this->_milestones.mark(Http2SsnMilestone::OPEN); + + // Unique client session identifier. + this->con_id = ProxySession::next_connection_id(); + this->_vc = new_vc; + _vc->set_inactivity_timeout(HRTIME_SECONDS(Http2::accept_no_activity_timeout)); + this->schedule_event = nullptr; + this->mutex = new_vc->mutex; + + this->connection_state.mutex = this->mutex; + + SSLNetVConnection *ssl_vc = dynamic_cast(new_vc); + if (ssl_vc != nullptr) { + this->read_from_early_data = ssl_vc->read_from_early_data; + Debug("ssl_early_data", "read_from_early_data = %" PRId64, this->read_from_early_data); + } + + Http2SsnDebug("session born, netvc %p", this->_vc); + + this->_vc->set_tcp_congestion_control(CLIENT_SIDE); + + this->read_buffer = iobuf ? iobuf : new_MIOBuffer(HTTP2_HEADER_BUFFER_SIZE_INDEX); + this->read_buffer->water_mark = connection_state.local_settings.get(HTTP2_SETTINGS_MAX_FRAME_SIZE); + this->_read_buffer_reader = reader ? reader : this->read_buffer->alloc_reader(); + + // Set write buffer size to max size of TLS record (16KB) + // This block size is the buffer size that we pass to SSLWriteBuffer + auto buffer_block_size_index = iobuffer_size_to_index(Http2::write_buffer_block_size, MAX_BUFFER_SIZE_INDEX); + this->write_buffer = new_MIOBuffer(buffer_block_size_index); + this->_write_buffer_reader = this->write_buffer->alloc_reader(); + this->_write_size_threshold = index_to_buffer_size(buffer_block_size_index) * Http2::write_size_threshold; + + this->_handle_if_ssl(new_vc); + + do_api_callout(TS_HTTP_SSN_START_HOOK); + + this->add_session(); +} + +// implement that. After we send a GOAWAY, there +// are scenarios where we would like to complete the outstanding streams. + +void +Http2ServerSession::do_io_close(int alerrno) +{ + REMEMBER(NO_EVENT, this->recursion) + + if (!this->connection_state.is_state_closed()) { + Http2SsnDebug("session closed"); + this->remove_session(); + + ink_assert(this->mutex->thread_holding == this_ethread()); + send_connection_event(&this->connection_state, HTTP2_SESSION_EVENT_FINI, this); + + // Destroy will be called from connection_state.release_stream() once the number of active streams goes to 0 + } +} + +int +Http2ServerSession::main_event_handler(int event, void *edata) +{ + ink_assert(this->mutex->thread_holding == this_ethread()); + int retval; + + recursion++; + + Event *e = static_cast(edata); + if (e == schedule_event) { + schedule_event = nullptr; + } + + Http2SsnDebug("main_event_handler=%d edata=%p", event, edata); + + switch (event) { + case VC_EVENT_READ_COMPLETE: + case VC_EVENT_READ_READY: { + bool is_zombie = connection_state.get_zombie_event() != nullptr; + retval = (this->*session_handler)(event, edata); + if (is_zombie && connection_state.get_zombie_event() != nullptr) { + Warning("Processed read event for zombie session %" PRId64, connection_id()); + } + break; + } + + case HTTP2_SESSION_EVENT_REENABLE: + // VIO will be reenableed in this handler + retval = (this->*session_handler)(VC_EVENT_READ_READY, static_cast(e->cookie)); + // Clear the event after calling session_handler to not reschedule REENABLE in it + this->_reenable_event = nullptr; + break; + + case VC_EVENT_ACTIVE_TIMEOUT: + case VC_EVENT_INACTIVITY_TIMEOUT: + case VC_EVENT_ERROR: + case VC_EVENT_EOS: + this->set_dying_event(event); + this->do_io_close(); + retval = 0; + break; + + case VC_EVENT_WRITE_READY: + case VC_EVENT_WRITE_COMPLETE: + this->connection_state.restart_streams(); + if ((Thread::get_hrtime() >= this->_write_buffer_last_flush + HRTIME_MSECONDS(this->_write_time_threshold))) { + this->flush(); + } + + retval = 0; + break; + + case HTTP2_SESSION_EVENT_XMIT: + default: + Http2SsnDebug("unexpected event=%d edata=%p", event, edata); + ink_release_assert(0); + retval = 0; + break; + } + + if (!this->is_draining() && this->connection_state.get_shutdown_reason() == Http2ErrorCode::HTTP2_ERROR_MAX) { + this->connection_state.set_shutdown_state(HTTP2_SHUTDOWN_NONE); + } + + if (this->connection_state.get_shutdown_state() == HTTP2_SHUTDOWN_NONE) { + if (this->is_draining()) { // For a case we already checked Connection header and it didn't exist + Http2SsnDebug("Preparing for graceful shutdown because of draining state"); + this->connection_state.set_shutdown_state(HTTP2_SHUTDOWN_NOT_INITIATED); + } /*else if (this->connection_state.get_stream_error_rate() > + Http2::stream_error_rate_threshold) { // For a case many stream errors happened + ip_port_text_buffer ipb; + const char *client_ip = ats_ip_ntop(get_remote_addr(), ipb, sizeof(ipb)); + SiteThrottledWarning("HTTP/2 session error origin_ip=%s session_id=%" PRId64 + " closing a connection, because its stream error rate (%f) exceeded the threshold (%f)", + client_ip, connection_id(), this->connection_state.get_stream_error_rate(), Http2::stream_error_rate_threshold); + Http2SsnDebug("Preparing for graceful shutdown because of a high stream error rate"); + cause_of_death = Http2SessionCod::HIGH_ERROR_RATE; + this->connection_state.set_shutdown_state(HTTP2_SHUTDOWN_NOT_INITIATED, Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM); + } */ + } + + if (this->connection_state.get_shutdown_state() == HTTP2_SHUTDOWN_NOT_INITIATED) { + send_connection_event(&this->connection_state, HTTP2_SESSION_EVENT_SHUTDOWN_INIT, this); + } + + recursion--; + if (!connection_state.is_recursing() && this->recursion == 0 && kill_me) { + this->free(); + } + return retval; +} + +void +Http2ServerSession::increment_current_active_connections_stat() +{ + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_ACTIVE_SERVER_CONNECTION_COUNT, this_ethread()); +} + +void +Http2ServerSession::decrement_current_active_connections_stat() +{ + HTTP2_DECREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_ACTIVE_SERVER_CONNECTION_COUNT, this_ethread()); +} + +sockaddr const * +Http2ServerSession::get_remote_addr() const +{ + return _vc ? _vc->get_remote_addr() : &cached_client_addr.sa; +} + +sockaddr const * +Http2ServerSession::get_local_addr() +{ + return _vc ? _vc->get_local_addr() : &cached_local_addr.sa; +} + +int +Http2ServerSession::get_transact_count() const +{ + return connection_state.get_stream_requests(); +} + +const char * +Http2ServerSession::get_protocol_string() const +{ + return "http/2"; +} + +void +Http2ServerSession::release(ProxyTransaction *trans) +{ +} + +int +Http2ServerSession::populate_protocol(std::string_view *result, int size) const +{ + int retval = 0; + if (size > retval) { + result[retval++] = IP_PROTO_TAG_HTTP_2_0; + if (size > retval) { + retval += super::populate_protocol(result + retval, size - retval); + } + } + return retval; +} + +const char * +Http2ServerSession::protocol_contains(std::string_view prefix) const +{ + const char *retval = nullptr; + + if (prefix.size() <= IP_PROTO_TAG_HTTP_2_0.size() && strncmp(IP_PROTO_TAG_HTTP_2_0.data(), prefix.data(), prefix.size()) == 0) { + retval = IP_PROTO_TAG_HTTP_2_0.data(); + } else { + retval = super::protocol_contains(prefix); + } + return retval; +} + +ProxySession * +Http2ServerSession::get_proxy_session() +{ + return this; +} + +ProxyTransaction * +Http2ServerSession::new_transaction() +{ + this->set_session_active(); + + // Create a new stream/transaction + Http2Error error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); + Http2Stream *stream = connection_state.create_initiating_stream(true, error); + + if (!stream || connection_state.is_peer_concurrent_stream_ub()) { + remove_session(); + } + + return stream; +} + +void +Http2ServerSession::add_session() +{ + if (this->in_session_table) { + return; + } + Http2SsnDebug("Add session to pool"); + EThread *ethread = this_ethread(); + ServerSessionPool *pool = ethread->server_session_pool; + MUTEX_TRY_LOCK(lock, pool->mutex, ethread); + if (lock.is_locked()) { + pool->addSession(this); + this->in_session_table = true; + } +} + +void +Http2ServerSession::remove_session() +{ + if (!this->in_session_table) { + return; + } + Http2SsnDebug("Remove session from pool"); + EThread *ethread = this_ethread(); + ServerSessionPool *pool = ethread->server_session_pool; + MUTEX_TRY_LOCK(lock, pool->mutex, ethread); + if (lock.is_locked()) { + pool->removeSession(this); + in_session_table = false; + } else { + ink_release_assert(!"How did we not get the pool lock?"); + } +} + +bool +Http2ServerSession::is_multiplexing() const +{ + return true; +} + +bool +Http2ServerSession::is_outbound() const +{ + return true; +} + +void +Http2ServerSession::set_netvc(NetVConnection *netvc) +{ + super::set_netvc(netvc); + if (netvc == nullptr) { + write_vio = nullptr; + } +} + +void +Http2ServerSession::set_no_activity_timeout() +{ + // Only set if not previously set + if (this->_vc->get_inactivity_timeout() == 0) { + this->set_inactivity_timeout(HRTIME_SECONDS(Http2::no_activity_timeout_out)); + } +} + +HTTPVersion +Http2ServerSession::get_version(HTTPHdr &hdr) const +{ + return HTTP_2_0; +} + +IOBufferReader * +Http2ServerSession::get_remote_reader() +{ + return _read_buffer_reader; +} + +std::function Create_h2_server_session = []() -> PoolableSession * { + return http2ServerSessionAllocator.alloc(); +}; diff --git a/proxy/http2/Http2ServerSession.h b/proxy/http2/Http2ServerSession.h new file mode 100644 index 00000000000..6bd4e330ff0 --- /dev/null +++ b/proxy/http2/Http2ServerSession.h @@ -0,0 +1,94 @@ +/** @file + + Http2ServerSession. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "Plugin.h" +#include "Http2CommonSession.h" +#include +#include "tscore/ink_inet.h" +#include "tscore/History.h" +#include "Milestones.h" +#include "PoolableSession.h" + +class Http2ServerSession : public PoolableSession, public Http2CommonSession +{ +public: + using super = PoolableSession; ///< Parent type. + using SessionHandler = int (Http2ServerSession::*)(int, void *); + + Http2ServerSession(); + + ///////////////////// + // Methods + + // Implement VConnection interface + void do_io_close(int lerrno = -1) override; + + // Implement ProxySession interface + void new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOBufferReader *reader) override; + void start() override; + void destroy() override; + void release(ProxyTransaction *trans) override; + void free() override; + ProxyTransaction *new_transaction() override; + + void add_session() override; + void remove_session(); + + //////////////////// + // Accessors + sockaddr const *get_remote_addr() const override; + sockaddr const *get_local_addr() override; + int get_transact_count() const override; + const char *get_protocol_string() const override; + int populate_protocol(std::string_view *result, int size) const override; + const char *protocol_contains(std::string_view prefix) const override; + HTTPVersion get_version(HTTPHdr &hdr) const override; + void increment_current_active_connections_stat() override; + void decrement_current_active_connections_stat() override; + IOBufferReader *get_remote_reader() override; + + ProxySession *get_proxy_session() override; + + // noncopyable + Http2ServerSession(Http2ServerSession &) = delete; + Http2ServerSession &operator=(const Http2ServerSession &) = delete; + + bool is_multiplexing() const override; + bool is_outbound() const override; + + void set_netvc(NetVConnection *netvc) override; + + void set_no_activity_timeout() override; + +private: + int main_event_handler(int, void *); + + IpEndpoint cached_client_addr; + IpEndpoint cached_local_addr; + + bool in_session_table = false; +}; + +extern ClassAllocator http2ServerSessionAllocator; diff --git a/proxy/http2/Http2Stream.cc b/proxy/http2/Http2Stream.cc index 87747e4d900..e45400a5198 100644 --- a/proxy/http2/Http2Stream.cc +++ b/proxy/http2/Http2Stream.cc @@ -25,6 +25,7 @@ #include "HTTP2.h" #include "Http2ClientSession.h" +#include "Http2ServerSession.h" #include "../http/HttpSM.h" #include @@ -39,25 +40,33 @@ ClassAllocator http2StreamAllocator("http2StreamAllocator"); -Http2Stream::Http2Stream(ProxySession *session, Http2StreamId sid, ssize_t initial_rwnd) - : super(session), _id(sid), _client_rwnd(initial_rwnd) +Http2Stream::Http2Stream(ProxySession *session, Http2StreamId sid, ssize_t initial_rwnd, bool outbound_connection, + bool registered_stream) + : super(session), _id(sid), _registered_stream(registered_stream), _client_rwnd(initial_rwnd) { SET_HANDLER(&Http2Stream::main_event_handler); this->mark_milestone(Http2StreamMilestone::OPEN); - this->_sm = nullptr; - this->_id = sid; - this->_thread = this_ethread(); - this->_client_rwnd = initial_rwnd; - this->_server_rwnd = Http2::initial_window_size; + this->_sm = nullptr; + this->_id = sid; + this->_thread = this_ethread(); + this->_state = Http2StreamState::HTTP2_STREAM_STATE_IDLE; + this->_outbound_flag = outbound_connection; + this->_client_rwnd = initial_rwnd; + this->_server_rwnd = Http2::initial_window_size; - this->_reader = this->_request_buffer.alloc_reader(); + this->_reader = this->_recv_buffer.alloc_reader(); - _req_header.create(HTTP_TYPE_REQUEST); - response_header.create(HTTP_TYPE_RESPONSE); - // TODO: init _req_header instead of response_header if this Http2Stream is outgoing - http2_init_pseudo_headers(response_header); + if (this->is_outbound_connection()) { // Flip the sense of the expected headers. Fix naming later + _recv_header.create(HTTP_TYPE_RESPONSE); + _send_header.create(HTTP_TYPE_REQUEST); + } else { + _recv_header.create(HTTP_TYPE_REQUEST); + _send_header.create(HTTP_TYPE_RESPONSE); + } + + http2_init_pseudo_headers(_send_header); http_parser_init(&http_parser); } @@ -66,6 +75,14 @@ Http2Stream::~Http2Stream() { REMEMBER(NO_EVENT, this->reentrancy_count); Http2StreamDebug("Destroy stream, sent %" PRIu64 " bytes", this->bytes_sent); + + // In the case of a temporary stream used to parse the header to keep the HPACK + // up to date, there may not be a mutex. Nothing was set up, so nothing to + // clean up in the destructor + if (this->mutex == nullptr) { + return; + } + SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); // Clean up after yourself if this was an EOS ink_release_assert(this->closed); @@ -73,23 +90,25 @@ Http2Stream::~Http2Stream() uint64_t cid = 0; - // Safe to initiate SSN_CLOSE if this is the last stream - if (_proxy_ssn) { - cid = _proxy_ssn->connection_id(); + if (_registered_stream) { + // Safe to initiate SSN_CLOSE if this is the last stream + if (_proxy_ssn) { + cid = _proxy_ssn->connection_id(); - Http2ClientSession *h2_proxy_ssn = static_cast(_proxy_ssn); - SCOPED_MUTEX_LOCK(lock, h2_proxy_ssn->mutex, this_ethread()); - // Make sure the stream is removed from the stream list and priority tree - // In many cases, this has been called earlier, so this call is a no-op - h2_proxy_ssn->connection_state.delete_stream(this); + SCOPED_MUTEX_LOCK(lock, _proxy_ssn->mutex, this_ethread()); + Http2ConnectionState &connection_state = this->get_connection_state(); + // Make sure the stream is removed from the stream list and priority tree + // In many cases, this has been called earlier, so this call is a no-op + connection_state.delete_stream(this); - h2_proxy_ssn->connection_state.decrement_stream_count(); + connection_state.decrement_stream_count(); - // Update session's stream counts, so it accurately goes into keep-alive state - h2_proxy_ssn->connection_state.release_stream(); + // Update session's stream counts, so it accurately goes into keep-alive state + connection_state.release_stream(); - // Do not access `_proxy_ssn` in below. It might be freed by `release_stream`. - } + // Do not access `_proxy_ssn` in below. It might be freed by `release_stream`. + } + } // Otherwise, not registered with the connection_state (i.e. a temporary stream used for HPACK header processing) // Clean up the write VIO in case of inactivity timeout this->do_io_write(nullptr, 0, nullptr); @@ -119,11 +138,11 @@ Http2Stream::~Http2Stream() this->_milestones.difference_sec(Http2StreamMilestone::OPEN, Http2StreamMilestone::CLOSE)); } - _req_header.destroy(); - response_header.destroy(); + _recv_header.destroy(); + _send_header.destroy(); // Drop references to all buffer data - this->_request_buffer.clear(); + this->_recv_buffer.clear(); // Free the mutexes in the VIO read_vio.mutex.clear(); @@ -190,6 +209,8 @@ Http2Stream::main_event_handler(int event, void *edata) } break; case VC_EVENT_READ_COMPLETE: + read_vio.nbytes = read_vio.ndone; + /* fall through */ case VC_EVENT_READ_READY: _timeout.update_inactivity(); if (e->cookie == &read_vio) { @@ -203,10 +224,14 @@ Http2Stream::main_event_handler(int event, void *edata) case VC_EVENT_EOS: if (e->cookie == &read_vio) { SCOPED_MUTEX_LOCK(lock, read_vio.mutex, this_ethread()); - read_vio.cont->handleEvent(VC_EVENT_EOS, &read_vio); + if (read_vio.cont) { + read_vio.cont->handleEvent(VC_EVENT_EOS, &read_vio); + } } else if (e->cookie == &write_vio) { SCOPED_MUTEX_LOCK(lock, write_vio.mutex, this_ethread()); - write_vio.cont->handleEvent(VC_EVENT_EOS, &write_vio); + if (write_vio.cont) { + write_vio.cont->handleEvent(VC_EVENT_EOS, &write_vio); + } } break; } @@ -220,18 +245,29 @@ Http2Stream::main_event_handler(int event, void *edata) Http2ErrorCode Http2Stream::decode_header_blocks(HpackHandle &hpack_handle, uint32_t maximum_table_size) { - return http2_decode_header_blocks(&_req_header, (const uint8_t *)header_blocks, header_blocks_length, nullptr, hpack_handle, - trailing_header, maximum_table_size); + return http2_decode_header_blocks(&_recv_header, (const uint8_t *)header_blocks, header_blocks_length, nullptr, hpack_handle, + _possible_trailing_header, maximum_table_size, _outbound_flag); } void Http2Stream::send_request(Http2ConnectionState &cstate) { - ink_release_assert(this->_sm != nullptr); - this->_http_sm_id = this->_sm->sm_id; + if (closed) { + return; + } + REMEMBER(NO_EVENT, this->reentrancy_count); // Convert header to HTTP/1.1 format - http2_convert_header_from_2_to_1_1(&_req_header); + http2_convert_header_from_2_to_1_1(&_recv_header); + + if (this->expect_send_trailer()) { + // Send read complete to terminate previous data tunnel + this->read_vio.nbytes = this->read_vio.ndone; + this->signal_read_event(VC_EVENT_READ_COMPLETE); + } + + ink_release_assert(this->_sm != nullptr); + this->_http_sm_id = this->_sm->sm_id; // Write header to a buffer. Borrowing logic from HttpSM::write_header_into_buffer. // Seems like a function like this ought to be in HTTPHdr directly @@ -241,16 +277,16 @@ Http2Stream::send_request(Http2ConnectionState &cstate) do { bufindex = 0; tmp = dumpoffset; - IOBufferBlock *block = this->_request_buffer.get_current_block(); + IOBufferBlock *block = this->_recv_buffer.get_current_block(); if (!block) { - this->_request_buffer.add_block(); - block = this->_request_buffer.get_current_block(); + this->_recv_buffer.add_block(); + block = this->_recv_buffer.get_current_block(); } - done = _req_header.print(block->start(), block->write_avail(), &bufindex, &tmp); + done = _recv_header.print(block->end(), block->write_avail(), &bufindex, &tmp); dumpoffset += bufindex; - this->_request_buffer.fill(bufindex); + this->_recv_buffer.fill(bufindex); if (!done) { - this->_request_buffer.add_block(); + this->_recv_buffer.add_block(); } } while (!done); @@ -263,7 +299,12 @@ Http2Stream::send_request(Http2ConnectionState &cstate) if (this->read_vio.nbytes > 0) { if (this->recv_end_stream) { this->read_vio.nbytes = bufindex; - this->signal_read_event(VC_EVENT_READ_COMPLETE); + this->read_vio.ndone = bufindex; + if (_outbound_flag) { + this->signal_read_event(VC_EVENT_EOS); + } else { + this->signal_read_event(VC_EVENT_READ_COMPLETE); + } } else { // End of header but not end of stream, must have some body frames coming this->has_body = true; @@ -295,6 +336,8 @@ Http2Stream::change_state(uint8_t type, uint8_t flags) } } else if (type == HTTP2_FRAME_TYPE_PUSH_PROMISE) { _state = Http2StreamState::HTTP2_STREAM_STATE_RESERVED_LOCAL; + } else if (type == HTTP2_FRAME_TYPE_RST_STREAM) { + _state = Http2StreamState::HTTP2_STREAM_STATE_CLOSED; } else { return false; } @@ -305,7 +348,11 @@ Http2Stream::change_state(uint8_t type, uint8_t flags) _state = Http2StreamState::HTTP2_STREAM_STATE_CLOSED; } else if (type == HTTP2_FRAME_TYPE_HEADERS || type == HTTP2_FRAME_TYPE_DATA) { if (recv_end_stream) { - _state = Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_REMOTE; + if (send_end_stream) { + _state = Http2StreamState::HTTP2_STREAM_STATE_CLOSED; + } else { + _state = Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_REMOTE; + } } else if (send_end_stream) { _state = Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_LOCAL; } else { @@ -340,8 +387,8 @@ Http2Stream::change_state(uint8_t type, uint8_t flags) _state = Http2StreamState::HTTP2_STREAM_STATE_CLOSED; } else { // Error, set state closed - _state = Http2StreamState::HTTP2_STREAM_STATE_CLOSED; - return false; + //_state = Http2StreamState::HTTP2_STREAM_STATE_CLOSED; + // return false; } break; @@ -356,8 +403,8 @@ Http2Stream::change_state(uint8_t type, uint8_t flags) return true; } else { // Error, set state closed - _state = Http2StreamState::HTTP2_STREAM_STATE_CLOSED; - return false; + //_state = Http2StreamState::HTTP2_STREAM_STATE_CLOSED; + // return false; } break; @@ -409,10 +456,11 @@ Http2Stream::do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *abuffe write_vio.ndone = 0; write_vio.vc_server = this; write_vio.op = VIO::WRITE; + _send_reader = abuffer; - if (c != nullptr && nbytes > 0 && this->is_client_state_writeable()) { + if (c != nullptr && nbytes > 0 && this->is_state_writeable()) { update_write_request(false); - } else if (!this->is_client_state_writeable()) { + } else if (!this->is_state_writeable()) { // Cannot start a write on a closed stream return nullptr; } @@ -434,18 +482,13 @@ Http2Stream::do_io_close(int /* flags */) // by the time this is called from transaction_done. closed = true; - if (_proxy_ssn && this->is_client_state_writeable()) { - // Make sure any trailing end of stream frames are sent - // We will be removed at send_data_frames or closing connection phase - Http2ClientSession *h2_proxy_ssn = static_cast(this->_proxy_ssn); - SCOPED_MUTEX_LOCK(lock, h2_proxy_ssn->mutex, this_ethread()); - h2_proxy_ssn->connection_state.send_data_frames(this); - } + // Adjust state, so we don't process any more data + _state = Http2StreamState::HTTP2_STREAM_STATE_CLOSED; _clear_timers(); clear_io_events(); - // Wait until transaction_done is called from HttpSM to signal that the TXN_CLOSE hook has been executed + // Otherwise, Wait until transaction_done is called from HttpSM to signal that the TXN_CLOSE hook has been executed } } @@ -461,7 +504,8 @@ Http2Stream::transaction_done() if (!closed) { do_io_close(); // Make sure we've been closed. If we didn't close the _proxy_ssn session better still be open } - ink_release_assert(closed || !static_cast(_proxy_ssn)->connection_state.is_state_closed()); + Http2ConnectionState &state = this->get_connection_state(); + ink_release_assert(closed || !state.is_state_closed()); _sm = nullptr; if (closed) { @@ -476,11 +520,11 @@ Http2Stream::transaction_done() void Http2Stream::terminate_if_possible() { - if (terminate_stream && reentrancy_count == 0) { + // if (terminate_stream && reentrancy_count == 0) { + if (reentrancy_count == 0 && closed && terminate_stream) { REMEMBER(NO_EVENT, this->reentrancy_count); - Http2ClientSession *h2_proxy_ssn = static_cast(this->_proxy_ssn); - SCOPED_MUTEX_LOCK(lock, h2_proxy_ssn->mutex, this_ethread()); + SCOPED_MUTEX_LOCK(lock, _proxy_ssn->mutex, this_ethread()); THREAD_FREE(this, http2StreamAllocator, this_ethread()); } } @@ -492,7 +536,8 @@ Http2Stream::initiating_close() if (!closed) { SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); REMEMBER(NO_EVENT, this->reentrancy_count); - Http2StreamDebug("initiating_close"); + Http2StreamDebug("initiating_close client_window=%" PRId64 " session_window=%" PRId64, _client_rwnd, + this->get_connection_state().client_rwnd()); // Set the state of the connection to closed // TODO - these states should be combined @@ -512,31 +557,42 @@ Http2Stream::initiating_close() // kill yourself signal // We are sending signals rather than calling the handlers directly to avoid the case where // the HttpTunnel handler causes the HttpSM to be deleted on the stack. - bool sent_write_complete = false; + bool sent_io_event = false; if (_sm) { // Push out any last IO events - if (write_vio.cont) { + // First look for active write or read + if (write_vio.cont && write_vio.nbytes > 0) { SCOPED_MUTEX_LOCK(lock, write_vio.mutex, this_ethread()); - // Are we done? - if (write_vio.nbytes > 0 && write_vio.nbytes == write_vio.ndone) { - Http2StreamDebug("handle write from destroy (event=%d)", VC_EVENT_WRITE_COMPLETE); - write_event = send_tracked_event(write_event, VC_EVENT_WRITE_COMPLETE, &write_vio); - } else { - write_event = send_tracked_event(write_event, VC_EVENT_EOS, &write_vio); - Http2StreamDebug("handle write from destroy (event=%d)", VC_EVENT_EOS); + if (write_vio.nbytes > 0 && write_vio.ndone == write_vio.nbytes) { + write_event = send_tracked_event(write_event, VC_EVENT_WRITE_COMPLETE, &write_vio); + sent_io_event = true; } - sent_write_complete = true; } - } - // Send EOS to let SM know that we aren't sticking around - if (_sm && read_vio.cont) { - // Only bother with the EOS if we haven't sent the write complete - if (!sent_write_complete) { + if (!sent_io_event && read_vio.cont && read_vio.nbytes > 0) { SCOPED_MUTEX_LOCK(lock, read_vio.mutex, this_ethread()); - Http2StreamDebug("send EOS to read cont"); - read_event = send_tracked_event(read_event, VC_EVENT_EOS, &read_vio); + if (read_vio.nbytes > 0 && read_vio.ndone == read_vio.nbytes) { + write_event = send_tracked_event(read_event, VC_EVENT_READ_COMPLETE, &read_vio); + sent_io_event = true; + } + } + + if (!sent_io_event) { + if (write_vio.cont && write_vio.buffer.writer()) { + SCOPED_MUTEX_LOCK(lock, write_vio.mutex, this_ethread()); + Http2StreamDebug("handle write from destroy (event=%d)", VC_EVENT_EOS); + write_event = send_tracked_event(write_event, VC_EVENT_EOS, &write_vio); + } else if (read_vio.cont && read_vio.buffer.writer()) { + SCOPED_MUTEX_LOCK(lock, read_vio.mutex, this_ethread()); + Http2StreamDebug("send EOS to read cont"); + read_event = send_tracked_event(read_event, VC_EVENT_EOS, &read_vio); + } else { + Http2StreamDebug("send EOS to SM"); + // Just send EOS to the _sm + _sm->handleEvent(VC_EVENT_EOS, nullptr); + } } - } else if (!sent_write_complete) { + } else if (!terminate_stream) { + Http2StreamDebug("No SM to signal"); // Transaction is already gone or not started. Kill yourself terminate_stream = true; terminate_if_possible(); @@ -577,7 +633,7 @@ Http2Stream::update_read_request(bool call_update) ink_release_assert(this->_thread == this_ethread()); SCOPED_MUTEX_LOCK(lock, read_vio.mutex, this_ethread()); - if (read_vio.nbytes == 0) { + if (read_vio.nbytes == 0 || read_vio.is_disabled()) { return; } @@ -604,11 +660,25 @@ Http2Stream::update_read_request(bool call_update) void Http2Stream::restart_sending() { - if (!this->response_header_done) { + // Make sure the stream is in a good state to be sending + if (this->is_closed()) { + return; + } + if (!this->parsing_header_done) { + this->update_write_request(true); return; } + if (_outbound_flag) { + if (this->get_state() != Http2StreamState::HTTP2_STREAM_STATE_OPEN || write_vio.ntodo() == 0) { + return; + } + } else { + if (this->get_state() != Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_REMOTE) { + return; + } + } - IOBufferReader *reader = this->response_get_data_reader(); + IOBufferReader *reader = this->send_get_data_reader(); if (reader && !reader->is_read_avail_more_than(0)) { return; } @@ -617,14 +687,14 @@ Http2Stream::restart_sending() return; } - this->send_response_body(true); + this->send_body(true); } void Http2Stream::update_write_request(bool call_update) { - if (!this->is_client_state_writeable() || closed || _proxy_ssn == nullptr || write_vio.mutex == nullptr || - write_vio.get_reader() == nullptr) { + if (!this->is_state_writeable() || closed || _proxy_ssn == nullptr || write_vio.mutex == nullptr || + write_vio.get_reader() == nullptr || this->_send_reader == nullptr) { return; } @@ -634,73 +704,94 @@ Http2Stream::update_write_request(bool call_update) } ink_release_assert(this->_thread == this_ethread()); - Http2ClientSession *h2_proxy_ssn = static_cast(this->_proxy_ssn); + Http2StreamDebug("update_write_request parse_done=%d", parsing_header_done); + + Http2ConnectionState &connection_state = this->get_connection_state(); SCOPED_MUTEX_LOCK(lock, write_vio.mutex, this_ethread()); IOBufferReader *vio_reader = write_vio.get_reader(); - if (write_vio.ntodo() == 0 || !vio_reader->is_read_avail_more_than(0)) { + if (write_vio.ntodo() > 0 && (!vio_reader->is_read_avail_more_than(0) || + // If there is no window left, just give up now too + std::min(_client_rwnd, this->get_connection_state().client_rwnd()) == 0)) { + Http2StreamDebug("update_write_request give up without doing anything ntodo=%" PRId64 " is_read_avail=%d client_window=%" PRId64 + " session_window=%" PRId64, + write_vio.ntodo(), vio_reader->is_read_avail_more_than(0), _client_rwnd, + this->get_connection_state().client_rwnd()); return; } // Process the new data - if (!this->response_header_done) { - // Still parsing the response_header + if (!this->parsing_header_done) { + // Still parsing the request or response header int bytes_used = 0; - int state = this->response_header.parse_resp(&http_parser, vio_reader, &bytes_used, false); - // HTTPHdr::parse_resp() consumed the vio_reader in above (consumed size is `bytes_used`) + int state; + if (this->is_outbound_connection()) { + state = this->_send_header.parse_req(&http_parser, this->_send_reader, &bytes_used, false); + } else { + state = this->_send_header.parse_resp(&http_parser, this->_send_reader, &bytes_used, false); + } + // HTTPHdr::parse_resp() consumed the send_reader in above write_vio.ndone += bytes_used; switch (state) { case PARSE_RESULT_DONE: { - this->response_header_done = true; + this->parsing_header_done = true; + Http2StreamDebug("update_write_request parsing done, read %d bytes", bytes_used); // Schedule session shutdown if response header has "Connection: close" - MIMEField *field = this->response_header.field_find(MIME_FIELD_CONNECTION, MIME_LEN_CONNECTION); + MIMEField *field = this->_send_header.field_find(MIME_FIELD_CONNECTION, MIME_LEN_CONNECTION); if (field) { int len; const char *value = field->value_get(&len); if (memcmp(HTTP_VALUE_CLOSE, value, HTTP_LEN_CLOSE) == 0) { - SCOPED_MUTEX_LOCK(lock, h2_proxy_ssn->mutex, this_ethread()); - if (h2_proxy_ssn->connection_state.get_shutdown_state() == HTTP2_SHUTDOWN_NONE) { - h2_proxy_ssn->connection_state.set_shutdown_state(HTTP2_SHUTDOWN_NOT_INITIATED, Http2ErrorCode::HTTP2_ERROR_NO_ERROR); + SCOPED_MUTEX_LOCK(lock, _proxy_ssn->mutex, this_ethread()); + if (connection_state.get_shutdown_state() == HTTP2_SHUTDOWN_NONE) { + connection_state.set_shutdown_state(HTTP2_SHUTDOWN_NOT_INITIATED, Http2ErrorCode::HTTP2_ERROR_NO_ERROR); } } } { - SCOPED_MUTEX_LOCK(lock, h2_proxy_ssn->mutex, this_ethread()); + SCOPED_MUTEX_LOCK(lock, _proxy_ssn->mutex, this_ethread()); // Send the response header back - h2_proxy_ssn->connection_state.send_headers_frame(this); + connection_state.send_headers_frame(this); } // Roll back states of response header to read final response - if (this->response_header.expect_final_response()) { - this->response_header_done = false; - response_header.destroy(); - response_header.create(HTTP_TYPE_RESPONSE); - http2_init_pseudo_headers(response_header); + if (!this->is_outbound_connection() && this->_send_header.expect_final_response()) { + this->parsing_header_done = false; + } + if (this->is_outbound_connection() || this->_send_header.expect_final_response()) { + _send_header.destroy(); + _send_header.create(this->is_outbound_connection() ? HTTP_TYPE_REQUEST : HTTP_TYPE_RESPONSE); + http2_init_pseudo_headers(_send_header); http_parser_clear(&http_parser); http_parser_init(&http_parser); } + bool final_write = this->write_vio.ntodo() == 0; + if (final_write) { + this->signal_write_event(VC_EVENT_WRITE_COMPLETE, false); + } - this->signal_write_event(call_update); - - if (vio_reader->is_read_avail_more_than(0)) { + if (!final_write && this->_send_reader->is_read_avail_more_than(0)) { + Http2StreamDebug("update_write_request done parsing, still more to send"); this->_milestones.mark(Http2StreamMilestone::START_TX_DATA_FRAMES); - this->send_response_body(call_update); + this->send_body(call_update); } break; } case PARSE_RESULT_CONT: // Let it ride for next time + Http2StreamDebug("update_write_request still parsing, read %d bytes", bytes_used); break; default: + Http2StreamDebug("update_write_request state %d, read %d bytes", state, bytes_used); break; } } else { this->_milestones.mark(Http2StreamMilestone::START_TX_DATA_FRAMES); - this->send_response_body(call_update); + this->send_body(call_update); } return; @@ -709,10 +800,12 @@ Http2Stream::update_write_request(bool call_update) void Http2Stream::signal_read_event(int event) { - if (this->read_vio.cont == nullptr || this->read_vio.cont->mutex == nullptr || this->read_vio.op == VIO::NONE) { + if (this->terminate_stream || this->closed || this->read_vio.cont == nullptr || this->read_vio.cont->mutex == nullptr || + this->read_vio.op == VIO::NONE) { return; } + reentrancy_count++; MUTEX_TRY_LOCK(lock, read_vio.cont->mutex, this_ethread()); if (lock.is_locked()) { _timeout.update_inactivity(); @@ -723,79 +816,82 @@ Http2Stream::signal_read_event(int event) } this->_read_vio_event = this_ethread()->schedule_in(this, retry_delay, event, &read_vio); } + reentrancy_count--; + // Clean stream up if the terminate flag is set and we are at the bottom of the handler stack + terminate_if_possible(); } void -Http2Stream::signal_write_event(int event) +Http2Stream::signal_write_event(int event, bool call_update) { // Don't signal a write event if in fact nothing was written if (this->write_vio.cont == nullptr || this->write_vio.cont->mutex == nullptr || this->write_vio.op == VIO::NONE || - this->write_vio.nbytes == 0) { - return; - } - - MUTEX_TRY_LOCK(lock, write_vio.cont->mutex, this_ethread()); - if (lock.is_locked()) { - _timeout.update_inactivity(); - this->write_vio.cont->handleEvent(event, &this->write_vio); - } else { - if (this->_write_vio_event) { - this->_write_vio_event->cancel(); - } - this->_write_vio_event = this_ethread()->schedule_in(this, retry_delay, event, &write_vio); - } -} - -void -Http2Stream::signal_write_event(bool call_update) -{ - if (this->write_vio.cont == nullptr || this->write_vio.op == VIO::NONE) { + this->terminate_stream) { return; } - if (this->write_vio.get_writer()->write_avail() == 0) { - return; - } - - int send_event = this->write_vio.ntodo() == 0 ? VC_EVENT_WRITE_COMPLETE : VC_EVENT_WRITE_READY; - + reentrancy_count++; if (call_update) { - // Coming from reenable. Safe to call the handler directly - if (write_vio.cont && this->_sm) { - write_vio.cont->handleEvent(send_event, &write_vio); + MUTEX_TRY_LOCK(lock, write_vio.cont->mutex, this_ethread()); + if (lock.is_locked()) { + if (write_event) { + write_event->cancel(); + write_event = nullptr; + } + _timeout.update_inactivity(); + this->write_vio.cont->handleEvent(event, &this->write_vio); + } else { + if (this->_write_vio_event) { + this->_write_vio_event->cancel(); + } + this->_write_vio_event = this_ethread()->schedule_in(this, retry_delay, event, &write_vio); } } else { // Called from do_io_write. Might still be setting up state. Send an event to let the dust settle - write_event = send_tracked_event(write_event, send_event, &write_vio); + write_event = send_tracked_event(write_event, event, &write_vio); } + reentrancy_count--; + // Clean stream up if the terminate flag is set and we are at the bottom of the handler stack + terminate_if_possible(); } bool Http2Stream::push_promise(URL &url, const MIMEField *accept_encoding) { - Http2ClientSession *h2_proxy_ssn = static_cast(this->_proxy_ssn); - SCOPED_MUTEX_LOCK(lock, h2_proxy_ssn->mutex, this_ethread()); - return h2_proxy_ssn->connection_state.send_push_promise_frame(this, url, accept_encoding); + SCOPED_MUTEX_LOCK(lock, _proxy_ssn->mutex, this_ethread()); + return this->get_connection_state().send_push_promise_frame(this, url, accept_encoding); } void -Http2Stream::send_response_body(bool call_update) +Http2Stream::send_body(bool call_update) { - Http2ClientSession *h2_proxy_ssn = static_cast(this->_proxy_ssn); + Http2ConnectionState &connection_state = this->get_connection_state(); _timeout.update_inactivity(); + reentrancy_count++; + + SCOPED_MUTEX_LOCK(lock, _proxy_ssn->mutex, this_ethread()); if (Http2::stream_priority_enabled) { - SCOPED_MUTEX_LOCK(lock, h2_proxy_ssn->mutex, this_ethread()); - h2_proxy_ssn->connection_state.schedule_stream(this); + connection_state.schedule_stream(this); // signal_write_event() will be called from `Http2ConnectionState::send_data_frames_depends_on_priority()` // when write_vio is consumed } else { - SCOPED_MUTEX_LOCK(lock, h2_proxy_ssn->mutex, this_ethread()); - h2_proxy_ssn->connection_state.send_data_frames(this); - this->signal_write_event(call_update); + connection_state.send_data_frames(this); // XXX The call to signal_write_event can destroy/free the Http2Stream. // Don't modify the Http2Stream after calling this method. } + + reentrancy_count--; + terminate_if_possible(); +} + +void +Http2Stream::reenable_write() +{ + if (this->_proxy_ssn) { + SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); + update_write_request(true); + } } void @@ -806,22 +902,20 @@ Http2Stream::reenable(VIO *vio) SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); update_write_request(true); } else if (vio->op == VIO::READ) { - Http2ClientSession *h2_proxy_ssn = static_cast(this->_proxy_ssn); - { - SCOPED_MUTEX_LOCK(ssn_lock, h2_proxy_ssn->mutex, this_ethread()); - h2_proxy_ssn->connection_state.restart_receiving(this); - } + SCOPED_MUTEX_LOCK(ssn_lock, _proxy_ssn->mutex, this_ethread()); + Http2ConnectionState &connection_state = this->get_connection_state(); + connection_state.restart_receiving(this); - SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); - update_read_request(true); + // SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread()); + // update_read_request(true); } } } IOBufferReader * -Http2Stream::response_get_data_reader() const +Http2Stream::send_get_data_reader() const { - return write_vio.get_reader(); + return this->_send_reader; } void @@ -899,14 +993,23 @@ Http2Stream::release() void Http2Stream::increment_transactions_stat() { - HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_STREAM_COUNT, _thread); - HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_TOTAL_CLIENT_STREAM_COUNT, _thread); + if (_outbound_flag) { + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_SERVER_STREAM_COUNT, _thread); + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_TOTAL_SERVER_STREAM_COUNT, _thread); + } else { + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_STREAM_COUNT, _thread); + HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_TOTAL_CLIENT_STREAM_COUNT, _thread); + } } void Http2Stream::decrement_transactions_stat() { - HTTP2_DECREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_STREAM_COUNT, _thread); + if (_outbound_flag) { + HTTP2_DECREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_SERVER_STREAM_COUNT, _thread); + } else { + HTTP2_DECREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_STREAM_COUNT, _thread); + } } ssize_t @@ -1012,3 +1115,68 @@ Http2Stream::has_request_body(int64_t content_length, bool is_chunked_set) const { return has_body; } + +Http2ConnectionState & +Http2Stream::get_connection_state() +{ + if (_outbound_flag) { + Http2ServerSession *session = static_cast(_proxy_ssn); + return session->connection_state; + } else { + Http2ClientSession *session = static_cast(_proxy_ssn); + return session->connection_state; + } +} + +bool +Http2Stream::is_read_closed() const +{ + return this->recv_end_stream; +} + +bool +Http2Stream::expect_send_trailer() const +{ + return this->_expect_send_trailer; +} + +void +Http2Stream::set_expect_send_trailer() +{ + _expect_send_trailer = true; + parsing_header_done = false; + reset_send_headers(); +} +bool +Http2Stream::expect_receive_trailer() const +{ + return this->_expect_receive_trailer; +} + +void +Http2Stream::set_expect_receive_trailer() +{ + _expect_receive_trailer = true; +} + +void +Http2Stream::set_rx_error_code(ProxyError e) +{ + if (!this->is_outbound_connection() && this->_sm) { + this->_sm->t_state.client_info.rx_error_code = e; + } +} + +void +Http2Stream::set_tx_error_code(ProxyError e) +{ + if (!this->is_outbound_connection() && this->_sm) { + this->_sm->t_state.client_info.tx_error_code = e; + } +} + +HTTPVersion +Http2Stream::get_version(HTTPHdr &hdr) const +{ + return HTTP_2_0; +} diff --git a/proxy/http2/Http2Stream.h b/proxy/http2/Http2Stream.h index 03d9decea09..aac73963e4f 100644 --- a/proxy/http2/Http2Stream.h +++ b/proxy/http2/Http2Stream.h @@ -55,13 +55,14 @@ class Http2Stream : public ProxyTransaction using super = ProxyTransaction; ///< Parent type. Http2Stream() {} // Just to satisfy ClassAllocator - Http2Stream(ProxySession *session, Http2StreamId sid, ssize_t initial_rwnd); + Http2Stream(ProxySession *session, Http2StreamId sid, ssize_t initial_rwnd, bool outbound_connection, bool registered_stream); ~Http2Stream(); int main_event_handler(int event, void *edata); void release() override; void reenable(VIO *vio) override; + void reenable_write(); void transaction_done() override; void do_io_shutdown(ShutdownHowTo_t) override {} @@ -69,16 +70,25 @@ class Http2Stream : public ProxyTransaction VIO *do_io_write(Continuation *c, int64_t nbytes, IOBufferReader *abuffer, bool owner = false) override; void do_io_close(int lerrno = -1) override; + bool expect_send_trailer() const override; + void set_expect_send_trailer() override; + bool expect_receive_trailer() const override; + void set_expect_receive_trailer() override; + Http2ErrorCode decode_header_blocks(HpackHandle &hpack_handle, uint32_t maximum_table_size); void send_request(Http2ConnectionState &cstate); void initiating_close(); + bool + is_outbound_connection() const + { + return _outbound_flag; + } void terminate_if_possible(); void update_read_request(bool send_update); void update_write_request(bool send_update); void signal_read_event(int event); - void signal_write_event(int event); - void signal_write_event(bool call_update); + void signal_write_event(int event, bool call_update = true); void restart_sending(); bool push_promise(URL &url, const MIMEField *accept_encoding); @@ -103,17 +113,31 @@ class Http2Stream : public ProxyTransaction bool is_first_transaction() const override; void increment_transactions_stat() override; void decrement_transactions_stat() override; + void set_transaction_id(int new_id); int get_transaction_id() const override; int get_transaction_priority_weight() const override; int get_transaction_priority_dependence() const override; + bool is_read_closed() const override; + + HTTPHdr * + get_send_header() + { + return &_send_header; + } + + void read_update(int count); + void read_done(); void clear_io_events(); - bool is_client_state_writeable() const; + bool is_state_writeable() const; bool is_closed() const; - IOBufferReader *response_get_data_reader() const; + IOBufferReader *send_get_data_reader() const; + void set_rx_error_code(ProxyError e) override; + void set_tx_error_code(ProxyError e) override; bool has_request_body(int64_t content_length, bool is_chunked_set) const override; + HTTPVersion get_version(HTTPHdr &hdr) const override; void mark_milestone(Http2StreamMilestone type); @@ -125,29 +149,36 @@ class Http2Stream : public ProxyTransaction Http2StreamState get_state() const; bool change_state(uint8_t type, uint8_t flags); void update_initial_rwnd(Http2WindowSize new_size); - bool has_trailing_header() const; - void set_request_headers(HTTPHdr &h2_headers); + bool trailing_header_is_possible() const; + void set_trailing_header_is_possible(); + void set_recv_headers(HTTPHdr &h2_headers); + void reset_recv_headers(); + void reset_send_headers(); MIOBuffer *read_vio_writer() const; int64_t read_vio_read_avail(); + bool read_enabled() const; ////////////////// // Variables uint8_t *header_blocks = nullptr; - uint32_t header_blocks_length = 0; // total length of header blocks (not include Padding or other fields) + uint32_t header_blocks_length = 0; // total length of header blocks (not include + // Padding or other fields) bool recv_end_stream = false; bool send_end_stream = false; - bool response_header_done = false; + bool parsing_header_done = false; bool is_first_transaction_flag = false; - HTTPHdr response_header; + HTTPHdr _send_header; + IOBufferReader *_send_reader = nullptr; Http2DependencyTree::Node *priority_node = nullptr; + Http2ConnectionState &get_connection_state(); + private: - bool response_is_data_available() const; Event *send_tracked_event(Event *event, int send_event, VIO *vio); - void send_response_body(bool call_update); + void send_body(bool call_update); void _clear_timers(); /** @@ -164,17 +195,22 @@ class Http2Stream : public ProxyTransaction Http2StreamState _state = Http2StreamState::HTTP2_STREAM_STATE_IDLE; int64_t _http_sm_id = -1; - HTTPHdr _req_header; - MIOBuffer _request_buffer = CLIENT_CONNECTION_FIRST_READ_BUFFER_SIZE_INDEX; - int64_t read_vio_nbytes; + HTTPHdr _recv_header; + MIOBuffer _recv_buffer = CLIENT_CONNECTION_FIRST_READ_BUFFER_SIZE_INDEX; VIO read_vio; VIO write_vio; History _history; Milestones(Http2StreamMilestone::LAST_ENTRY)> _milestones; - bool trailing_header = false; - bool has_body = false; + bool _possible_trailing_header = false; + bool _expect_send_trailer = false; + bool _expect_receive_trailer = false; + + bool has_body = false; + + bool _outbound_flag = false; + bool _registered_stream = true; // A brief discussion of similar flags and state variables: _state, closed, terminate_stream // @@ -251,6 +287,12 @@ Http2Stream::get_transaction_id() const return _id; } +inline void +Http2Stream::set_transaction_id(int new_id) +{ + _id = new_id; +} + inline Http2StreamState Http2Stream::get_state() const { @@ -264,15 +306,35 @@ Http2Stream::update_initial_rwnd(Http2WindowSize new_size) } inline bool -Http2Stream::has_trailing_header() const +Http2Stream::trailing_header_is_possible() const +{ + return _possible_trailing_header; +} + +inline void +Http2Stream::set_trailing_header_is_possible() +{ + _possible_trailing_header = true; +} + +inline void +Http2Stream::set_recv_headers(HTTPHdr &h2_headers) +{ + _recv_header.copy(&h2_headers); +} + +inline void +Http2Stream::reset_recv_headers() { - return trailing_header; + this->_recv_header.destroy(); + this->_recv_header.create(HTTP_TYPE_RESPONSE); } inline void -Http2Stream::set_request_headers(HTTPHdr &h2_headers) +Http2Stream::reset_send_headers() { - _req_header.copy(&h2_headers); + this->_send_header.destroy(); + this->_send_header.create(HTTP_TYPE_RESPONSE); } // Check entire DATA payload length if content-length: header is exist @@ -285,15 +347,16 @@ Http2Stream::increment_data_length(uint64_t length) inline bool Http2Stream::payload_length_is_valid() const { - uint32_t content_length = _req_header.get_content_length(); + uint32_t content_length = _recv_header.get_content_length(); return content_length == 0 || content_length == data_length; } inline bool -Http2Stream::is_client_state_writeable() const +Http2Stream::is_state_writeable() const { return _state == Http2StreamState::HTTP2_STREAM_STATE_OPEN || _state == Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_REMOTE || - _state == Http2StreamState::HTTP2_STREAM_STATE_RESERVED_LOCAL; + _state == Http2StreamState::HTTP2_STREAM_STATE_RESERVED_LOCAL || + (_outbound_flag && _state == Http2StreamState::HTTP2_STREAM_STATE_IDLE); } inline bool @@ -314,9 +377,27 @@ Http2Stream::read_vio_writer() const return this->read_vio.get_writer(); } +inline bool +Http2Stream::read_enabled() const +{ + return !this->read_vio.is_disabled(); +} + inline void Http2Stream::_clear_timers() { _timeout.cancel_active_timeout(); _timeout.cancel_inactive_timeout(); } + +inline void +Http2Stream::read_update(int count) +{ + read_vio.ndone += count; +} + +inline void +Http2Stream::read_done() +{ + read_vio.nbytes = read_vio.ndone; +} diff --git a/proxy/http2/Makefile.am b/proxy/http2/Makefile.am index 301d3119c24..932daed4b2d 100644 --- a/proxy/http2/Makefile.am +++ b/proxy/http2/Makefile.am @@ -42,8 +42,10 @@ libhttp2_a_SOURCES = \ Http2Frame.h \ Http2ClientSession.cc \ Http2ClientSession.h \ - Http2CommonSession.cc \ Http2CommonSession.h \ + Http2CommonSession.cc \ + Http2ServerSession.cc \ + Http2ServerSession.h \ Http2ConnectionState.cc \ Http2ConnectionState.h \ Http2DebugNames.cc \ diff --git a/proxy/http2/unit_tests/test_HTTP2.cc b/proxy/http2/unit_tests/test_HTTP2.cc index 21a772ae9f7..3febe204efb 100644 --- a/proxy/http2/unit_tests/test_HTTP2.cc +++ b/proxy/http2/unit_tests/test_HTTP2.cc @@ -108,8 +108,8 @@ TEST_CASE("Convert HTTPHdr", "[HTTP2]") // check CHECK_THAT(buf, Catch::StartsWith("GET https://trafficserver.apache.org/index.html HTTP/1.1\r\n" - "Host: trafficserver.apache.org\r\n" "User-Agent: foobar\r\n" + "Host: trafficserver.apache.org\r\n" "\r\n")); } diff --git a/proxy/logging/LogAccess.cc b/proxy/logging/LogAccess.cc index da82c04ced3..fae5a589aba 100644 --- a/proxy/logging/LogAccess.cc +++ b/proxy/logging/LogAccess.cc @@ -1276,7 +1276,7 @@ LogAccess::marshal_client_provided_cert(char *buf) { int provided_cert = 0; if (m_http_sm) { - auto txn = m_http_sm->get_ua_txn(); + auto txn = m_http_sm->ua_txn; if (txn) { auto ssn = txn->get_proxy_ssn(); if (ssn) { diff --git a/src/shared/overridable_txn_vars.cc b/src/shared/overridable_txn_vars.cc index ac8baa5f8bf..f99ca665ed2 100644 --- a/src/shared/overridable_txn_vars.cc +++ b/src/shared/overridable_txn_vars.cc @@ -160,4 +160,5 @@ const std::unordered_map(ssnp); if (ss != nullptr) { - vconn = reinterpret_cast(ss->get_netvc()); + return reinterpret_cast(ss->get_netvc()); } - return vconn; + return nullptr; } TSVConn @@ -7888,9 +7887,8 @@ TSHttpTxnServerFdGet(TSHttpTxn txnp, int *fdp) sdk_assert(sdk_sanity_check_txn(txnp) == TS_SUCCESS); sdk_assert(sdk_sanity_check_null_ptr((void *)fdp) == TS_SUCCESS); - HttpSM *sm = reinterpret_cast(txnp); - *fdp = -1; - + HttpSM *sm = reinterpret_cast(txnp); + *fdp = -1; TSReturnCode retval = TS_ERROR; ProxyTransaction *ss = sm->get_server_txn(); if (ss != nullptr) { @@ -8821,6 +8819,7 @@ _conf_to_memberp(TSOverridableConfigKey conf, OverridableHttpConfigParams *overr case TS_CONFIG_SSL_CERT_FILEPATH: case TS_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME: case TS_CONFIG_SSL_CLIENT_CA_CERT_FILENAME: + case TS_CONFIG_SSL_CLIENT_ALPN_PROTOCOLS: // String, must be handled elsewhere break; case TS_CONFIG_PARENT_FAILURES_UPDATE_HOSTDB: @@ -9059,6 +9058,11 @@ TSHttpTxnConfigStringSet(TSHttpTxn txnp, TSOverridableConfigKey conf, const char s->t_state.my_txn_conf().ssl_client_ca_cert_filename = const_cast(value); } break; + case TS_CONFIG_SSL_CLIENT_ALPN_PROTOCOLS: + if (value && length > 0) { + s->t_state.my_txn_conf().ssl_client_alpn_protocols = const_cast(value); + } + break; case TS_CONFIG_SSL_CERT_FILEPATH: /* noop */ break; diff --git a/src/traffic_server/InkAPITest.cc b/src/traffic_server/InkAPITest.cc index a8f27d21dfe..fcc1721afec 100644 --- a/src/traffic_server/InkAPITest.cc +++ b/src/traffic_server/InkAPITest.cc @@ -8695,7 +8695,8 @@ std::array SDK_Overridable_Configs = { "proxy.config.hostdb.ip_resolve", "proxy.config.http.connect.dead.policy", "proxy.config.plugin.vc.default_buffer_index", - "proxy.config.plugin.vc.default_buffer_water_mark"}}; + "proxy.config.plugin.vc.default_buffer_water_mark", + "proxy.config.ssl.client.alpn_protocols"}}; extern ClassAllocator httpSMAllocator; diff --git a/tests/gold_tests/h2/gold/nghttp_0_stdout.gold b/tests/gold_tests/h2/gold/nghttp_0_stdout.gold index e8e9acabd41..f19a43516d3 100644 --- a/tests/gold_tests/h2/gold/nghttp_0_stdout.gold +++ b/tests/gold_tests/h2/gold/nghttp_0_stdout.gold @@ -12,5 +12,3 @@ `` [``] recv (stream_id=1) :status: 200 `` -``; END_STREAM -`` diff --git a/tests/gold_tests/h2/gold/nghttp_1_stdout.gold b/tests/gold_tests/h2/gold/nghttp_1_stdout.gold index c1045323b0e..c1cd10559be 100644 --- a/tests/gold_tests/h2/gold/nghttp_1_stdout.gold +++ b/tests/gold_tests/h2/gold/nghttp_1_stdout.gold @@ -7,6 +7,6 @@ [``] recv GOAWAY frame (last_stream_id=1, error_code=NO_ERROR(0x00), opaque_data(0)=[]) `` -[``] recv DATA frame +[``] recv DATA frame ; END_STREAM `` diff --git a/tests/gold_tests/h2/h2origin.test.py b/tests/gold_tests/h2/h2origin.test.py new file mode 100644 index 00000000000..f0e02db026d --- /dev/null +++ b/tests/gold_tests/h2/h2origin.test.py @@ -0,0 +1,94 @@ +''' +Test communication to origin with H2 +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Test communication to origin with H2 +''' + +Test.ContinueOnFail = True + +# +# Communicate to origin with HTTP/2 +# +ts = Test.MakeATSProcess("ts", enable_tls="true") + +# add ssl materials like key, certificates for the server +ts.addDefaultSSLFiles() +replay_file = "replay/" +server = Test.MakeVerifierServerProcess("h2-origin", replay_file) +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http', + 'proxy.config.exec_thread.autoconfig': 0, + # Allow for more parallelism + 'proxy.config.exec_thread.limit': 4, + 'proxy.config.ssl.client.alpn_protocols': 'h2,http/1.1', + # Sticking with thread pool because global pool does not work with h2 + 'proxy.config.http.server_session_sharing.pool': 'thread', + 'proxy.config.http.server_session_sharing.match': 'ip,sni,cert', + 'proxy.config.ssl.client.verify.server.policy': 'PERMISSIVE', +}) + +ts.Disk.remap_config.AddLine( + 'map / https://127.0.0.1:{0}'.format(server.Variables.https_port) +) +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.logging_yaml.AddLines( + ''' +logging: + formats: + - name: testformat + format: '% % % % %<{uuid}cqh> % % % % % %' + logs: + - mode: ascii + format: testformat + filename: squid +'''.split("\n") +) + +tr = Test.AddTestRun("Test traffic to origin using HTTP/2") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(ts) +tr.AddVerifierClientProcess("client", replay_file, http_ports=[ts.Variables.port], https_ports=[ts.Variables.ssl_port]) +tr.StillRunningAfter = ts +tr.TimeOut = 60 + +# Just a check to flush out the traffic log until we have a clean shutdown for traffic_server +tr = Test.AddTestRun("Wait for the access log to write out") +tr.DelayStart = 10 +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'ls' +tr.Processes.Default.ReturnCode = 0 + +# UUIDs 1-4 should be http/1.1 clients and H2 origin +# UUIDs 5-9 should be http/2 clients and H2 origins +ts.Disk.squid_log.Content = Testers.ContainsExpression(" [1-4] http/1.1 http/2", "cases 1-4 request http/1.1") +ts.Disk.squid_log.Content += Testers.ExcludesExpression(" [1-4] http/2 http/2", "cases 1-4 request http/1.1") +ts.Disk.squid_log.Content = Testers.ContainsExpression(" 1[1-4] http/1.1 http/2", "cases 12-14 request http/1.1") +ts.Disk.squid_log.Content += Testers.ExcludesExpression(" 1[2-4] http/2 http/2", "cases 12-14 request http/1.1") +ts.Disk.squid_log.Content += Testers.ContainsExpression(" [5-9] http/2 http/2", "cases 5-11 request http/2") +ts.Disk.squid_log.Content += Testers.ExcludesExpression(" [5-9] http/1.1 http/2", "cases 5-11 request http/2") +ts.Disk.squid_log.Content += Testers.ContainsExpression(" 1[0-1] http/2 http/2", "cases 5-11 request http/2") +ts.Disk.squid_log.Content += Testers.ExcludesExpression(" 1[0-1] http/1.1 http/2", "cases 5-11 request http/2") diff --git a/tests/gold_tests/h2/h2origin_single_thread.test.py b/tests/gold_tests/h2/h2origin_single_thread.test.py new file mode 100644 index 00000000000..2254702c308 --- /dev/null +++ b/tests/gold_tests/h2/h2origin_single_thread.test.py @@ -0,0 +1,90 @@ +''' +Test communication to origin with H2 +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.Summary = ''' +Test communication to origin with H2 +''' + +Test.ContinueOnFail = True + +# +# Communicate to origin with HTTP/2 +# +ts = Test.MakeATSProcess("ts", enable_tls="true") + +# add ssl materials like key, certificates for the server +ts.addDefaultSSLFiles() +replay_file = "replay" +server = Test.MakeVerifierServerProcess("h2-origin", replay_file) +ts.Disk.records_config.update({ + 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http', + 'proxy.config.exec_thread.autoconfig': 0, + # Limiting ourselves to 1 thread to exercise origin reuse + 'proxy.config.exec_thread.limit': 1, + 'proxy.config.ssl.client.alpn_protocols': 'h2,http1.1', + # Sticking with hybrid pool because global pool does not work with h2 + 'proxy.config.http.server_session_sharing.pool': 'hybrid', + 'proxy.config.http.server_session_sharing.match': 'hostonly', + 'proxy.config.ssl.client.verify.server.policy': 'PERMISSIVE', +}) + +ts.Disk.remap_config.AddLine( + 'map / https://127.0.0.1:{0}'.format(server.Variables.https_port) +) +ts.Disk.ssl_multicert_config.AddLine( + 'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key' +) + +ts.Disk.logging_yaml.AddLines( + ''' +logging: + formats: + - name: testformat + format: '% % % % %<{uuid}cqh> % % % % % %' + logs: + - mode: ascii + format: testformat + filename: squid +'''.split("\n") +) + +tr = Test.AddTestRun("Test traffic to origin using HTTP/2") +tr.Processes.Default.StartBefore(server) +tr.Processes.Default.StartBefore(ts) +tr.AddVerifierClientProcess("client", replay_file, http_ports=[ts.Variables.port], https_ports=[ts.Variables.ssl_port]) +tr.StillRunningAfter = ts + +# Just a check to flush out the traffic log until we have a clean shutdown for traffic_server +tr = Test.AddTestRun("Wait for the access log to write out") +tr.DelayStart = 10 +tr.StillRunningAfter = ts +tr.StillRunningAfter = server +tr.Processes.Default.Command = 'ls' +tr.Processes.Default.ReturnCode = 0 + +# UUIDs 1-4 should be http/1.1 clients and H2 origin +# UUIDs 5-9 should be http/2 clients and H2 origins +ts.Disk.squid_log.Content = Testers.ContainsExpression(" [1-4] http/1.1 http/2", "cases 1-4 request http/1.1") +ts.Disk.squid_log.Content += Testers.ExcludesExpression(" [1-4] http/2 http/2", "cases 1-4 request http/1.1") +ts.Disk.squid_log.Content += Testers.ContainsExpression(" [5-9] http/2 http/2", "cases 5-9 request http/2") +ts.Disk.squid_log.Content += Testers.ExcludesExpression(" [5-9] http/1.1 http/2", "cases 5-9 request http/2") +ts.Disk.squid_log.Content += Testers.ContainsExpression(" http/2 1 1 1 [2-9]", "At least one case of origin reuse") diff --git a/tests/gold_tests/h2/http2.test.py b/tests/gold_tests/h2/http2.test.py index e6d2801ae35..b0b2e8caae8 100644 --- a/tests/gold_tests/h2/http2.test.py +++ b/tests/gold_tests/h2/http2.test.py @@ -189,6 +189,7 @@ post_body, ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = "gold/post_chunked.gold" +tr.TimeOut = 60 tr.StillRunningAfter = server # Test Case 7: Post with big chunked body @@ -199,6 +200,7 @@ ts.Variables.ssl_port) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Streams.All = "gold/post_chunked.gold" +tr.TimeOut = 60 tr.StillRunningAfter = server # Test Case 8: Huge response header @@ -208,6 +210,7 @@ tr.Processes.Default.Streams.stdout = "gold/http2_8_stdout.gold" # Different versions of curl will have different cases for HTTP/2 field names. tr.Processes.Default.Streams.stderr = Testers.GoldFile("gold/http2_8_stderr.gold", case_insensitive=True) +tr.TimeOut = 60 tr.StillRunningAfter = server # Test Case 9: Header Only Response - e.g. 204 @@ -217,4 +220,5 @@ tr.Processes.Default.Streams.stdout = "gold/http2_9_stdout.gold" # Different versions of curl will have different cases for HTTP/2 field names. tr.Processes.Default.Streams.stderr = Testers.GoldFile("gold/http2_9_stderr.gold", case_insensitive=True) +tr.TimeOut = 60 tr.StillRunningAfter = server diff --git a/tests/gold_tests/h2/httpbin.test.py b/tests/gold_tests/h2/httpbin.test.py index ff8eb95a712..7300b4faaaf 100644 --- a/tests/gold_tests/h2/httpbin.test.py +++ b/tests/gold_tests/h2/httpbin.test.py @@ -56,7 +56,7 @@ 'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir), 'proxy.config.diags.debug.enabled': 1, - 'proxy.config.diags.debug.tags': 'http2', + 'proxy.config.diags.debug.tags': 'http', }) ts.Disk.logging_yaml.AddLines( diff --git a/tests/gold_tests/h2/replay/h1-client-h2-origin.yaml b/tests/gold_tests/h2/replay/h1-client-h2-origin.yaml new file mode 100644 index 00000000000..93fa1cff5df --- /dev/null +++ b/tests/gold_tests/h2/replay/h1-client-h2-origin.yaml @@ -0,0 +1,596 @@ +meta: + version: '1.0' + +sessions: + - protocol: + - name: http + version: '1.1' + - name: tls + version: TLSv1.3 + sni: data.brian.example.com + proxy-verify-mode: 0 + proxy-provided-cert: true + - name: tcp + - name: ip + version: '4' + + transactions: + # + # Test 1: Zero length response. + # + - all: { headers: { fields: [[ uuid, 1 ]]}} + + client-request: + protocol: + - name: http + version: '1.1' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '1.1' + scheme: https + method: GET + url: /some/path + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + proxy-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: GET + url: /some/path + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + server-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + proxy-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + + # + # Test 2: Non-zero length response. + # + - all: { headers: { fields: [[ uuid, 2 ]]}} + + client-request: + protocol: + - name: http + version: '1.1' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '1.1' + scheme: https + method: GET + url: /some/path2 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + proxy-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: GET + url: /some/path2 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + server-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 16 ] + content: + encoding: plain + size: 16 + + proxy-response: + version: '1.1' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 16 ] + content: + encoding: plain + size: 16 + + - protocol: + - name: http + version: '1.1' + - name: tls + version: TLSv1.3 + sni: data.brian.example.com + proxy-verify-mode: 0 + proxy-provided-cert: true + - name: tcp + - name: ip + version: '4' + + transactions: + # + # Test 3: 8 byte post with a 404 response. + # + - all: { headers: { fields: [[ uuid, 3 ]]}} + + client-request: + protocol: + - name: http + version: '1.1' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '1.1' + scheme: https + method: POST + url: /some/path3 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 8 ] + content: + encoding: plain + size: 8 + + proxy-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path3 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 8 ] + content: + encoding: plain + size: 8 + + server-response: + version: '2' + status: 404 + reason: "Not Found" + headers: + encoding: esc_json + fields: + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + proxy-response: + version: '2' + status: 404 + reason: "Not Found" + headers: + encoding: esc_json + fields: + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + # + # Test 4: 32 byte POST with a 200 response. + # + - all: { headers: { fields: [[ uuid, 4 ]]}} + + client-request: + protocol: + - name: http + version: '1.1' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '1.1' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 32 ] + content: + encoding: plain + size: 32 + + proxy-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 32 ] + content: + encoding: plain + size: 32 + + server-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 32 ] + content: + encoding: plain + size: 32 + + proxy-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 32 ] + content: + encoding: plain + size: 32 + + # + # Test 5: 3200 byte POST with a 200 response. + # + - all: { headers: { fields: [[ uuid, 12 ]]}} + + client-request: + protocol: + - name: http + version: '1.1' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '1.1' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 3200 ] + content: + encoding: plain + size: 3200 + + proxy-request: + protocol: + - name: http + version: '1.1' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 3200 ] + content: + encoding: plain + size: 3200 + + server-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 1600 ] + content: + encoding: plain + size: 1600 + + proxy-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 1600 ] + content: + encoding: plain + size: 1600 + + # + # Test 6: large post body small response + # + - all: { headers: { fields: [[ uuid, 13 ]]}} + + client-request: + protocol: + - name: http + version: '1.1' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '1.1' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 3200 ] + content: + encoding: plain + size: 3200 + + proxy-request: + protocol: + - name: http + version: '1.1' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '1.1' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 3200 ] + content: + encoding: plain + size: 3200 + + server-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 16 ] + content: + encoding: plain + size: 16 + + proxy-response: + version: '1.1' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 16 ] + content: + encoding: plain + size: 16 + + # + # Test 7: small post body large response + # + - all: { headers: { fields: [[ uuid, 14 ]]}} + + client-request: + protocol: + - name: http + version: '1.1' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '1.1' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 32 ] + content: + encoding: plain + size: 32 + + proxy-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 32 ] + content: + encoding: plain + size: 32 + + server-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 3200 ] + content: + encoding: plain + size: 3200 + + proxy-response: + version: '1.1' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 3200 ] + content: + encoding: plain + size: 3200 diff --git a/tests/gold_tests/h2/replay/h2-origin.yaml b/tests/gold_tests/h2/replay/h2-origin.yaml new file mode 100644 index 00000000000..65b133375a4 --- /dev/null +++ b/tests/gold_tests/h2/replay/h2-origin.yaml @@ -0,0 +1,624 @@ +meta: + version: '1.0' + +sessions: + - protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.3 + sni: data.brian.example.com + proxy-verify-mode: 0 + proxy-provided-cert: true + - name: tcp + - name: ip + version: '4' + + transactions: + # + # Test 1: Zero length response. + # + - all: { headers: { fields: [[ uuid, 5 ]]}} + + client-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: GET + url: /some/path;arg=1;arg=2?foo + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + content: + encoding: plain + size: 0 + + proxy-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: GET + url: + - [ path, { value: /some/path;arg=1;arg=2, as: equal } ] + - [ query, { value: foo, as: equal } ] + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + server-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + proxy-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + + # + # Test 2: Non-zero length response. + # + - all: { headers: { fields: [[ uuid, 6 ]]}} + + client-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: GET + url: /some/path2 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + proxy-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: GET + url: + - [ path, { value: /some/path2, as: equal } ] + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + server-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 16 ] + content: + encoding: plain + size: 16 + + proxy-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 16 ] + content: + encoding: plain + size: 16 + + - protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.3 + sni: data.brian.example.com + proxy-verify-mode: 0 + proxy-provided-cert: true + - name: tcp + - name: ip + version: '4' + + transactions: + # + # Test 3: 8 byte post with a 404 response. + # + - all: { headers: { fields: [[ uuid, 7 ]]}} + + client-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path3?foo=bar + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 8 ] + content: + encoding: plain + size: 8 + + proxy-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: + - [ path, { value: /some/path3, as: equal }] + - [ query, { value: foo=bar, as: equal }] + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 8 ] + content: + encoding: plain + size: 8 + + server-response: + version: '2' + status: 404 + reason: "Not Found" + headers: + encoding: esc_json + fields: + - [ bob, 0 ] + content: + encoding: plain + size: 0 + + proxy-response: + version: '2' + status: 404 + reason: "Not Found" + headers: + encoding: esc_json + fields: + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + proxy-response: + version: '2' + status: 404 + reason: "Not Found" + headers: + encoding: esc_json + fields: + - [ Content-Length, 0 ] + content: + encoding: plain + size: 0 + + # + # Test 4: 32 byte POST with a 200 response. + # + - all: { headers: { fields: [[ uuid, 8 ]]}} + + client-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 32 ] + content: + encoding: plain + size: 32 + + proxy-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 32 ] + content: + encoding: plain + size: 32 + + server-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 32 ] + content: + encoding: plain + size: 32 + + proxy-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 32 ] + content: + encoding: plain + size: 32 + + # + # Test 5: 3200 byte POST with a 200 response. + # + - all: { headers: { fields: [[ uuid, 9 ]]}} + + client-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 3200 ] + content: + encoding: plain + size: 3200 + + proxy-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 3200 ] + content: + encoding: plain + size: 3200 + + server-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ bob, 1600 ] + content: + encoding: plain + size: 1600 + + proxy-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 1600 ] + content: + encoding: plain + size: 1600 + + proxy-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 1600 ] + content: + encoding: plain + size: 1600 + + # + # Test 6: large post body small response + # + - all: { headers: { fields: [[ uuid, 10 ]]}} + + client-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + content: + encoding: plain + size: 3200 + + proxy-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 3200 ] + content: + encoding: plain + size: 3200 + + server-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 16 ] + content: + encoding: plain + size: 16 + + proxy-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 16 ] + content: + encoding: plain + size: 16 + + # + # Test 7: small post body large response + # + - all: { headers: { fields: [[ uuid, 11 ]]}} + + client-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 32 ] + content: + encoding: plain + size: 32 + + proxy-request: + protocol: + - name: http + version: '2' + - name: tls + version: TLSv1.2 + sni: data.brian.example.com + proxy-verify-mode: 1 + proxy-provided-cert: false + - name: tcp + - name: ip + version: '4' + + version: '2' + scheme: https + method: POST + url: /some/path4 + headers: + encoding: esc_json + fields: + - [ Host, data.brian.example.com ] + - [ Content-Length, 32 ] + content: + encoding: plain + size: 32 + + server-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 3200 ] + content: + encoding: plain + size: 3200 + + proxy-response: + version: '2' + status: 200 + reason: OK + headers: + encoding: esc_json + fields: + - [ Content-Length, 3200 ] + content: + encoding: plain + size: 3200 diff --git a/tests/gold_tests/timeout/tls_conn_timeout.test.py b/tests/gold_tests/timeout/tls_conn_timeout.test.py index 86da7ecc4ee..8187220c6cf 100644 --- a/tests/gold_tests/timeout/tls_conn_timeout.test.py +++ b/tests/gold_tests/timeout/tls_conn_timeout.test.py @@ -69,7 +69,7 @@ tr.Processes.Default.Command = 'curl -H"Connection:close" -d "bob" -i http://127.0.0.1:{0}/connect_blocked --tlsv1.2'.format( ts.Variables.port) tr.Processes.Default.Streams.All = Testers.ContainsExpression( - "HTTP/1.1 502 connect failed", "Connect failed") + "HTTP/1.1 502 Connection timed out", "Connect failed") tr.Processes.Default.ReturnCode = 0 tr.StillRunningAfter = delay_post_connect tr.StillRunningAfter = Test.Processes.ts @@ -93,7 +93,7 @@ tr.Processes.Default.Command = 'curl -H"Connection:close" -i http://127.0.0.1:{0}/get_connect_blocked --tlsv1.2'.format( ts.Variables.port) tr.Processes.Default.Streams.All = Testers.ContainsExpression( - "HTTP/1.1 502 connect failed", "Connect failed") + "HTTP/1.1 502 Connection timed out", "Connect failed") tr.Processes.Default.ReturnCode = 0 tr.StillRunningAfter = delay_get_connect