From 4d9e1f0bd9006753fb1ea2a496acc9286c234779 Mon Sep 17 00:00:00 2001 From: Masaori Koshiba Date: Mon, 14 Dec 2020 13:31:57 +0900 Subject: [PATCH] Add Outbound PROXY Protocol (v1/v2) Support --- .../configuration/proxy-protocol.en.rst | 16 +++--- doc/admin-guide/files/records.config.en.rst | 15 +++++ include/ts/apidefs.h.in | 1 + iocore/net/I_NetVConnection.h | 4 ++ iocore/net/P_NetVConnection.h | 21 +++++++ iocore/net/SSLNetVConnection.cc | 29 ++++++++++ mgmt/RecordsConfig.cc | 2 + plugins/lua/ts_lua_http_config.c | 2 + proxy/http/HttpConfig.cc | 3 + proxy/http/HttpConfig.h | 1 + proxy/http/HttpSM.cc | 56 ++++++++++++++++++- src/shared/overridable_txn_vars.cc | 1 + src/traffic_server/InkAPI.cc | 3 + src/traffic_server/InkAPITest.cc | 1 + 14 files changed, 145 insertions(+), 10 deletions(-) diff --git a/doc/admin-guide/configuration/proxy-protocol.en.rst b/doc/admin-guide/configuration/proxy-protocol.en.rst index adf61f624bb..a83fe5b1375 100644 --- a/doc/admin-guide/configuration/proxy-protocol.en.rst +++ b/doc/admin-guide/configuration/proxy-protocol.en.rst @@ -31,12 +31,7 @@ TLS connections. .. note:: - The current version only supports transforming client IP from PROXY Version 1/2 - header to the Forwarded: header. - -In the current implementation, the client IP address in the PROXY protocol header -is passed to the origin server via an HTTP `Forwarded: -`_ header. + The current implementation doesn't support TLV fields of Version 2. The Proxy Protocol must be enabled on each port. See :ts:cv:`proxy.config.http.server_ports` for information on how to enable the @@ -52,11 +47,18 @@ configured with :ts:cv:`proxy.config.http.proxy_protocol_allowlist`. If the allowlist is configured, requests will only be accepted from these IP addresses and must be prefaced with the PROXY v1/v2 header. -See :ts:cv:`proxy.config.http.insert_forwarded` for configuration information. +1. HTTP Forwarded Header + +The client IP address in the PROXY protocol header is passed to the origin server via an HTTP `Forwarded: +`_ header. See :ts:cv:`proxy.config.http.insert_forwarded` for configuration information. Detection of the PROXY protocol header is automatic. If the PROXY header precludes the request, it will automatically be parse and made available to the Forwarded: request header sent to the origin server. +2. Outbound PROXY protocol + +See :ts:cv:`proxy.config.http.proxy_protocol_out` for configuration information. + Example ------- diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index 47e18806bd2..cf025bd2d84 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -1792,6 +1792,21 @@ Proxy User Variables See :ref:`proxy-protocol` for more discussion on how |TS| transforms the `Forwarded: header`. +.. ts:cv:: CONFIG proxy.config.http.proxy_protocol_out INT ``-1`` + :reloadable: + :overridable: + + Set the behavior of outbound PROXY Protocol. + + =========== ====================================================================== + Value Description + =========== ====================================================================== + ``-1`` Disable (default) + ``0`` Forward received PROXY protocol to the next hop + ``1`` Send client information in PROXY protocol version 1 + ``2`` Send client information in PROXY protocol version 2 + =========== ====================================================================== + .. ts:cv:: CONFIG proxy.config.http.normalize_ae INT 1 :reloadable: :overridable: diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index e8ee795feaf..93098b1f8a9 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -785,6 +785,7 @@ typedef enum { TS_CONFIG_HTTP_PARENT_CONNECT_ATTEMPT_TIMEOUT, TS_CONFIG_HTTP_NORMALIZE_AE, TS_CONFIG_HTTP_INSERT_FORWARDED, + TS_CONFIG_HTTP_PROXY_PROTOCOL_OUT, TS_CONFIG_HTTP_ALLOW_MULTI_RANGE, TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED, TS_CONFIG_HTTP_ALLOW_HALF_OPEN, diff --git a/iocore/net/I_NetVConnection.h b/iocore/net/I_NetVConnection.h index 1113b819c06..9f42e5b566b 100644 --- a/iocore/net/I_NetVConnection.h +++ b/iocore/net/I_NetVConnection.h @@ -623,6 +623,7 @@ class NetVConnection : public VConnection, public PluginUserArgsget_proxy_protocol_addr(ProxyProtocolData::DST)); }; + void set_proxy_protocol_info(const ProxyProtocol &src); + const ProxyProtocol &get_proxy_protocol_info() const; + bool has_proxy_protocol(IOBufferReader *); bool has_proxy_protocol(char *, int64_t *); diff --git a/iocore/net/P_NetVConnection.h b/iocore/net/P_NetVConnection.h index 3f379af2fb4..17538436fe1 100644 --- a/iocore/net/P_NetVConnection.h +++ b/iocore/net/P_NetVConnection.h @@ -59,6 +59,13 @@ NetVConnection::get_remote_port() return ats_ip_port_host_order(this->get_remote_addr()); } +inline IpEndpoint const & +NetVConnection::get_local_endpoint() +{ + get_local_addr(); + return local_addr; +} + inline sockaddr const * NetVConnection::get_local_addr() { @@ -99,3 +106,17 @@ NetVConnection::get_proxy_protocol_addr(const ProxyProtocolData src_or_dst) cons return nullptr; } + +inline void +NetVConnection::set_proxy_protocol_info(const ProxyProtocol &src) +{ + if (pp_info.version == ProxyProtocolVersion::UNDEFINED) { + pp_info = src; + } +} + +inline const ProxyProtocol & +NetVConnection::get_proxy_protocol_info() const +{ + return pp_info; +} diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc index f4eaaa840fa..fc2bde20019 100644 --- a/iocore/net/SSLNetVConnection.cc +++ b/iocore/net/SSLNetVConnection.cc @@ -1404,6 +1404,35 @@ SSLNetVConnection::sslClientHandShakeEvent(int &err) // Initialize properly for a client connection if (sslHandshakeHookState == HANDSHAKE_HOOKS_PRE) { + if (this->pp_info.version != ProxyProtocolVersion::UNDEFINED) { + // Outbound PROXY Protocol + VIO &vio = this->write.vio; + int64_t ntodo = vio.ntodo(); + int64_t towrite = vio.buffer.reader()->read_avail(); + + if (ntodo > 0 && towrite > 0) { + MIOBufferAccessor &buf = vio.buffer; + int needs = 0; + int64_t total_written = 0; + int64_t r = super::load_buffer_and_write(towrite, buf, total_written, needs); + + if (total_written > 0) { + vio.ndone += total_written; + if (vio.ntodo() != 0) { + return SSL_WAIT_FOR_HOOK; + } + } + + if (r < 0) { + if (r == -EAGAIN || r == -ENOTCONN || -r == EINPROGRESS) { + return SSL_WAIT_FOR_HOOK; + } else { + return EVENT_ERROR; + } + } + } + } + sslHandshakeHookState = HANDSHAKE_HOOKS_OUTBOUND_PRE; } diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index 833849a2037..a7f7db2e379 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -525,6 +525,8 @@ static const RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.http.proxy_protocol_allowlist", RECD_STRING, "none", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , + {RECT_CONFIG, "proxy.config.http.proxy_protocol_out", RECD_INT, "-1", RECU_DYNAMIC, RR_NULL, RECC_STR, "^-?[0-2]+$", RECA_NULL} + , {RECT_CONFIG, "proxy.config.http.insert_age_in_response", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} , {RECT_CONFIG, "proxy.config.http.enable_http_stats", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL} diff --git a/plugins/lua/ts_lua_http_config.c b/plugins/lua/ts_lua_http_config.c index 72146aa10e2..4f6f26b0f5b 100644 --- a/plugins/lua/ts_lua_http_config.c +++ b/plugins/lua/ts_lua_http_config.c @@ -41,6 +41,7 @@ typedef enum { TS_LUA_CONFIG_HTTP_RESPONSE_SERVER_ENABLED = TS_CONFIG_HTTP_RESPONSE_SERVER_ENABLED, TS_LUA_CONFIG_HTTP_INSERT_SQUID_X_FORWARDED_FOR = TS_CONFIG_HTTP_INSERT_SQUID_X_FORWARDED_FOR, TS_LUA_CONFIG_HTTP_INSERT_FORWARDED = TS_CONFIG_HTTP_INSERT_FORWARDED, + TS_LUA_CONFIG_HTTP_PROXY_PROTOCOL_OUT = TS_CONFIG_HTTP_PROXY_PROTOCOL_OUT, TS_LUA_CONFIG_HTTP_SEND_HTTP11_REQUESTS = TS_CONFIG_HTTP_SEND_HTTP11_REQUESTS, TS_LUA_CONFIG_HTTP_CACHE_HTTP = TS_CONFIG_HTTP_CACHE_HTTP, TS_LUA_CONFIG_HTTP_CACHE_IGNORE_CLIENT_NO_CACHE = TS_CONFIG_HTTP_CACHE_IGNORE_CLIENT_NO_CACHE, @@ -170,6 +171,7 @@ ts_lua_var_item ts_lua_http_config_vars[] = { TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_RESPONSE_SERVER_ENABLED), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_INSERT_SQUID_X_FORWARDED_FOR), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_INSERT_FORWARDED), + TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_PROXY_PROTOCOL_OUT), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_SEND_HTTP11_REQUESTS), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CACHE_HTTP), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CACHE_IGNORE_CLIENT_NO_CACHE), diff --git a/proxy/http/HttpConfig.cc b/proxy/http/HttpConfig.cc index 52a302e32ec..daafac8b563 100644 --- a/proxy/http/HttpConfig.cc +++ b/proxy/http/HttpConfig.cc @@ -1235,6 +1235,8 @@ HttpConfig::startup() HttpEstablishStaticConfigByte(c.oride.insert_squid_x_forwarded_for, "proxy.config.http.insert_squid_x_forwarded_for"); + HttpEstablishStaticConfigLongLong(c.oride.proxy_protocol_out, "proxy.config.http.proxy_protocol_out"); + HttpEstablishStaticConfigByte(c.oride.insert_age_in_response, "proxy.config.http.insert_age_in_response"); HttpEstablishStaticConfigByte(c.enable_http_stats, "proxy.config.http.enable_http_stats"); HttpEstablishStaticConfigByte(c.oride.normalize_ae, "proxy.config.http.normalize_ae"); @@ -1517,6 +1519,7 @@ HttpConfig::reconfigure() params->oride.insert_age_in_response = INT_TO_BOOL(m_master.oride.insert_age_in_response); params->enable_http_stats = INT_TO_BOOL(m_master.enable_http_stats); params->oride.normalize_ae = m_master.oride.normalize_ae; + params->oride.proxy_protocol_out = m_master.oride.proxy_protocol_out; params->oride.cache_heuristic_min_lifetime = m_master.oride.cache_heuristic_min_lifetime; params->oride.cache_heuristic_max_lifetime = m_master.oride.cache_heuristic_max_lifetime; diff --git a/proxy/http/HttpConfig.h b/proxy/http/HttpConfig.h index 06e1ccb2393..8bce9393f0c 100644 --- a/proxy/http/HttpConfig.h +++ b/proxy/http/HttpConfig.h @@ -518,6 +518,7 @@ struct OverridableHttpConfigParams { // Forwarded // /////////////// HttpForwarded::OptionBitSet insert_forwarded; + MgmtInt proxy_protocol_out = -1; ////////////////////// // Version Hell // diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 0bc9621485c..c8db1965f50 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -43,6 +43,9 @@ #include "P_SSLSNI.h" #include "HttpPages.h" #include "IPAllow.h" + +#include "ProxyProtocol.h" + #include "tscore/I_Layout.h" #include "tscore/bwf_std_format.h" #include "ts/sdt.h" @@ -127,6 +130,44 @@ milestone_update_api_time(TransactionMilestones &milestones, ink_hrtime &api_tim // Unique state machine identifier std::atomic next_sm_id(0); + +/** + Outbound PROXY Protocol + + Write PROXY Protocol to the first block of given MIOBuffer + FIXME: make @vc_in const + */ +int64_t +do_outbound_proxy_protocol(MIOBuffer *miob, NetVConnection *vc_out, NetVConnection *vc_in, int conf) +{ + ink_release_assert(conf >= 0); + + ProxyProtocol info = vc_in->get_proxy_protocol_info(); + ProxyProtocolVersion pp_version = proxy_protocol_version_cast(conf); + + if (info.version == ProxyProtocolVersion::UNDEFINED) { + if (conf == 0) { + // nothing to forward + return 0; + } else { + // set info from incoming NetVConnection + IpEndpoint local = vc_in->get_local_endpoint(); + info = ProxyProtocol{pp_version, local.family(), local, vc_in->get_remote_endpoint()}; + } + } + + vc_out->set_proxy_protocol_info(info); + + IOBufferBlock *block = miob->first_write_block(); + size_t len = proxy_protocol_build(reinterpret_cast(block->buf()), block->write_avail(), info, pp_version); + + if (len > 0) { + miob->fill(len); + } + + return len; +} + } // namespace ClassAllocator httpSMAllocator("httpSMAllocator"); @@ -1816,19 +1857,28 @@ HttpSM::state_http_server_open(int event, void *data) session->to_parent_proxy = true; HTTP_INCREMENT_DYN_STAT(http_current_parent_proxy_connections_stat); HTTP_INCREMENT_DYN_STAT(http_total_parent_proxy_connections_stat); - } else { session->to_parent_proxy = false; } + if (plugin_tunnel_type == HTTP_NO_PLUGIN_TUNNEL) { SMDebug("http", "[%" PRId64 "] setting handler for TCP handshake", sm_id); // Just want to get a write-ready event so we know that the TCP handshake is complete. server_entry->vc_handler = &HttpSM::state_http_server_open; - server_entry->write_vio = server_session->do_io_write(this, 1, server_session->get_reader()); - } else { // in the case of an intercept plugin don't to the connect timeout change + + int64_t nbytes = 1; + if (t_state.txn_conf->proxy_protocol_out >= 0) { + nbytes = + do_outbound_proxy_protocol(server_session->read_buffer, vc, ua_txn->get_netvc(), t_state.txn_conf->proxy_protocol_out); + } + + server_entry->write_vio = server_session->do_io_write(this, nbytes, server_session->get_reader()); + } else { + // in the case of an intercept plugin don't to the connect timeout change SMDebug("http", "[%" PRId64 "] not setting handler for TCP handshake", sm_id); handle_http_server_open(); } + return 0; } case VC_EVENT_READ_COMPLETE: diff --git a/src/shared/overridable_txn_vars.cc b/src/shared/overridable_txn_vars.cc index 3eb2a8dd976..667416ec3ca 100644 --- a/src/shared/overridable_txn_vars.cc +++ b/src/shared/overridable_txn_vars.cc @@ -57,6 +57,7 @@ const std::unordered_mapinsert_forwarded, conv); break; + case TS_CONFIG_HTTP_PROXY_PROTOCOL_OUT: + ret = _memberp_to_generic(&overridableHttpConfig->proxy_protocol_out, conv); + break; case TS_CONFIG_HTTP_SEND_HTTP11_REQUESTS: ret = _memberp_to_generic(&overridableHttpConfig->send_http11_requests, conv); break; diff --git a/src/traffic_server/InkAPITest.cc b/src/traffic_server/InkAPITest.cc index f8648307ed3..3405755723c 100644 --- a/src/traffic_server/InkAPITest.cc +++ b/src/traffic_server/InkAPITest.cc @@ -8682,6 +8682,7 @@ std::array SDK_Overridable_Configs = { "proxy.config.http.parent_proxy.connect_attempts_timeout", "proxy.config.http.normalize_ae", "proxy.config.http.insert_forwarded", + "proxy.config.http.proxy_protocol_out", "proxy.config.http.allow_multi_range", "proxy.config.http.request_buffer_enabled", "proxy.config.http.allow_half_open",