Add Outbound PROXY Protocol (v1/v2) Support

doc/admin-guide/configuration/proxy-protocol.en.rst

@@ -31,12 +31,7 @@ TLS connections.
 
 .. note::
 
-    The current version only supports transforming client IP from PROXY Version 1/2
-    header to the Forwarded: header.
-
-In the current implementation, the client IP address in the PROXY protocol header
-is passed to the origin server via an HTTP `Forwarded:
-<https://tools.ietf.org/html/rfc7239>`_ header.
+    The current implementation doesn't support TLV fields of Version 2.
 
 The Proxy Protocol must be enabled on each port.  See
 :ts:cv:`proxy.config.http.server_ports` for information on how to enable the
@@ -52,11 +47,18 @@ configured with :ts:cv:`proxy.config.http.proxy_protocol_allowlist`.
        If the allowlist is configured, requests will only be accepted from these
        IP addresses and must be prefaced with the PROXY v1/v2 header.
 
-See :ts:cv:`proxy.config.http.insert_forwarded` for configuration information.
+1. HTTP Forwarded Header
+
+The client IP address in the PROXY protocol header is passed to the origin server via an HTTP `Forwarded:
+<https://tools.ietf.org/html/rfc7239>`_ header. See :ts:cv:`proxy.config.http.insert_forwarded` for configuration information.
 Detection of the PROXY protocol header is automatic.  If the PROXY header
 precludes the request, it will automatically be parse and made available to the
 Forwarded: request header sent to the origin server.
 
+2. Outbound PROXY protocol
+
+See :ts:cv:`proxy.config.http.proxy_protocol_out` for configuration information.
+
 Example
 -------
 

doc/admin-guide/files/records.config.en.rst

@@ -1792,6 +1792,21 @@ Proxy User Variables
 
    See :ref:`proxy-protocol` for more discussion on how |TS| transforms the `Forwarded: header`.
 
+.. ts:cv:: CONFIG proxy.config.http.proxy_protocol_out INT ``-1``
+   :reloadable:
+   :overridable:
+
+   Set the behavior of outbound PROXY Protocol.
+
+   =========== ======================================================================
+   Value       Description
+   =========== ======================================================================
+   ``-1``      Disable (default)
+   ``0``       Forward received PROXY protocol to the next hop
+   ``1``       Send client information in PROXY protocol version 1
+   ``2``       Send client information in PROXY protocol version 2
+   =========== ======================================================================
+
 .. ts:cv:: CONFIG proxy.config.http.normalize_ae INT 1
    :reloadable:
    :overridable:

include/ts/apidefs.h.in

@@ -785,6 +785,7 @@ typedef enum {
   TS_CONFIG_HTTP_PARENT_CONNECT_ATTEMPT_TIMEOUT,
   TS_CONFIG_HTTP_NORMALIZE_AE,
   TS_CONFIG_HTTP_INSERT_FORWARDED,
+  TS_CONFIG_HTTP_PROXY_PROTOCOL_OUT,
   TS_CONFIG_HTTP_ALLOW_MULTI_RANGE,
   TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED,
   TS_CONFIG_HTTP_ALLOW_HALF_OPEN,

iocore/net/I_NetVConnection.h

@@ -623,6 +623,7 @@ class NetVConnection : public VConnection, public PluginUserArgs<TS_USER_ARGS_VC
 
   /** Returns local sockaddr storage. */
   sockaddr const *get_local_addr();
+  IpEndpoint const &get_local_endpoint();
 
   /** Returns local ip.
       @deprecated get_local_addr() should be used instead for AF_INET6 compatibility.
@@ -840,6 +841,9 @@ class NetVConnection : public VConnection, public PluginUserArgs<TS_USER_ARGS_VC
     return ats_ip_port_host_order(this->get_proxy_protocol_addr(ProxyProtocolData::DST));
   };
 
+  void set_proxy_protocol_info(const ProxyProtocol &src);
+  const ProxyProtocol &get_proxy_protocol_info() const;
+
   bool has_proxy_protocol(IOBufferReader *);
   bool has_proxy_protocol(char *, int64_t *);
 

iocore/net/P_NetVConnection.h

@@ -59,6 +59,13 @@ NetVConnection::get_remote_port()
   return ats_ip_port_host_order(this->get_remote_addr());
 }
 
+inline IpEndpoint const &
+NetVConnection::get_local_endpoint()
+{
+  get_local_addr();
+  return local_addr;
+}
+
 inline sockaddr const *
 NetVConnection::get_local_addr()
 {
@@ -99,3 +106,17 @@ NetVConnection::get_proxy_protocol_addr(const ProxyProtocolData src_or_dst) cons
 
   return nullptr;
 }
+
+inline void
+NetVConnection::set_proxy_protocol_info(const ProxyProtocol &src)
+{
+  if (pp_info.version == ProxyProtocolVersion::UNDEFINED) {
+    pp_info = src;
+  }
+}
+
+inline const ProxyProtocol &
+NetVConnection::get_proxy_protocol_info() const
+{
+  return pp_info;
+}

iocore/net/SSLNetVConnection.cc

@@ -1404,6 +1404,35 @@ SSLNetVConnection::sslClientHandShakeEvent(int &err)
 
   // Initialize properly for a client connection
   if (sslHandshakeHookState == HANDSHAKE_HOOKS_PRE) {
+    if (this->pp_info.version != ProxyProtocolVersion::UNDEFINED) {
+      // Outbound PROXY Protocol
+      VIO &vio        = this->write.vio;
+      int64_t ntodo   = vio.ntodo();
+      int64_t towrite = vio.buffer.reader()->read_avail();
+
+      if (ntodo > 0 && towrite > 0) {
+        MIOBufferAccessor &buf = vio.buffer;
+        int needs              = 0;
+        int64_t total_written  = 0;
+        int64_t r              = super::load_buffer_and_write(towrite, buf, total_written, needs);
+
+        if (total_written > 0) {
+          vio.ndone += total_written;
+          if (vio.ntodo() != 0) {
+            return SSL_WAIT_FOR_HOOK;
+          }
+        }
+
+        if (r < 0) {
+          if (r == -EAGAIN || r == -ENOTCONN || -r == EINPROGRESS) {
+            return SSL_WAIT_FOR_HOOK;
+          } else {
+            return EVENT_ERROR;
+          }
+        }
+      }
+    }
+
     sslHandshakeHookState = HANDSHAKE_HOOKS_OUTBOUND_PRE;
   }
 

mgmt/RecordsConfig.cc

@@ -525,6 +525,8 @@ static const RecordElement RecordsConfig[] =
   ,
   {RECT_CONFIG, "proxy.config.http.proxy_protocol_allowlist", RECD_STRING, "none", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
   ,
+  {RECT_CONFIG, "proxy.config.http.proxy_protocol_out", RECD_INT, "-1", RECU_DYNAMIC, RR_NULL, RECC_STR, "^-?[0-2]+$", RECA_NULL}
+  ,
   {RECT_CONFIG, "proxy.config.http.insert_age_in_response", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL}
   ,
   {RECT_CONFIG, "proxy.config.http.enable_http_stats", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-1]", RECA_NULL}

plugins/lua/ts_lua_http_config.c

@@ -41,6 +41,7 @@ typedef enum {
   TS_LUA_CONFIG_HTTP_RESPONSE_SERVER_ENABLED                  = TS_CONFIG_HTTP_RESPONSE_SERVER_ENABLED,
   TS_LUA_CONFIG_HTTP_INSERT_SQUID_X_FORWARDED_FOR             = TS_CONFIG_HTTP_INSERT_SQUID_X_FORWARDED_FOR,
   TS_LUA_CONFIG_HTTP_INSERT_FORWARDED                         = TS_CONFIG_HTTP_INSERT_FORWARDED,
+  TS_LUA_CONFIG_HTTP_PROXY_PROTOCOL_OUT                       = TS_CONFIG_HTTP_PROXY_PROTOCOL_OUT,
   TS_LUA_CONFIG_HTTP_SEND_HTTP11_REQUESTS                     = TS_CONFIG_HTTP_SEND_HTTP11_REQUESTS,
   TS_LUA_CONFIG_HTTP_CACHE_HTTP                               = TS_CONFIG_HTTP_CACHE_HTTP,
   TS_LUA_CONFIG_HTTP_CACHE_IGNORE_CLIENT_NO_CACHE             = TS_CONFIG_HTTP_CACHE_IGNORE_CLIENT_NO_CACHE,
@@ -170,6 +171,7 @@ ts_lua_var_item ts_lua_http_config_vars[] = {
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_RESPONSE_SERVER_ENABLED),
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_INSERT_SQUID_X_FORWARDED_FOR),
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_INSERT_FORWARDED),
+  TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_PROXY_PROTOCOL_OUT),
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_SEND_HTTP11_REQUESTS),
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CACHE_HTTP),
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CACHE_IGNORE_CLIENT_NO_CACHE),

proxy/http/HttpConfig.cc

@@ -1235,6 +1235,8 @@ HttpConfig::startup()
 
   HttpEstablishStaticConfigByte(c.oride.insert_squid_x_forwarded_for, "proxy.config.http.insert_squid_x_forwarded_for");
 
+  HttpEstablishStaticConfigLongLong(c.oride.proxy_protocol_out, "proxy.config.http.proxy_protocol_out");
+
   HttpEstablishStaticConfigByte(c.oride.insert_age_in_response, "proxy.config.http.insert_age_in_response");
   HttpEstablishStaticConfigByte(c.enable_http_stats, "proxy.config.http.enable_http_stats");
   HttpEstablishStaticConfigByte(c.oride.normalize_ae, "proxy.config.http.normalize_ae");
@@ -1517,6 +1519,7 @@ HttpConfig::reconfigure()
   params->oride.insert_age_in_response       = INT_TO_BOOL(m_master.oride.insert_age_in_response);
   params->enable_http_stats                  = INT_TO_BOOL(m_master.enable_http_stats);
   params->oride.normalize_ae                 = m_master.oride.normalize_ae;
+  params->oride.proxy_protocol_out           = m_master.oride.proxy_protocol_out;
 
   params->oride.cache_heuristic_min_lifetime = m_master.oride.cache_heuristic_min_lifetime;
   params->oride.cache_heuristic_max_lifetime = m_master.oride.cache_heuristic_max_lifetime;

proxy/http/HttpConfig.h

@@ -518,6 +518,7 @@ struct OverridableHttpConfigParams {
   // Forwarded //
   ///////////////
   HttpForwarded::OptionBitSet insert_forwarded;
+  MgmtInt proxy_protocol_out = -1;
 
   //////////////////////
   //  Version Hell    //

proxy/http/HttpSM.cc

@@ -43,6 +43,9 @@
 #include "P_SSLSNI.h"
 #include "HttpPages.h"
 #include "IPAllow.h"
+
+#include "ProxyProtocol.h"
+
 #include "tscore/I_Layout.h"
 #include "tscore/bwf_std_format.h"
 #include "ts/sdt.h"
@@ -127,6 +130,44 @@ milestone_update_api_time(TransactionMilestones &milestones, ink_hrtime &api_tim
 
 // Unique state machine identifier
 std::atomic<int64_t> next_sm_id(0);
+
+/**
+   Outbound PROXY Protocol
+
+   Write PROXY Protocol to the first block of given MIOBuffer
+   FIXME: make @vc_in const
+ */
+int64_t
+do_outbound_proxy_protocol(MIOBuffer *miob, NetVConnection *vc_out, NetVConnection *vc_in, int conf)
+{
+  ink_release_assert(conf >= 0);
+
+  ProxyProtocol info              = vc_in->get_proxy_protocol_info();
+  ProxyProtocolVersion pp_version = proxy_protocol_version_cast(conf);
+
+  if (info.version == ProxyProtocolVersion::UNDEFINED) {
+    if (conf == 0) {
+      // nothing to forward
+      return 0;
+    } else {
+      // set info from incoming NetVConnection
+      IpEndpoint local = vc_in->get_local_endpoint();
+      info             = ProxyProtocol{pp_version, local.family(), local, vc_in->get_remote_endpoint()};
+    }
+  }
+
+  vc_out->set_proxy_protocol_info(info);
+
+  IOBufferBlock *block = miob->first_write_block();
+  size_t len           = proxy_protocol_build(reinterpret_cast<uint8_t *>(block->buf()), block->write_avail(), info, pp_version);
+
+  if (len > 0) {
+    miob->fill(len);
+  }
+
+  return len;
+}
+
 } // namespace
 
 ClassAllocator<HttpSM> httpSMAllocator("httpSMAllocator");
@@ -1816,19 +1857,28 @@ HttpSM::state_http_server_open(int event, void *data)
       session->to_parent_proxy = true;
       HTTP_INCREMENT_DYN_STAT(http_current_parent_proxy_connections_stat);
       HTTP_INCREMENT_DYN_STAT(http_total_parent_proxy_connections_stat);
-
     } else {
       session->to_parent_proxy = false;
     }
+
     if (plugin_tunnel_type == HTTP_NO_PLUGIN_TUNNEL) {
       SMDebug("http", "[%" PRId64 "] setting handler for TCP handshake", sm_id);
       // Just want to get a write-ready event so we know that the TCP handshake is complete.
       server_entry->vc_handler = &HttpSM::state_http_server_open;
-      server_entry->write_vio  = server_session->do_io_write(this, 1, server_session->get_reader());
-    } else { // in the case of an intercept plugin don't to the connect timeout change
+
+      int64_t nbytes = 1;
+      if (t_state.txn_conf->proxy_protocol_out >= 0) {
+        nbytes =
+          do_outbound_proxy_protocol(server_session->read_buffer, vc, ua_txn->get_netvc(), t_state.txn_conf->proxy_protocol_out);
+      }
+
+      server_entry->write_vio = server_session->do_io_write(this, nbytes, server_session->get_reader());
+    } else {
+      // in the case of an intercept plugin don't to the connect timeout change
       SMDebug("http", "[%" PRId64 "] not setting handler for TCP handshake", sm_id);
       handle_http_server_open();
     }
+
     return 0;
   }
   case VC_EVENT_READ_COMPLETE:

src/shared/overridable_txn_vars.cc

@@ -57,6 +57,7 @@ const std::unordered_map<std::string_view, std::tuple<const TSOverridableConfigK
      {"proxy.config.http.forward_connect_method", {TS_CONFIG_HTTP_FORWARD_CONNECT_METHOD, TS_RECORDDATATYPE_INT}},
      {"proxy.config.http.request_buffer_enabled", {TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED, TS_RECORDDATATYPE_INT}},
      {"proxy.config.http.down_server.cache_time", {TS_CONFIG_HTTP_DOWN_SERVER_CACHE_TIME, TS_RECORDDATATYPE_INT}},
+     {"proxy.config.http.proxy_protocol_out", {TS_CONFIG_HTTP_PROXY_PROTOCOL_OUT, TS_RECORDDATATYPE_INT}},
      {"proxy.config.http.insert_age_in_response", {TS_CONFIG_HTTP_INSERT_AGE_IN_RESPONSE, TS_RECORDDATATYPE_INT}},
      {"proxy.config.url_remap.pristine_host_hdr", {TS_CONFIG_URL_REMAP_PRISTINE_HOST_HDR, TS_RECORDDATATYPE_INT}},
      {"proxy.config.http.insert_request_via_str", {TS_CONFIG_HTTP_INSERT_REQUEST_VIA_STR, TS_RECORDDATATYPE_INT}},

src/traffic_server/InkAPI.cc

@@ -8500,6 +8500,9 @@ _conf_to_memberp(TSOverridableConfigKey conf, OverridableHttpConfigParams *overr
   case TS_CONFIG_HTTP_INSERT_FORWARDED:
     ret = _memberp_to_generic(&overridableHttpConfig->insert_forwarded, conv);
     break;
+  case TS_CONFIG_HTTP_PROXY_PROTOCOL_OUT:
+    ret = _memberp_to_generic(&overridableHttpConfig->proxy_protocol_out, conv);
+    break;
   case TS_CONFIG_HTTP_SEND_HTTP11_REQUESTS:
     ret = _memberp_to_generic(&overridableHttpConfig->send_http11_requests, conv);
     break;

src/traffic_server/InkAPITest.cc

@@ -8682,6 +8682,7 @@ std::array<std::string_view, TS_CONFIG_LAST_ENTRY> SDK_Overridable_Configs = {
    "proxy.config.http.parent_proxy.connect_attempts_timeout",
    "proxy.config.http.normalize_ae",
    "proxy.config.http.insert_forwarded",
+   "proxy.config.http.proxy_protocol_out",
    "proxy.config.http.allow_multi_range",
    "proxy.config.http.request_buffer_enabled",
    "proxy.config.http.allow_half_open",