[9.2.x] Support upstream mptcp

configure.ac

@@ -2089,6 +2089,7 @@ AC_CHECK_MEMBER([struct sockaddr.sa_len], [], [], [#include <netinet/in.h>])
 AC_CHECK_MEMBER([struct sockaddr_in.sin_len], [], [], [#include <netinet/in.h>])
 AC_CHECK_MEMBER([struct sockaddr_in6.sin6_len], [], [], [#include <netinet/in.h>])
 AC_CHECK_MEMBER([struct tcp_info.tcpi_data_segs_out], [], [], [#include <linux/tcp.h>])
+AC_CHECK_MEMBER([struct mptcp_info.mptcpi_subflows], [], [], [#include <linux/mptcp.h>])
 
 if test "x${ac_cv_member_struct_sockaddr_sa_len}" = "xyes"; then
     AC_DEFINE(HAVE_STRUCT_SOCKADDR_SA_LEN, 1,
@@ -2110,6 +2111,11 @@ if test "x${ac_cv_member_struct_tcp_info_tcpi_data_segs_out}" = "xyes"; then
             [Whether struct tcp_info have the tcpi_data_segs_{in,out} member])
 fi
 
+if test "x${ac_cv_member_struct_mptcp_info_mptcpi_subflows}" = "xyes"; then
+    AC_DEFINE(HAVE_STRUCT_MPTCP_INFO_SUBFLOWS, 1,
+            [Whether struct mptcp_info have the mptcpi_subflows member])
+fi
+
 if test "x${with_profiler}" = "xyes"; then
 AC_CHECK_HEADERS([gperftools/profiler.h \
                   ], [], [])

include/tscore/ink_platform.h

@@ -201,3 +201,17 @@ typedef unsigned int in_addr_t;
 #define MPTCP_ENABLED 0
 #endif
 #endif
+
+// If kernel headers do not support IPPROTO_MPTCP definition
+#ifndef IPPROTO_MPTCP
+#define IPPROTO_MPTCP 262
+#endif
+
+#ifndef SOL_MPTCP
+#define SOL_MPTCP 284
+#endif
+
+// Undefined in upstream until 5.16
+#ifndef MPTCP_INFO
+#define MPTCP_INFO 1
+#endif

iocore/net/Connection.cc

@@ -42,6 +42,7 @@
 // #define SEND_BUF_SIZE            (1024*64)
 #define FIRST_RANDOM_PORT 16000
 #define LAST_RANDOM_PORT 32000
+#define MPTCP_V1 2
 
 int
 get_listen_backlog()
@@ -229,7 +230,7 @@ Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions
 #endif
   }
 
-  if ((opt.sockopt_flags & NetVCOptions::SOCK_OPT_NO_DELAY) &&
+  if ((opt.sockopt_flags & NetVCOptions::SOCK_OPT_NO_DELAY) && (opt.f_mptcp != MPTCP_V1) &&
       safe_setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, SOCKOPT_ON, sizeof(int)) < 0) {
     goto Lerror;
   }
@@ -241,7 +242,7 @@ Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions
   }
 
 #ifdef TCP_FASTOPEN
-  if ((opt.sockopt_flags & NetVCOptions::SOCK_OPT_TCP_FAST_OPEN) &&
+  if ((opt.sockopt_flags & NetVCOptions::SOCK_OPT_TCP_FAST_OPEN) && (opt.f_mptcp != MPTCP_V1) &&
       safe_setsockopt(fd, IPPROTO_TCP, TCP_FASTOPEN, (char *)&opt.tfo_queue_length, sizeof(int))) {
     goto Lerror;
   }
@@ -263,14 +264,14 @@ Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions
   }
 
 #if defined(TCP_MAXSEG)
-  if (NetProcessor::accept_mss > 0) {
+  if (NetProcessor::accept_mss > 0 && (opt.f_mptcp != MPTCP_V1)) {
     if (safe_setsockopt(fd, IPPROTO_TCP, TCP_MAXSEG, reinterpret_cast<char *>(&NetProcessor::accept_mss), sizeof(int)) < 0) {
       goto Lerror;
     }
   }
 #endif
 
-  if (opt.f_mptcp) {
+  if (opt.f_mptcp == 1) {
 #if MPTCP_ENABLED
     if (safe_setsockopt(fd, IPPROTO_TCP, MPTCP_ENABLED, SOCKOPT_ON, sizeof(int)) < 0) {
       Error("[Server::listen] Unable to enable MPTCP socket-option [%d] %s\n", errno, strerror(errno));
@@ -284,7 +285,8 @@ Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions
 #ifdef TCP_DEFER_ACCEPT
   // set tcp defer accept timeout if it is configured, this will not trigger an accept until there is
   // data on the socket ready to be read
-  if (opt.defer_accept > 0 && setsockopt(fd, IPPROTO_TCP, TCP_DEFER_ACCEPT, &opt.defer_accept, sizeof(int)) < 0) {
+  if (opt.defer_accept > 0 && (opt.f_mptcp != MPTCP_V1) &&
+      setsockopt(fd, IPPROTO_TCP, TCP_DEFER_ACCEPT, &opt.defer_accept, sizeof(int)) < 0) {
     // FIXME: should we go to the error
     // goto error;
     Error("[Server::listen] Defer accept is configured but set failed: %d", errno);
@@ -346,14 +348,20 @@ Server::listen(bool non_blocking, const NetProcessor::AcceptOptions &opt)
   ink_assert(fd == NO_FD);
   int res = 0;
   int namelen;
+  int prot = IPPROTO_TCP;
 
   if (!ats_is_ip(&accept_addr)) {
     ats_ip4_set(&addr, INADDR_ANY, 0);
   } else {
     ats_ip_copy(&addr, &accept_addr);
   }
 
-  fd = res = socketManager.socket(addr.sa.sa_family, SOCK_STREAM, IPPROTO_TCP);
+  if (opt.f_mptcp == MPTCP_V1) {
+    Debug("connection", "Define socket with MPTCP");
+    prot = IPPROTO_MPTCP;
+  }
+
+  fd = res = socketManager.socket(addr.sa.sa_family, SOCK_STREAM, prot);
   if (res < 0) {
     goto Lerror;
   }
@@ -363,7 +371,7 @@ Server::listen(bool non_blocking, const NetProcessor::AcceptOptions &opt)
     goto Lerror;
   }
 
-  if ((res = socketManager.ink_bind(fd, &addr.sa, ats_ip_size(&addr.sa), IPPROTO_TCP)) < 0) {
+  if ((res = socketManager.ink_bind(fd, &addr.sa, ats_ip_size(&addr.sa), prot)) < 0) {
     goto Lerror;
   }
 
@@ -390,6 +398,7 @@ Server::listen(bool non_blocking, const NetProcessor::AcceptOptions &opt)
     fd = NO_FD;
   }
 
-  Error("Could not bind or listen to port %d (error: %d)", ats_ip_port_host_order(&addr), res);
+  Fatal("Could not bind or listen to port %d, mptcp enabled: %d (error: %d) %s %d", ats_ip_port_host_order(&addr),
+        prot == IPPROTO_MPTCP, errno, strerror(errno), res);
   return res;
 }

iocore/net/I_NetProcessor.h

@@ -101,7 +101,7 @@ class NetProcessor : public Processor
         @internal For logging and metrics purposes to know whether the
         listener enabled MPTCP or not.
     */
-    bool f_mptcp;
+    int f_mptcp;
 
     /// Proxy Protocol enabled
     bool f_proxy_protocol;

iocore/net/P_UnixNetVConnection.h

@@ -38,6 +38,10 @@
 #include "P_NetAccept.h"
 #include "NetEvent.h"
 
+#if HAVE_STRUCT_MPTCP_INFO_SUBFLOWS
+#include <linux/mptcp.h>
+#endif
+
 class UnixNetVConnection;
 class NetHandler;
 struct PollDescriptor;
@@ -315,10 +319,29 @@ UnixNetVConnection::set_mptcp_state()
 
   if (0 == safe_getsockopt(con.fd, IPPROTO_TCP, MPTCP_ENABLED, (char *)&mptcp_enabled, &mptcp_enabled_size)) {
     Debug("socket_mptcp", "MPTCP socket state: %d", mptcp_enabled);
-    mptcp_state = mptcp_enabled > 0 ? true : false;
+    mptcp_state = (mptcp_enabled > 0);
+    return;
+  } else {
+    Debug("socket_mptcp", "MPTCP failed getsockopt(MPTCP_ENABLED): %s", strerror(errno));
+  }
+
+#if defined(HAVE_STRUCT_MPTCP_INFO_SUBFLOWS) && defined(MPTCP_INFO) && MPTCP_INFO == 1
+  struct mptcp_info minfo;
+  int minfo_len = sizeof(minfo);
+
+  Debug("socket_mptcp", "MPTCP_INFO and struct mptcp_info defined");
+  if (0 == safe_getsockopt(con.fd, SOL_MPTCP, MPTCP_INFO, &minfo, &minfo_len)) {
+    if (minfo_len > 0) {
+      Debug("socket_mptcp", "MPTCP socket state (remote key received): %ld",
+            (minfo.mptcpi_flags & MPTCP_INFO_FLAG_REMOTE_KEY_RECEIVED));
+      mptcp_state = (minfo.mptcpi_flags & MPTCP_INFO_FLAG_REMOTE_KEY_RECEIVED);
+      return;
+    }
   } else {
-    Debug("socket_mptcp", "MPTCP failed getsockopt(): %s", strerror(errno));
+    mptcp_state = 0;
+    Debug("socket_mptcp", "MPTCP failed getsockopt(%d, MPTCP_INFO): %s", con.fd, strerror(errno));
   }
+#endif
 }
 
 inline ink_hrtime

iocore/net/UnixNetProcessor.cc

@@ -51,7 +51,7 @@ NetProcessor::AcceptOptions::reset()
   packet_notsent_lowat  = 0;
   tfo_queue_length      = 0;
   f_inbound_transparent = false;
-  f_mptcp               = false;
+  f_mptcp               = 0;
   f_proxy_protocol      = false;
   return *this;
 }

lib/records/I_RecHttp.h

@@ -264,8 +264,8 @@ struct HttpProxyPort {
   bool m_outbound_transparent_p = false;
   // True if transparent pass-through is enabled on this port.
   bool m_transparent_passthrough = false;
-  /// True if MPTCP is enabled on this port.
-  bool m_mptcp = false;
+  /// [1, 2] if MPTCP is enabled on this port.
+  int m_mptcp = 0;
   /// Local address for inbound connections (listen address).
   IpAddr m_inbound_ip;
   /// Local address for outbound connections (to origin server).

lib/records/RecHttp.cc

@@ -85,20 +85,23 @@ SessionProtocolSet DEFAULT_NON_TLS_SESSION_PROTOCOL_SET;
 SessionProtocolSet DEFAULT_TLS_SESSION_PROTOCOL_SET;
 SessionProtocolSet DEFAULT_QUIC_SESSION_PROTOCOL_SET;
 
-static bool
+static int
 mptcp_supported()
 {
   ats_scoped_fd fd(::open("/proc/sys/net/mptcp/mptcp_enabled", O_RDONLY));
-  int value = 0;
-
-  if (fd) {
-    TextBuffer buffer(16);
+  // Newer kernel mptcp config
+  ats_scoped_fd fd_new(::open("/proc/sys/net/mptcp/enabled", O_RDONLY));
+  TextBuffer buffer(16);
 
+  if (fd > 0) {
     buffer.slurp(fd.get());
-    value = atoi(buffer.bufPtr());
+    return (atoi(buffer.bufPtr()) != 0) ? 1 : 0;
+  } else if (fd_new > 0) {
+    buffer.slurp(fd_new.get());
+    return (atoi(buffer.bufPtr()) != 0) ? 2 : 0;
   }
 
-  return value != 0;
+  return 0;
 }
 
 void
@@ -441,9 +444,8 @@ HttpProxyPort::processOptions(const char *opts)
       Warning("Transparent pass-through requested [%s] in port descriptor '%s' but TPROXY was not configured.", item, opts);
 #endif
     } else if (0 == strcasecmp(OPT_MPTCP, item)) {
-      if (mptcp_supported()) {
-        m_mptcp = true;
-      } else {
+      m_mptcp = mptcp_supported();
+      if (m_mptcp == 0) {
         Warning("Multipath TCP requested [%s] in port descriptor '%s' but it is not supported by this host.", item, opts);
       }
     } else if (nullptr != (value = this->checkPrefix(item, OPT_HOST_RES_PREFIX, OPT_HOST_RES_PREFIX_LEN))) {

mgmt/LocalManager.cc

@@ -1048,7 +1048,7 @@ LocalManager::bindTcpProxyPort(HttpProxyPort &port)
     }
   }
 
-  if (port.m_mptcp) {
+  if (port.m_mptcp == 1) {
 #if MPTCP_ENABLED
     int err;