diff --git a/configure.ac b/configure.ac index d6e8a7d2261..f5566c06f23 100644 --- a/configure.ac +++ b/configure.ac @@ -2089,6 +2089,7 @@ AC_CHECK_MEMBER([struct sockaddr.sa_len], [], [], [#include ]) AC_CHECK_MEMBER([struct sockaddr_in.sin_len], [], [], [#include ]) AC_CHECK_MEMBER([struct sockaddr_in6.sin6_len], [], [], [#include ]) AC_CHECK_MEMBER([struct tcp_info.tcpi_data_segs_out], [], [], [#include ]) +AC_CHECK_MEMBER([struct mptcp_info.mptcpi_subflows], [], [], [#include ]) if test "x${ac_cv_member_struct_sockaddr_sa_len}" = "xyes"; then AC_DEFINE(HAVE_STRUCT_SOCKADDR_SA_LEN, 1, @@ -2110,6 +2111,11 @@ if test "x${ac_cv_member_struct_tcp_info_tcpi_data_segs_out}" = "xyes"; then [Whether struct tcp_info have the tcpi_data_segs_{in,out} member]) fi +if test "x${ac_cv_member_struct_mptcp_info_mptcpi_subflows}" = "xyes"; then + AC_DEFINE(HAVE_STRUCT_MPTCP_INFO_SUBFLOWS, 1, + [Whether struct mptcp_info have the mptcpi_subflows member]) +fi + if test "x${with_profiler}" = "xyes"; then AC_CHECK_HEADERS([gperftools/profiler.h \ ], [], []) diff --git a/include/tscore/ink_platform.h b/include/tscore/ink_platform.h index 922b474c57c..8aab32836d7 100644 --- a/include/tscore/ink_platform.h +++ b/include/tscore/ink_platform.h @@ -201,3 +201,17 @@ typedef unsigned int in_addr_t; #define MPTCP_ENABLED 0 #endif #endif + +// If kernel headers do not support IPPROTO_MPTCP definition +#ifndef IPPROTO_MPTCP +#define IPPROTO_MPTCP 262 +#endif + +#ifndef SOL_MPTCP +#define SOL_MPTCP 284 +#endif + +// Undefined in upstream until 5.16 +#ifndef MPTCP_INFO +#define MPTCP_INFO 1 +#endif diff --git a/iocore/net/Connection.cc b/iocore/net/Connection.cc index df6d51fc7b7..134e834fdeb 100644 --- a/iocore/net/Connection.cc +++ b/iocore/net/Connection.cc @@ -42,6 +42,7 @@ // #define SEND_BUF_SIZE (1024*64) #define FIRST_RANDOM_PORT 16000 #define LAST_RANDOM_PORT 32000 +#define MPTCP_V1 2 int get_listen_backlog() @@ -229,7 +230,7 @@ Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions #endif } - if ((opt.sockopt_flags & NetVCOptions::SOCK_OPT_NO_DELAY) && + if ((opt.sockopt_flags & NetVCOptions::SOCK_OPT_NO_DELAY) && (opt.f_mptcp != MPTCP_V1) && safe_setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, SOCKOPT_ON, sizeof(int)) < 0) { goto Lerror; } @@ -241,7 +242,7 @@ Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions } #ifdef TCP_FASTOPEN - if ((opt.sockopt_flags & NetVCOptions::SOCK_OPT_TCP_FAST_OPEN) && + if ((opt.sockopt_flags & NetVCOptions::SOCK_OPT_TCP_FAST_OPEN) && (opt.f_mptcp != MPTCP_V1) && safe_setsockopt(fd, IPPROTO_TCP, TCP_FASTOPEN, (char *)&opt.tfo_queue_length, sizeof(int))) { goto Lerror; } @@ -263,14 +264,14 @@ Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions } #if defined(TCP_MAXSEG) - if (NetProcessor::accept_mss > 0) { + if (NetProcessor::accept_mss > 0 && (opt.f_mptcp != MPTCP_V1)) { if (safe_setsockopt(fd, IPPROTO_TCP, TCP_MAXSEG, reinterpret_cast(&NetProcessor::accept_mss), sizeof(int)) < 0) { goto Lerror; } } #endif - if (opt.f_mptcp) { + if (opt.f_mptcp == 1) { #if MPTCP_ENABLED if (safe_setsockopt(fd, IPPROTO_TCP, MPTCP_ENABLED, SOCKOPT_ON, sizeof(int)) < 0) { Error("[Server::listen] Unable to enable MPTCP socket-option [%d] %s\n", errno, strerror(errno)); @@ -284,7 +285,8 @@ Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions #ifdef TCP_DEFER_ACCEPT // set tcp defer accept timeout if it is configured, this will not trigger an accept until there is // data on the socket ready to be read - if (opt.defer_accept > 0 && setsockopt(fd, IPPROTO_TCP, TCP_DEFER_ACCEPT, &opt.defer_accept, sizeof(int)) < 0) { + if (opt.defer_accept > 0 && (opt.f_mptcp != MPTCP_V1) && + setsockopt(fd, IPPROTO_TCP, TCP_DEFER_ACCEPT, &opt.defer_accept, sizeof(int)) < 0) { // FIXME: should we go to the error // goto error; Error("[Server::listen] Defer accept is configured but set failed: %d", errno); @@ -346,6 +348,7 @@ Server::listen(bool non_blocking, const NetProcessor::AcceptOptions &opt) ink_assert(fd == NO_FD); int res = 0; int namelen; + int prot = IPPROTO_TCP; if (!ats_is_ip(&accept_addr)) { ats_ip4_set(&addr, INADDR_ANY, 0); @@ -353,7 +356,12 @@ Server::listen(bool non_blocking, const NetProcessor::AcceptOptions &opt) ats_ip_copy(&addr, &accept_addr); } - fd = res = socketManager.socket(addr.sa.sa_family, SOCK_STREAM, IPPROTO_TCP); + if (opt.f_mptcp == MPTCP_V1) { + Debug("connection", "Define socket with MPTCP"); + prot = IPPROTO_MPTCP; + } + + fd = res = socketManager.socket(addr.sa.sa_family, SOCK_STREAM, prot); if (res < 0) { goto Lerror; } @@ -363,7 +371,7 @@ Server::listen(bool non_blocking, const NetProcessor::AcceptOptions &opt) goto Lerror; } - if ((res = socketManager.ink_bind(fd, &addr.sa, ats_ip_size(&addr.sa), IPPROTO_TCP)) < 0) { + if ((res = socketManager.ink_bind(fd, &addr.sa, ats_ip_size(&addr.sa), prot)) < 0) { goto Lerror; } @@ -390,6 +398,7 @@ Server::listen(bool non_blocking, const NetProcessor::AcceptOptions &opt) fd = NO_FD; } - Error("Could not bind or listen to port %d (error: %d)", ats_ip_port_host_order(&addr), res); + Fatal("Could not bind or listen to port %d, mptcp enabled: %d (error: %d) %s %d", ats_ip_port_host_order(&addr), + prot == IPPROTO_MPTCP, errno, strerror(errno), res); return res; } diff --git a/iocore/net/I_NetProcessor.h b/iocore/net/I_NetProcessor.h index 842ce082d3e..875d03b9b7c 100644 --- a/iocore/net/I_NetProcessor.h +++ b/iocore/net/I_NetProcessor.h @@ -101,7 +101,7 @@ class NetProcessor : public Processor @internal For logging and metrics purposes to know whether the listener enabled MPTCP or not. */ - bool f_mptcp; + int f_mptcp; /// Proxy Protocol enabled bool f_proxy_protocol; diff --git a/iocore/net/P_UnixNetVConnection.h b/iocore/net/P_UnixNetVConnection.h index 28a0f04166b..78fe030abd8 100644 --- a/iocore/net/P_UnixNetVConnection.h +++ b/iocore/net/P_UnixNetVConnection.h @@ -38,6 +38,10 @@ #include "P_NetAccept.h" #include "NetEvent.h" +#if HAVE_STRUCT_MPTCP_INFO_SUBFLOWS +#include +#endif + class UnixNetVConnection; class NetHandler; struct PollDescriptor; @@ -315,10 +319,29 @@ UnixNetVConnection::set_mptcp_state() if (0 == safe_getsockopt(con.fd, IPPROTO_TCP, MPTCP_ENABLED, (char *)&mptcp_enabled, &mptcp_enabled_size)) { Debug("socket_mptcp", "MPTCP socket state: %d", mptcp_enabled); - mptcp_state = mptcp_enabled > 0 ? true : false; + mptcp_state = (mptcp_enabled > 0); + return; + } else { + Debug("socket_mptcp", "MPTCP failed getsockopt(MPTCP_ENABLED): %s", strerror(errno)); + } + +#if defined(HAVE_STRUCT_MPTCP_INFO_SUBFLOWS) && defined(MPTCP_INFO) && MPTCP_INFO == 1 + struct mptcp_info minfo; + int minfo_len = sizeof(minfo); + + Debug("socket_mptcp", "MPTCP_INFO and struct mptcp_info defined"); + if (0 == safe_getsockopt(con.fd, SOL_MPTCP, MPTCP_INFO, &minfo, &minfo_len)) { + if (minfo_len > 0) { + Debug("socket_mptcp", "MPTCP socket state (remote key received): %ld", + (minfo.mptcpi_flags & MPTCP_INFO_FLAG_REMOTE_KEY_RECEIVED)); + mptcp_state = (minfo.mptcpi_flags & MPTCP_INFO_FLAG_REMOTE_KEY_RECEIVED); + return; + } } else { - Debug("socket_mptcp", "MPTCP failed getsockopt(): %s", strerror(errno)); + mptcp_state = 0; + Debug("socket_mptcp", "MPTCP failed getsockopt(%d, MPTCP_INFO): %s", con.fd, strerror(errno)); } +#endif } inline ink_hrtime diff --git a/iocore/net/UnixNetProcessor.cc b/iocore/net/UnixNetProcessor.cc index 4dd4c26eeb5..2f9aa600055 100644 --- a/iocore/net/UnixNetProcessor.cc +++ b/iocore/net/UnixNetProcessor.cc @@ -51,7 +51,7 @@ NetProcessor::AcceptOptions::reset() packet_notsent_lowat = 0; tfo_queue_length = 0; f_inbound_transparent = false; - f_mptcp = false; + f_mptcp = 0; f_proxy_protocol = false; return *this; } diff --git a/lib/records/I_RecHttp.h b/lib/records/I_RecHttp.h index 36f871447b0..eb57006f588 100644 --- a/lib/records/I_RecHttp.h +++ b/lib/records/I_RecHttp.h @@ -264,8 +264,8 @@ struct HttpProxyPort { bool m_outbound_transparent_p = false; // True if transparent pass-through is enabled on this port. bool m_transparent_passthrough = false; - /// True if MPTCP is enabled on this port. - bool m_mptcp = false; + /// [1, 2] if MPTCP is enabled on this port. + int m_mptcp = 0; /// Local address for inbound connections (listen address). IpAddr m_inbound_ip; /// Local address for outbound connections (to origin server). diff --git a/lib/records/RecHttp.cc b/lib/records/RecHttp.cc index 7e9a62d02d6..2484cda7bb1 100644 --- a/lib/records/RecHttp.cc +++ b/lib/records/RecHttp.cc @@ -85,20 +85,23 @@ SessionProtocolSet DEFAULT_NON_TLS_SESSION_PROTOCOL_SET; SessionProtocolSet DEFAULT_TLS_SESSION_PROTOCOL_SET; SessionProtocolSet DEFAULT_QUIC_SESSION_PROTOCOL_SET; -static bool +static int mptcp_supported() { ats_scoped_fd fd(::open("/proc/sys/net/mptcp/mptcp_enabled", O_RDONLY)); - int value = 0; - - if (fd) { - TextBuffer buffer(16); + // Newer kernel mptcp config + ats_scoped_fd fd_new(::open("/proc/sys/net/mptcp/enabled", O_RDONLY)); + TextBuffer buffer(16); + if (fd > 0) { buffer.slurp(fd.get()); - value = atoi(buffer.bufPtr()); + return (atoi(buffer.bufPtr()) != 0) ? 1 : 0; + } else if (fd_new > 0) { + buffer.slurp(fd_new.get()); + return (atoi(buffer.bufPtr()) != 0) ? 2 : 0; } - return value != 0; + return 0; } void @@ -441,9 +444,8 @@ HttpProxyPort::processOptions(const char *opts) Warning("Transparent pass-through requested [%s] in port descriptor '%s' but TPROXY was not configured.", item, opts); #endif } else if (0 == strcasecmp(OPT_MPTCP, item)) { - if (mptcp_supported()) { - m_mptcp = true; - } else { + m_mptcp = mptcp_supported(); + if (m_mptcp == 0) { Warning("Multipath TCP requested [%s] in port descriptor '%s' but it is not supported by this host.", item, opts); } } else if (nullptr != (value = this->checkPrefix(item, OPT_HOST_RES_PREFIX, OPT_HOST_RES_PREFIX_LEN))) { diff --git a/mgmt/LocalManager.cc b/mgmt/LocalManager.cc index 9a1a0d2f4b8..f4a93d847a2 100644 --- a/mgmt/LocalManager.cc +++ b/mgmt/LocalManager.cc @@ -1048,7 +1048,7 @@ LocalManager::bindTcpProxyPort(HttpProxyPort &port) } } - if (port.m_mptcp) { + if (port.m_mptcp == 1) { #if MPTCP_ENABLED int err;