From b7ae9f7f6348072e4e475454c0300a729a2ea724 Mon Sep 17 00:00:00 2001 From: Serris Lew Date: Tue, 7 Nov 2023 10:42:33 -0800 Subject: [PATCH 1/4] [9.2.x] Support upstream mptcp --- configure.ac | 6 ++++++ include/tscore/ink_platform.h | 14 ++++++++++++++ iocore/net/Connection.cc | 32 ++++++++++++++----------------- iocore/net/P_UnixNetVConnection.h | 24 ++++++++++++++++++++++- lib/records/RecHttp.cc | 10 +++++++--- mgmt/LocalManager.cc | 17 ---------------- 6 files changed, 64 insertions(+), 39 deletions(-) diff --git a/configure.ac b/configure.ac index d6e8a7d2261..f5566c06f23 100644 --- a/configure.ac +++ b/configure.ac @@ -2089,6 +2089,7 @@ AC_CHECK_MEMBER([struct sockaddr.sa_len], [], [], [#include ]) AC_CHECK_MEMBER([struct sockaddr_in.sin_len], [], [], [#include ]) AC_CHECK_MEMBER([struct sockaddr_in6.sin6_len], [], [], [#include ]) AC_CHECK_MEMBER([struct tcp_info.tcpi_data_segs_out], [], [], [#include ]) +AC_CHECK_MEMBER([struct mptcp_info.mptcpi_subflows], [], [], [#include ]) if test "x${ac_cv_member_struct_sockaddr_sa_len}" = "xyes"; then AC_DEFINE(HAVE_STRUCT_SOCKADDR_SA_LEN, 1, @@ -2110,6 +2111,11 @@ if test "x${ac_cv_member_struct_tcp_info_tcpi_data_segs_out}" = "xyes"; then [Whether struct tcp_info have the tcpi_data_segs_{in,out} member]) fi +if test "x${ac_cv_member_struct_mptcp_info_mptcpi_subflows}" = "xyes"; then + AC_DEFINE(HAVE_STRUCT_MPTCP_INFO_SUBFLOWS, 1, + [Whether struct mptcp_info have the mptcpi_subflows member]) +fi + if test "x${with_profiler}" = "xyes"; then AC_CHECK_HEADERS([gperftools/profiler.h \ ], [], []) diff --git a/include/tscore/ink_platform.h b/include/tscore/ink_platform.h index 922b474c57c..8aab32836d7 100644 --- a/include/tscore/ink_platform.h +++ b/include/tscore/ink_platform.h @@ -201,3 +201,17 @@ typedef unsigned int in_addr_t; #define MPTCP_ENABLED 0 #endif #endif + +// If kernel headers do not support IPPROTO_MPTCP definition +#ifndef IPPROTO_MPTCP +#define IPPROTO_MPTCP 262 +#endif + +#ifndef SOL_MPTCP +#define SOL_MPTCP 284 +#endif + +// Undefined in upstream until 5.16 +#ifndef MPTCP_INFO +#define MPTCP_INFO 1 +#endif diff --git a/iocore/net/Connection.cc b/iocore/net/Connection.cc index df6d51fc7b7..8888338cb5e 100644 --- a/iocore/net/Connection.cc +++ b/iocore/net/Connection.cc @@ -229,7 +229,7 @@ Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions #endif } - if ((opt.sockopt_flags & NetVCOptions::SOCK_OPT_NO_DELAY) && + if ((opt.sockopt_flags & NetVCOptions::SOCK_OPT_NO_DELAY) && !opt.f_mptcp && safe_setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, SOCKOPT_ON, sizeof(int)) < 0) { goto Lerror; } @@ -241,7 +241,7 @@ Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions } #ifdef TCP_FASTOPEN - if ((opt.sockopt_flags & NetVCOptions::SOCK_OPT_TCP_FAST_OPEN) && + if ((opt.sockopt_flags & NetVCOptions::SOCK_OPT_TCP_FAST_OPEN) && !opt.f_mptcp && safe_setsockopt(fd, IPPROTO_TCP, TCP_FASTOPEN, (char *)&opt.tfo_queue_length, sizeof(int))) { goto Lerror; } @@ -263,28 +263,17 @@ Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions } #if defined(TCP_MAXSEG) - if (NetProcessor::accept_mss > 0) { + if (NetProcessor::accept_mss > 0 && !opt.f_mptcp) { if (safe_setsockopt(fd, IPPROTO_TCP, TCP_MAXSEG, reinterpret_cast(&NetProcessor::accept_mss), sizeof(int)) < 0) { goto Lerror; } } #endif - if (opt.f_mptcp) { -#if MPTCP_ENABLED - if (safe_setsockopt(fd, IPPROTO_TCP, MPTCP_ENABLED, SOCKOPT_ON, sizeof(int)) < 0) { - Error("[Server::listen] Unable to enable MPTCP socket-option [%d] %s\n", errno, strerror(errno)); - goto Lerror; - } -#else - Error("[Server::listen] Multipath TCP requested but not configured on this host\n"); -#endif - } - #ifdef TCP_DEFER_ACCEPT // set tcp defer accept timeout if it is configured, this will not trigger an accept until there is // data on the socket ready to be read - if (opt.defer_accept > 0 && setsockopt(fd, IPPROTO_TCP, TCP_DEFER_ACCEPT, &opt.defer_accept, sizeof(int)) < 0) { + if (opt.defer_accept > 0 && !opt.f_mptcp && setsockopt(fd, IPPROTO_TCP, TCP_DEFER_ACCEPT, &opt.defer_accept, sizeof(int)) < 0) { // FIXME: should we go to the error // goto error; Error("[Server::listen] Defer accept is configured but set failed: %d", errno); @@ -346,6 +335,7 @@ Server::listen(bool non_blocking, const NetProcessor::AcceptOptions &opt) ink_assert(fd == NO_FD); int res = 0; int namelen; + int prot = IPPROTO_TCP; if (!ats_is_ip(&accept_addr)) { ats_ip4_set(&addr, INADDR_ANY, 0); @@ -353,7 +343,12 @@ Server::listen(bool non_blocking, const NetProcessor::AcceptOptions &opt) ats_ip_copy(&addr, &accept_addr); } - fd = res = socketManager.socket(addr.sa.sa_family, SOCK_STREAM, IPPROTO_TCP); + if (opt.f_mptcp) { + Debug("connection", "Define socket with MPTCP"); + prot = IPPROTO_MPTCP; + } + + fd = res = socketManager.socket(addr.sa.sa_family, SOCK_STREAM, prot); if (res < 0) { goto Lerror; } @@ -363,7 +358,7 @@ Server::listen(bool non_blocking, const NetProcessor::AcceptOptions &opt) goto Lerror; } - if ((res = socketManager.ink_bind(fd, &addr.sa, ats_ip_size(&addr.sa), IPPROTO_TCP)) < 0) { + if ((res = socketManager.ink_bind(fd, &addr.sa, ats_ip_size(&addr.sa), prot)) < 0) { goto Lerror; } @@ -390,6 +385,7 @@ Server::listen(bool non_blocking, const NetProcessor::AcceptOptions &opt) fd = NO_FD; } - Error("Could not bind or listen to port %d (error: %d)", ats_ip_port_host_order(&addr), res); + Fatal("Could not bind or listen to port %d, mptcp enabled: %d (error: %d) %s %d", ats_ip_port_host_order(&addr), + prot == IPPROTO_MPTCP, errno, strerror(errno), res); return res; } diff --git a/iocore/net/P_UnixNetVConnection.h b/iocore/net/P_UnixNetVConnection.h index 28a0f04166b..7547e6bd9a8 100644 --- a/iocore/net/P_UnixNetVConnection.h +++ b/iocore/net/P_UnixNetVConnection.h @@ -38,6 +38,10 @@ #include "P_NetAccept.h" #include "NetEvent.h" +#if HAVE_STRUCT_MPTCP_INFO_SUBFLOWS +#include +#endif + class UnixNetVConnection; class NetHandler; struct PollDescriptor; @@ -317,8 +321,26 @@ UnixNetVConnection::set_mptcp_state() Debug("socket_mptcp", "MPTCP socket state: %d", mptcp_enabled); mptcp_state = mptcp_enabled > 0 ? true : false; } else { - Debug("socket_mptcp", "MPTCP failed getsockopt(): %s", strerror(errno)); + Debug("socket_mptcp", "MPTCP failed getsockopt(MPTCP_ENABLED): %s", strerror(errno)); + } + +#if defined(HAVE_STRUCT_MPTCP_INFO_SUBFLOWS) && defined(MPTCP_INFO) && MPTCP_INFO == 1 + struct mptcp_info minfo; + int minfo_len = sizeof(minfo); + + Debug("socket_mptcp", "MPTCP_INFO and struct mptcp_info defined"); + if (0 == safe_getsockopt(con.fd, SOL_MPTCP, MPTCP_INFO, &minfo, &minfo_len)) { + if (minfo_len > 0) { + Debug("socket_mptcp", "MPTCP socket state (remote key received): %d", + (minfo.mptcpi_flags & MPTCP_INFO_FLAG_REMOTE_KEY_RECEIVED)); + mptcp_state = (minfo.mptcpi_flags & MPTCP_INFO_FLAG_REMOTE_KEY_RECEIVED); + return; + } + } else { + mptcp_state = 0; + Debug("socket_mptcp", "MPTCP failed getsockopt(%d, MPTCP_INFO): %s", con.fd, strerror(errno)); } +#endif } inline ink_hrtime diff --git a/lib/records/RecHttp.cc b/lib/records/RecHttp.cc index 7e9a62d02d6..dfd871e3855 100644 --- a/lib/records/RecHttp.cc +++ b/lib/records/RecHttp.cc @@ -89,13 +89,17 @@ static bool mptcp_supported() { ats_scoped_fd fd(::open("/proc/sys/net/mptcp/mptcp_enabled", O_RDONLY)); + // Newer kernel mptcp config + ats_scoped_fd fd_new(::open("/proc/sys/net/mptcp/enabled", O_RDONLY)); int value = 0; + TextBuffer buffer(16); - if (fd) { - TextBuffer buffer(16); - + if (fd > 0) { buffer.slurp(fd.get()); value = atoi(buffer.bufPtr()); + } else if (fd_new > 0) { + buffer.slurp(fd_new.get()); + value = atoi(buffer.bufPtr()); } return value != 0; diff --git a/mgmt/LocalManager.cc b/mgmt/LocalManager.cc index 9a1a0d2f4b8..dd8733786a8 100644 --- a/mgmt/LocalManager.cc +++ b/mgmt/LocalManager.cc @@ -1048,23 +1048,6 @@ LocalManager::bindTcpProxyPort(HttpProxyPort &port) } } - if (port.m_mptcp) { -#if MPTCP_ENABLED - int err; - - err = setsockopt(port.m_fd, IPPROTO_TCP, MPTCP_ENABLED, &one, sizeof(one)); - if (err < 0) { - mgmt_log("[bindProxyPort] Unable to enable MPTCP: %s\n", strerror(errno)); - Debug("lm_mptcp", "[bindProxyPort] Unable to enable MPTCP: %s", strerror(errno)); - } else { - mgmt_log("[bindProxyPort] Successfully enabled MPTCP on %d\n", port.m_port); - Debug("lm_mptcp", "[bindProxyPort] Successfully enabled MPTCP on %d\n", port.m_port); - } -#else - Debug("lm_mptcp", "[bindProxyPort] Multipath TCP requested but not configured on this host"); -#endif - } - if (port.m_family == AF_INET6) { if (setsockopt(port.m_fd, IPPROTO_IPV6, IPV6_V6ONLY, SOCKOPT_ON, sizeof(int)) < 0) { mgmt_log("[bindProxyPort] Unable to set socket options: %d : %s\n", port.m_port, strerror(errno)); From e3a4d808ac1a422802b6e699c3382aad5e4c0cef Mon Sep 17 00:00:00 2001 From: Serris Lew Date: Tue, 7 Nov 2023 12:55:00 -0800 Subject: [PATCH 2/4] debug format fix --- iocore/net/P_UnixNetVConnection.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iocore/net/P_UnixNetVConnection.h b/iocore/net/P_UnixNetVConnection.h index 7547e6bd9a8..34a7c525183 100644 --- a/iocore/net/P_UnixNetVConnection.h +++ b/iocore/net/P_UnixNetVConnection.h @@ -331,7 +331,7 @@ UnixNetVConnection::set_mptcp_state() Debug("socket_mptcp", "MPTCP_INFO and struct mptcp_info defined"); if (0 == safe_getsockopt(con.fd, SOL_MPTCP, MPTCP_INFO, &minfo, &minfo_len)) { if (minfo_len > 0) { - Debug("socket_mptcp", "MPTCP socket state (remote key received): %d", + Debug("socket_mptcp", "MPTCP socket state (remote key received): %ld", (minfo.mptcpi_flags & MPTCP_INFO_FLAG_REMOTE_KEY_RECEIVED)); mptcp_state = (minfo.mptcpi_flags & MPTCP_INFO_FLAG_REMOTE_KEY_RECEIVED); return; From 31f1a81a66605116f099a3223ad7d6f523389118 Mon Sep 17 00:00:00 2001 From: Serris Lew Date: Wed, 8 Nov 2023 13:55:45 -0800 Subject: [PATCH 3/4] return after setting state --- iocore/net/P_UnixNetVConnection.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/iocore/net/P_UnixNetVConnection.h b/iocore/net/P_UnixNetVConnection.h index 34a7c525183..78fe030abd8 100644 --- a/iocore/net/P_UnixNetVConnection.h +++ b/iocore/net/P_UnixNetVConnection.h @@ -319,7 +319,8 @@ UnixNetVConnection::set_mptcp_state() if (0 == safe_getsockopt(con.fd, IPPROTO_TCP, MPTCP_ENABLED, (char *)&mptcp_enabled, &mptcp_enabled_size)) { Debug("socket_mptcp", "MPTCP socket state: %d", mptcp_enabled); - mptcp_state = mptcp_enabled > 0 ? true : false; + mptcp_state = (mptcp_enabled > 0); + return; } else { Debug("socket_mptcp", "MPTCP failed getsockopt(MPTCP_ENABLED): %s", strerror(errno)); } From 2c5403afb240d4abebad2de0df064d79aad90552 Mon Sep 17 00:00:00 2001 From: Serris Lew Date: Tue, 14 Nov 2023 11:24:45 -0800 Subject: [PATCH 4/4] keep old mptcp stack --- iocore/net/Connection.cc | 23 ++++++++++++++++++----- iocore/net/I_NetProcessor.h | 2 +- iocore/net/UnixNetProcessor.cc | 2 +- lib/records/I_RecHttp.h | 4 ++-- lib/records/RecHttp.cc | 14 ++++++-------- mgmt/LocalManager.cc | 17 +++++++++++++++++ 6 files changed, 45 insertions(+), 17 deletions(-) diff --git a/iocore/net/Connection.cc b/iocore/net/Connection.cc index 8888338cb5e..134e834fdeb 100644 --- a/iocore/net/Connection.cc +++ b/iocore/net/Connection.cc @@ -42,6 +42,7 @@ // #define SEND_BUF_SIZE (1024*64) #define FIRST_RANDOM_PORT 16000 #define LAST_RANDOM_PORT 32000 +#define MPTCP_V1 2 int get_listen_backlog() @@ -229,7 +230,7 @@ Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions #endif } - if ((opt.sockopt_flags & NetVCOptions::SOCK_OPT_NO_DELAY) && !opt.f_mptcp && + if ((opt.sockopt_flags & NetVCOptions::SOCK_OPT_NO_DELAY) && (opt.f_mptcp != MPTCP_V1) && safe_setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, SOCKOPT_ON, sizeof(int)) < 0) { goto Lerror; } @@ -241,7 +242,7 @@ Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions } #ifdef TCP_FASTOPEN - if ((opt.sockopt_flags & NetVCOptions::SOCK_OPT_TCP_FAST_OPEN) && !opt.f_mptcp && + if ((opt.sockopt_flags & NetVCOptions::SOCK_OPT_TCP_FAST_OPEN) && (opt.f_mptcp != MPTCP_V1) && safe_setsockopt(fd, IPPROTO_TCP, TCP_FASTOPEN, (char *)&opt.tfo_queue_length, sizeof(int))) { goto Lerror; } @@ -263,17 +264,29 @@ Server::setup_fd_for_listen(bool non_blocking, const NetProcessor::AcceptOptions } #if defined(TCP_MAXSEG) - if (NetProcessor::accept_mss > 0 && !opt.f_mptcp) { + if (NetProcessor::accept_mss > 0 && (opt.f_mptcp != MPTCP_V1)) { if (safe_setsockopt(fd, IPPROTO_TCP, TCP_MAXSEG, reinterpret_cast(&NetProcessor::accept_mss), sizeof(int)) < 0) { goto Lerror; } } #endif + if (opt.f_mptcp == 1) { +#if MPTCP_ENABLED + if (safe_setsockopt(fd, IPPROTO_TCP, MPTCP_ENABLED, SOCKOPT_ON, sizeof(int)) < 0) { + Error("[Server::listen] Unable to enable MPTCP socket-option [%d] %s\n", errno, strerror(errno)); + goto Lerror; + } +#else + Error("[Server::listen] Multipath TCP requested but not configured on this host\n"); +#endif + } + #ifdef TCP_DEFER_ACCEPT // set tcp defer accept timeout if it is configured, this will not trigger an accept until there is // data on the socket ready to be read - if (opt.defer_accept > 0 && !opt.f_mptcp && setsockopt(fd, IPPROTO_TCP, TCP_DEFER_ACCEPT, &opt.defer_accept, sizeof(int)) < 0) { + if (opt.defer_accept > 0 && (opt.f_mptcp != MPTCP_V1) && + setsockopt(fd, IPPROTO_TCP, TCP_DEFER_ACCEPT, &opt.defer_accept, sizeof(int)) < 0) { // FIXME: should we go to the error // goto error; Error("[Server::listen] Defer accept is configured but set failed: %d", errno); @@ -343,7 +356,7 @@ Server::listen(bool non_blocking, const NetProcessor::AcceptOptions &opt) ats_ip_copy(&addr, &accept_addr); } - if (opt.f_mptcp) { + if (opt.f_mptcp == MPTCP_V1) { Debug("connection", "Define socket with MPTCP"); prot = IPPROTO_MPTCP; } diff --git a/iocore/net/I_NetProcessor.h b/iocore/net/I_NetProcessor.h index 842ce082d3e..875d03b9b7c 100644 --- a/iocore/net/I_NetProcessor.h +++ b/iocore/net/I_NetProcessor.h @@ -101,7 +101,7 @@ class NetProcessor : public Processor @internal For logging and metrics purposes to know whether the listener enabled MPTCP or not. */ - bool f_mptcp; + int f_mptcp; /// Proxy Protocol enabled bool f_proxy_protocol; diff --git a/iocore/net/UnixNetProcessor.cc b/iocore/net/UnixNetProcessor.cc index 4dd4c26eeb5..2f9aa600055 100644 --- a/iocore/net/UnixNetProcessor.cc +++ b/iocore/net/UnixNetProcessor.cc @@ -51,7 +51,7 @@ NetProcessor::AcceptOptions::reset() packet_notsent_lowat = 0; tfo_queue_length = 0; f_inbound_transparent = false; - f_mptcp = false; + f_mptcp = 0; f_proxy_protocol = false; return *this; } diff --git a/lib/records/I_RecHttp.h b/lib/records/I_RecHttp.h index 36f871447b0..eb57006f588 100644 --- a/lib/records/I_RecHttp.h +++ b/lib/records/I_RecHttp.h @@ -264,8 +264,8 @@ struct HttpProxyPort { bool m_outbound_transparent_p = false; // True if transparent pass-through is enabled on this port. bool m_transparent_passthrough = false; - /// True if MPTCP is enabled on this port. - bool m_mptcp = false; + /// [1, 2] if MPTCP is enabled on this port. + int m_mptcp = 0; /// Local address for inbound connections (listen address). IpAddr m_inbound_ip; /// Local address for outbound connections (to origin server). diff --git a/lib/records/RecHttp.cc b/lib/records/RecHttp.cc index dfd871e3855..2484cda7bb1 100644 --- a/lib/records/RecHttp.cc +++ b/lib/records/RecHttp.cc @@ -85,24 +85,23 @@ SessionProtocolSet DEFAULT_NON_TLS_SESSION_PROTOCOL_SET; SessionProtocolSet DEFAULT_TLS_SESSION_PROTOCOL_SET; SessionProtocolSet DEFAULT_QUIC_SESSION_PROTOCOL_SET; -static bool +static int mptcp_supported() { ats_scoped_fd fd(::open("/proc/sys/net/mptcp/mptcp_enabled", O_RDONLY)); // Newer kernel mptcp config ats_scoped_fd fd_new(::open("/proc/sys/net/mptcp/enabled", O_RDONLY)); - int value = 0; TextBuffer buffer(16); if (fd > 0) { buffer.slurp(fd.get()); - value = atoi(buffer.bufPtr()); + return (atoi(buffer.bufPtr()) != 0) ? 1 : 0; } else if (fd_new > 0) { buffer.slurp(fd_new.get()); - value = atoi(buffer.bufPtr()); + return (atoi(buffer.bufPtr()) != 0) ? 2 : 0; } - return value != 0; + return 0; } void @@ -445,9 +444,8 @@ HttpProxyPort::processOptions(const char *opts) Warning("Transparent pass-through requested [%s] in port descriptor '%s' but TPROXY was not configured.", item, opts); #endif } else if (0 == strcasecmp(OPT_MPTCP, item)) { - if (mptcp_supported()) { - m_mptcp = true; - } else { + m_mptcp = mptcp_supported(); + if (m_mptcp == 0) { Warning("Multipath TCP requested [%s] in port descriptor '%s' but it is not supported by this host.", item, opts); } } else if (nullptr != (value = this->checkPrefix(item, OPT_HOST_RES_PREFIX, OPT_HOST_RES_PREFIX_LEN))) { diff --git a/mgmt/LocalManager.cc b/mgmt/LocalManager.cc index dd8733786a8..f4a93d847a2 100644 --- a/mgmt/LocalManager.cc +++ b/mgmt/LocalManager.cc @@ -1048,6 +1048,23 @@ LocalManager::bindTcpProxyPort(HttpProxyPort &port) } } + if (port.m_mptcp == 1) { +#if MPTCP_ENABLED + int err; + + err = setsockopt(port.m_fd, IPPROTO_TCP, MPTCP_ENABLED, &one, sizeof(one)); + if (err < 0) { + mgmt_log("[bindProxyPort] Unable to enable MPTCP: %s\n", strerror(errno)); + Debug("lm_mptcp", "[bindProxyPort] Unable to enable MPTCP: %s", strerror(errno)); + } else { + mgmt_log("[bindProxyPort] Successfully enabled MPTCP on %d\n", port.m_port); + Debug("lm_mptcp", "[bindProxyPort] Successfully enabled MPTCP on %d\n", port.m_port); + } +#else + Debug("lm_mptcp", "[bindProxyPort] Multipath TCP requested but not configured on this host"); +#endif + } + if (port.m_family == AF_INET6) { if (setsockopt(port.m_fd, IPPROTO_IPV6, IPV6_V6ONLY, SOCKOPT_ON, sizeof(int)) < 0) { mgmt_log("[bindProxyPort] Unable to set socket options: %d : %s\n", port.m_port, strerror(errno));