Use io_uring for sockets on Linux

src/libraries/Common/src/Interop/Unix/System.Native/Interop.Fcntl.cs

@@ -22,6 +22,9 @@ internal static partial class Fcntl
             [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_FcntlSetFD", SetLastError = true)]
             internal static partial int SetFD(SafeHandle fd, int flags);
 
+            [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_FcntlSetFD", SetLastError = true)]
+            internal static partial int SetFD(IntPtr fd, int flags);
+
             [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_FcntlGetFD", SetLastError = true)]
             internal static partial int GetFD(SafeHandle fd);
 

src/libraries/Common/src/Interop/Unix/System.Native/Interop.IoUringShim.cs

@@ -0,0 +1,58 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Runtime.InteropServices;
+
+internal static partial class Interop
+{
+    internal static partial class Sys
+    {
+        /// <summary>Wraps io_uring_setup(2): creates an io_uring instance.</summary>
+        [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_IoUringShimSetup")]
+        internal static unsafe partial Error IoUringShimSetup(
+            uint entries, void* parms, int* ringFd);
+
+        /// <summary>Wraps io_uring_enter(2): submits SQEs and/or waits for CQEs.</summary>
+        [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_IoUringShimEnter")]
+        internal static unsafe partial Error IoUringShimEnter(
+            int ringFd, uint toSubmit, uint minComplete, uint flags, int* result);
+
+        /// <summary>Wraps io_uring_enter2(2) with IORING_ENTER_EXT_ARG for bounded waits.</summary>
+        [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_IoUringShimEnterExt")]
+        internal static unsafe partial Error IoUringShimEnterExt(
+            int ringFd, uint toSubmit, uint minComplete, uint flags, void* arg, int* result);
+
+        /// <summary>Wraps io_uring_register(2): registers resources (files, buffers, ring fds).</summary>
+        [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_IoUringShimRegister")]
+        internal static unsafe partial Error IoUringShimRegister(
+            int ringFd, uint opcode, void* arg, uint nrArgs, int* result);
+
+        /// <summary>Wraps mmap(2): maps io_uring SQ/CQ ring memory.</summary>
+        [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_IoUringShimMmap")]
+        internal static unsafe partial Error IoUringShimMmap(
+            int ringFd, ulong size, ulong offset, void** mappedPtr);
+
+        /// <summary>Wraps munmap(2): unmaps io_uring ring memory.</summary>
+        [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_IoUringShimMunmap")]
+        internal static unsafe partial Error IoUringShimMunmap(
+            void* addr, ulong size);
+
+        /// <summary>Creates an eventfd for io_uring wakeup signaling.</summary>
+        [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_IoUringShimCreateEventFd")]
+        internal static unsafe partial Error IoUringShimCreateEventFd(
+            int* eventFd);
+
+        /// <summary>Writes to an eventfd to wake the io_uring event loop.</summary>
+        [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_IoUringShimWriteEventFd")]
+        internal static partial Error IoUringShimWriteEventFd(int eventFd);
+
+        /// <summary>Reads from an eventfd to consume a wakeup signal.</summary>
+        [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_IoUringShimReadEventFd")]
+        internal static unsafe partial Error IoUringShimReadEventFd(
+            int eventFd, ulong* value);
+
+        /// <summary>Wraps close(2): closes a file descriptor.</summary>
+        [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_IoUringShimCloseFd")]
+        internal static partial Error IoUringShimCloseFd(int fd);
+    }
+}

src/libraries/Common/src/Interop/Unix/System.Native/Interop.SocketEvent.Linux.cs

@@ -0,0 +1,150 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Net.Sockets;
+using System.Runtime.InteropServices;
+
+internal static partial class Interop
+{
+    internal static partial class Sys
+    {
+        /// <summary>Derived SQ ring state computed after mmap, used by the managed submission path.</summary>
+        [StructLayout(LayoutKind.Sequential)]
+        internal struct IoUringSqRingInfo
+        {
+            public IntPtr SqeBase;
+            public IntPtr SqTailPtr;
+            public IntPtr SqHeadPtr;
+            public uint SqMask;
+            public uint SqEntries;
+            public uint SqeSize;
+            public byte UsesNoSqArray;
+            public int RingFd;
+            public int RegisteredRingFd;
+            public byte UsesEnterExtArg;
+            public byte UsesRegisteredFiles;
+        }
+
+        /// <summary>Mirrors kernel <c>struct io_sqring_offsets</c> (40 bytes). Fields at offset 28+ (resv1, user_addr) are unused.</summary>
+        [StructLayout(LayoutKind.Explicit, Size = 40)]
+        internal struct IoUringSqOffsets
+        {
+            [FieldOffset(0)]  public uint Head;
+            [FieldOffset(4)]  public uint Tail;
+            [FieldOffset(8)]  public uint RingMask;
+            [FieldOffset(12)] public uint RingEntries;
+            [FieldOffset(16)] public uint Flags;
+            [FieldOffset(20)] public uint Dropped;
+            [FieldOffset(24)] public uint Array;
+            // resv1 at 28, user_addr at 32 - not needed by managed code
+        }
+
+        /// <summary>Mirrors kernel <c>struct io_cqring_offsets</c> (40 bytes). Fields at offset 28+ (resv1, user_addr) are unused.</summary>
+        [StructLayout(LayoutKind.Explicit, Size = 40)]
+        internal struct IoUringCqOffsets
+        {
+            [FieldOffset(0)]  public uint Head;
+            [FieldOffset(4)]  public uint Tail;
+            [FieldOffset(8)]  public uint RingMask;
+            [FieldOffset(12)] public uint RingEntries;
+            [FieldOffset(16)] public uint Overflow;
+            [FieldOffset(20)] public uint Cqes;
+            [FieldOffset(24)] public uint Flags;
+            // resv1 at 28, user_addr at 32 - not needed by managed code
+        }
+
+        /// <summary>Mirrors kernel <c>struct io_uring_params</c> (120 bytes), passed to io_uring_setup.</summary>
+        [StructLayout(LayoutKind.Explicit, Size = 120)]
+        internal struct IoUringParams
+        {
+            [FieldOffset(0)]  public uint SqEntries;
+            [FieldOffset(4)]  public uint CqEntries;
+            [FieldOffset(8)]  public uint Flags;
+            [FieldOffset(12)] public uint SqThreadCpu;
+            [FieldOffset(16)] public uint SqThreadIdle;
+            [FieldOffset(20)] public uint Features;
+            [FieldOffset(24)] public uint WqFd;
+            // resv[3] at 28-39
+            [FieldOffset(40)] public IoUringSqOffsets SqOff;
+            [FieldOffset(80)] public IoUringCqOffsets CqOff;
+        }
+
+        /// <summary>Mirrors kernel <c>struct io_uring_cqe</c> (16 bytes), read from the CQ ring.</summary>
+        [StructLayout(LayoutKind.Explicit, Size = 16)]
+        internal struct IoUringCqe
+        {
+            [FieldOffset(0)]  public ulong UserData;
+            [FieldOffset(8)]  public int Result;
+            [FieldOffset(12)] public uint Flags;
+        }
+
+        /// <summary>Mirrors kernel <c>struct io_uring_buf</c> (16 bytes), used by provided-buffer rings.</summary>
+        [StructLayout(LayoutKind.Explicit, Size = 16)]
+        internal struct IoUringBuf
+        {
+            [FieldOffset(0)]  public ulong Address;
+            [FieldOffset(8)]  public uint Length;
+            [FieldOffset(12)] public ushort BufferId;
+            [FieldOffset(14)] public ushort Reserved;
+        }
+
+        /// <summary>
+        /// Mirrors the header overlay of kernel <c>struct io_uring_buf_ring</c> (16 bytes).
+        /// In UAPI this shares offset 0 with the first <c>io_uring_buf</c> entry via a union.
+        /// </summary>
+        [StructLayout(LayoutKind.Explicit, Size = 16)]
+        internal struct IoUringBufRingHeader
+        {
+            [FieldOffset(0)]  public ulong Reserved1;
+            [FieldOffset(8)]  public uint Reserved2;
+            [FieldOffset(12)] public ushort Reserved3;
+            [FieldOffset(14)] public ushort Tail;
+        }
+
+        /// <summary>Mirrors kernel <c>struct io_uring_buf_reg</c> (40 bytes), used for pbuf ring registration.</summary>
+        [StructLayout(LayoutKind.Explicit, Size = 40)]
+        internal struct IoUringBufReg
+        {
+            [FieldOffset(0)]  public ulong RingAddress;
+            [FieldOffset(8)]  public uint RingEntries;
+            [FieldOffset(12)] public ushort BufferGroupId;
+            [FieldOffset(14)] public ushort Padding;
+            [FieldOffset(16)] public ulong Reserved0;
+            [FieldOffset(24)] public ulong Reserved1;
+            [FieldOffset(32)] public ulong Reserved2;
+        }
+
+        /// <summary>Derived CQ ring state computed after mmap, used by the managed completion drain path.</summary>
+        [StructLayout(LayoutKind.Sequential)]
+        internal struct IoUringCqRingInfo
+        {
+            public IntPtr CqeBase;       // io_uring_cqe* base of CQE array
+            public IntPtr CqTailPtr;     // uint32_t* kernel writes CQ tail
+            public IntPtr CqHeadPtr;     // uint32_t* managed advances CQ head
+            public uint CqMask;          // CqEntries - 1
+            public uint CqEntries;       // number of CQ slots
+            public uint CqeSize;         // sizeof(io_uring_cqe) = 16
+            public IntPtr CqOverflowPtr; // uint32_t* kernel CQ overflow counter
+        }
+
+        /// <summary>Mirrors kernel <c>struct io_uring_getevents_arg</c>, used with IORING_ENTER_EXT_ARG.</summary>
+        [StructLayout(LayoutKind.Sequential)]
+        internal struct IoUringGeteventsArg
+        {
+            public ulong Sigmask;
+            public uint SigmaskSize;
+            public uint MinWaitUsec;
+            public ulong Ts;
+        }
+
+        /// <summary>Mirrors kernel <c>struct __kernel_timespec</c>, used for io_uring timeout arguments.</summary>
+        [StructLayout(LayoutKind.Sequential)]
+        internal struct IoUringKernelTimespec
+        {
+            public long TvSec;
+            public long TvNsec;
+        }
+
+    }
+}

src/libraries/System.Net.Sockets/src/System.Net.Sockets.csproj

@@ -197,9 +197,34 @@
 
   <ItemGroup Condition="'$(TargetPlatformIdentifier)' == 'unix' or '$(TargetPlatformIdentifier)' == 'osx' or '$(TargetPlatformIdentifier)' == 'ios' or '$(TargetPlatformIdentifier)' == 'tvos'">
     <Compile Include="System\Net\Sockets\UnixDomainSocketEndPoint.Unix.cs"/>
+    <Compile Include="System\Net\Sockets\SocketAsyncEngine.Linux.cs" Condition="'$(TargetPlatformIdentifier)' == 'unix' and '$(TargetOS)' == 'linux'" />
+    <Compile Include="System\Net\Sockets\SocketAsyncEngine.IoUringConfiguration.Linux.cs" Condition="'$(TargetPlatformIdentifier)' == 'unix' and '$(TargetOS)' == 'linux'" />
+    <Compile Include="System\Net\Sockets\SocketAsyncEngine.IoUringSqeWriters.Linux.cs" Condition="'$(TargetPlatformIdentifier)' == 'unix' and '$(TargetOS)' == 'linux'" />
+    <Compile Include="System\Net\Sockets\SocketAsyncEngine.IoUringCompletionDispatch.Linux.cs" Condition="'$(TargetPlatformIdentifier)' == 'unix' and '$(TargetOS)' == 'linux'" />
+    <Compile Include="System\Net\Sockets\SocketAsyncEngine.IoUringSlots.Linux.cs" Condition="'$(TargetPlatformIdentifier)' == 'unix' and '$(TargetOS)' == 'linux'" />
+    <Compile Include="System\Net\Sockets\SocketAsyncEngine.IoUringRings.Linux.cs" Condition="'$(TargetPlatformIdentifier)' == 'unix' and '$(TargetOS)' == 'linux'" />
+    <Compile Include="System\Net\Sockets\SocketAsyncEngine.IoUringDiagnostics.Linux.cs" Condition="'$(TargetPlatformIdentifier)' == 'unix' and '$(TargetOS)' == 'linux'" />
+    <Compile Include="..\tests\FunctionalTests\IoUringTestInfrastructure\SocketAsyncEngine.IoUringTestHooks.Linux.cs"
+             Link="System\Net\Sockets\SocketAsyncEngine.IoUringTestHooks.Linux.cs"
+             Condition="'$(TargetPlatformIdentifier)' == 'unix' and '$(TargetOS)' == 'linux' and '$(Configuration)' == 'Debug'" />
+    <Compile Include="..\tests\FunctionalTests\IoUringTestInfrastructure\SocketAsyncEngine.IoUringTestAccessors.Linux.cs"
+             Link="System\Net\Sockets\SocketAsyncEngine.IoUringTestAccessors.Linux.cs"
+             Condition="'$(TargetPlatformIdentifier)' == 'unix' and '$(TargetOS)' == 'linux' and '$(Configuration)' == 'Debug'" />
+    <Compile Include="System\Net\Sockets\SocketAsyncEngine.IoUringTestHooks.Stubs.Linux.cs"
+             Condition="'$(TargetPlatformIdentifier)' == 'unix' and '$(TargetOS)' == 'linux' and '$(Configuration)' != 'Debug'" />
+    <Compile Include="System\Net\Sockets\MpscQueue.cs" Condition="'$(TargetPlatformIdentifier)' == 'unix' and '$(TargetOS)' == 'linux'" />
+    <Compile Include="System\Net\Sockets\IoUringProvidedBufferRing.Linux.cs" Condition="'$(TargetPlatformIdentifier)' == 'unix' and '$(TargetOS)' == 'linux'" />
+    <Compile Include="System\Net\Sockets\SocketPal.IoUring.Linux.cs" Condition="'$(TargetPlatformIdentifier)' == 'unix' and '$(TargetOS)' == 'linux'" />
+    <Compile Include="System\Net\Sockets\SocketAsyncContext.IoUring.Linux.cs" Condition="'$(TargetPlatformIdentifier)' == 'unix' and '$(TargetOS)' == 'linux'" />
     <Compile Include="System\Net\Sockets\SocketAsyncEngine.Unix.cs" />
     <Compile Include="$(CommonPath)Interop\Unix\System.Native\Interop.SocketEvent.cs"
              Link="Common\Interop\Unix\System.Native\Interop.SocketEvent.cs" />
+    <Compile Include="$(CommonPath)Interop\Unix\System.Native\Interop.SocketEvent.Linux.cs"
+             Link="Common\Interop\Unix\System.Native\Interop.SocketEvent.Linux.cs"
+             Condition="'$(TargetPlatformIdentifier)' == 'unix' and '$(TargetOS)' == 'linux'" />
+    <Compile Include="$(CommonPath)Interop\Unix\System.Native\Interop.IoUringShim.cs"
+             Link="Common\Interop\Unix\System.Native\Interop.IoUringShim.cs"
+             Condition="'$(TargetPlatformIdentifier)' == 'unix' and '$(TargetOS)' == 'linux'" />
   </ItemGroup>
 
   <ItemGroup Condition="'$(TargetPlatformIdentifier)' == 'unix' or '$(TargetPlatformIdentifier)' == 'wasi' or '$(TargetPlatformIdentifier)' == 'osx' or '$(TargetPlatformIdentifier)' == 'ios' or '$(TargetPlatformIdentifier)' == 'tvos'">