diff --git a/src/coreclr/vm/wasm/callhelpers-pinvoke.cpp b/src/coreclr/vm/wasm/callhelpers-pinvoke.cpp index 2be5428681fe54..3bc02cd24132eb 100644 --- a/src/coreclr/vm/wasm/callhelpers-pinvoke.cpp +++ b/src/coreclr/vm/wasm/callhelpers-pinvoke.cpp @@ -133,7 +133,6 @@ extern "C" { int32_t SystemNative_PosixFAdvise (void *, int64_t, int64_t, int32_t); int32_t SystemNative_Read (void *, void *, int32_t); int32_t SystemNative_ReadDir (void *, void *); - int32_t SystemNative_ReadFromNonblocking (void *, void *, int32_t); int32_t SystemNative_ReadLink (void *, void *, int32_t); int64_t SystemNative_ReadV (void *, void *, int32_t); void * SystemNative_Realloc (void *, void *); @@ -151,7 +150,6 @@ extern "C" { int32_t SystemNative_UTimensat (void *, void *); int32_t SystemNative_Unlink (void *); int32_t SystemNative_Write (void *, void *, int32_t); - int32_t SystemNative_WriteToNonblocking (void *, void *, int32_t); int64_t SystemNative_WriteV (void *, void *, int32_t); } // extern "C" @@ -277,7 +275,6 @@ static const Entry s_libSystem_Native [] = { DllImportEntry(SystemNative_PosixFAdvise) // System.Private.CoreLib DllImportEntry(SystemNative_Read) // System.Private.CoreLib DllImportEntry(SystemNative_ReadDir) // System.Private.CoreLib - DllImportEntry(SystemNative_ReadFromNonblocking) // System.Private.CoreLib DllImportEntry(SystemNative_ReadLink) // System.Private.CoreLib DllImportEntry(SystemNative_ReadV) // System.Private.CoreLib DllImportEntry(SystemNative_Realloc) // System.Private.CoreLib @@ -294,8 +291,7 @@ static const Entry s_libSystem_Native [] = { DllImportEntry(SystemNative_SysLog) // System.Private.CoreLib DllImportEntry(SystemNative_UTimensat) // System.Private.CoreLib DllImportEntry(SystemNative_Unlink) // System.IO.MemoryMappedFiles, System.Private.CoreLib - DllImportEntry(SystemNative_Write) // System.Console, System.Private.CoreLib - DllImportEntry(SystemNative_WriteToNonblocking) // System.Private.CoreLib + DllImportEntry(SystemNative_Write) // System.Console, System.Private.Core DllImportEntry(SystemNative_WriteV) // System.Private.CoreLib }; diff --git a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.HandleEvents.cs b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.HandleEvents.cs new file mode 100644 index 00000000000000..7da71b356c4ab4 --- /dev/null +++ b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.HandleEvents.cs @@ -0,0 +1,51 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Runtime.InteropServices; + +internal static partial class Interop +{ + internal static partial class Sys + { + [Flags] + internal enum HandleEvents : int + { + None = 0x00, + Read = 0x01, + Write = 0x02, + ReadClose = 0x04, + Close = 0x08, + Error = 0x10 + } + + [StructLayout(LayoutKind.Sequential)] + internal struct HandleEvent + { + public IntPtr Data; + public HandleEvents Events; + private int _padding; + } + + [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_CreateHandleEventPort")] + internal static unsafe partial Error CreateHandleEventPort(IntPtr* port); + + [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_CloseHandleEventPort")] + internal static partial Error CloseHandleEventPort(IntPtr port); + + [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_CreateHandleEventBuffer")] + internal static unsafe partial Error CreateHandleEventBuffer(int count, HandleEvent** buffer); + + [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_FreeHandleEventBuffer")] + internal static unsafe partial Error FreeHandleEventBuffer(HandleEvent* buffer); + + [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_TryChangeHandleEventRegistration")] + internal static partial Error TryChangeHandleEventRegistration(IntPtr port, SafeHandle socket, HandleEvents currentEvents, HandleEvents newEvents, IntPtr data); + + [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_TryChangeHandleEventRegistration")] + internal static partial Error TryChangeHandleEventRegistration(IntPtr port, IntPtr socket, HandleEvents currentEvents, HandleEvents newEvents, IntPtr data); + + [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_WaitForHandleEvents")] + internal static unsafe partial Error WaitForHandleEvents(IntPtr port, HandleEvent* buffer, int* count); + } +} diff --git a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.Read.Pipe.cs b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.Read.Pipe.cs deleted file mode 100644 index ef2c39ddbc0656..00000000000000 --- a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.Read.Pipe.cs +++ /dev/null @@ -1,24 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.InteropServices; -using Microsoft.Win32.SafeHandles; - -internal static partial class Interop -{ - internal static partial class Sys - { - /// - /// Reads a number of bytes from an open file descriptor into a specified buffer. - /// - /// The open file descriptor to try to read from - /// The buffer to read info into - /// The size of the buffer - /// - /// Returns the number of bytes read on success; otherwise, -1 is returned - /// Note - on fail. the position of the stream may change depending on the platform; consult man 2 read for more info - /// - [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_Read", SetLastError = true)] - internal static unsafe partial int Read(SafePipeHandle fd, byte* buffer, int count); - } -} diff --git a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.Read.cs b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.Read.cs index d1005d8740d26e..8318d1822c67c0 100644 --- a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.Read.cs +++ b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.Read.cs @@ -20,8 +20,5 @@ internal static partial class Sys /// [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_Read", SetLastError = true)] internal static unsafe partial int Read(SafeHandle fd, byte* buffer, int count); - - [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_ReadFromNonblocking", SetLastError = true)] - internal static unsafe partial int ReadFromNonblocking(SafeHandle fd, byte* buffer, int count); } } diff --git a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.SocketEvent.cs b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.SocketEvent.cs deleted file mode 100644 index ef14ce83a11713..00000000000000 --- a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.SocketEvent.cs +++ /dev/null @@ -1,51 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Runtime.InteropServices; - -internal static partial class Interop -{ - internal static partial class Sys - { - [Flags] - internal enum SocketEvents : int - { - None = 0x00, - Read = 0x01, - Write = 0x02, - ReadClose = 0x04, - Close = 0x08, - Error = 0x10 - } - - [StructLayout(LayoutKind.Sequential)] - internal struct SocketEvent - { - public IntPtr Data; - public SocketEvents Events; - private int _padding; - } - - [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_CreateSocketEventPort")] - internal static unsafe partial Error CreateSocketEventPort(IntPtr* port); - - [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_CloseSocketEventPort")] - internal static partial Error CloseSocketEventPort(IntPtr port); - - [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_CreateSocketEventBuffer")] - internal static unsafe partial Error CreateSocketEventBuffer(int count, SocketEvent** buffer); - - [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_FreeSocketEventBuffer")] - internal static unsafe partial Error FreeSocketEventBuffer(SocketEvent* buffer); - - [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_TryChangeSocketEventRegistration")] - internal static partial Error TryChangeSocketEventRegistration(IntPtr port, SafeHandle socket, SocketEvents currentEvents, SocketEvents newEvents, IntPtr data); - - [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_TryChangeSocketEventRegistration")] - internal static partial Error TryChangeSocketEventRegistration(IntPtr port, IntPtr socket, SocketEvents currentEvents, SocketEvents newEvents, IntPtr data); - - [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_WaitForSocketEvents")] - internal static unsafe partial Error WaitForSocketEvents(IntPtr port, SocketEvent* buffer, int* count); - } -} diff --git a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.Write.Pipe.cs b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.Write.Pipe.cs deleted file mode 100644 index bd192c24b8edf2..00000000000000 --- a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.Write.Pipe.cs +++ /dev/null @@ -1,23 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.InteropServices; -using Microsoft.Win32.SafeHandles; - -internal static partial class Interop -{ - internal static partial class Sys - { - /// - /// Writes the specified buffer to the provided open file descriptor - /// - /// The file descriptor to try and write to - /// The data to attempt to write - /// The amount of data to write, in bytes - /// - /// Returns the number of bytes written on success; otherwise, returns -1 and sets errno - /// - [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_Write", SetLastError = true)] - internal static unsafe partial int Write(SafePipeHandle fd, byte* buffer, int bufferSize); - } -} diff --git a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.Write.cs b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.Write.cs index b5bad8429e1937..e6dd4a900837a9 100644 --- a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.Write.cs +++ b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.Write.cs @@ -23,8 +23,5 @@ internal static partial class Sys [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_Write", SetLastError = true)] internal static unsafe partial int Write(IntPtr fd, byte* buffer, int bufferSize); - - [LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_WriteToNonblocking", SetLastError = true)] - internal static unsafe partial int WriteToNonblocking(SafeHandle fd, byte* buffer, int bufferSize); } } diff --git a/src/libraries/Common/src/Interop/Wasi/System.Native/Interop.SocketEvent.cs b/src/libraries/Common/src/Interop/Wasi/System.Native/Interop.HandleEvents.cs similarity index 94% rename from src/libraries/Common/src/Interop/Wasi/System.Native/Interop.SocketEvent.cs rename to src/libraries/Common/src/Interop/Wasi/System.Native/Interop.HandleEvents.cs index 2e8742bc4c7708..141ce7937c14a2 100644 --- a/src/libraries/Common/src/Interop/Wasi/System.Native/Interop.SocketEvent.cs +++ b/src/libraries/Common/src/Interop/Wasi/System.Native/Interop.HandleEvents.cs @@ -9,7 +9,7 @@ internal static partial class Interop internal static partial class Sys { [Flags] - internal enum SocketEvents : int + internal enum HandleEvents : int { None = 0x00, Read = 0x01, diff --git a/src/libraries/System.IO.Pipes/src/Microsoft/Win32/SafeHandles/SafePipeHandle.Unix.cs b/src/libraries/System.IO.Pipes/src/Microsoft/Win32/SafeHandles/SafePipeHandle.Unix.cs index eea6ef6410a907..7412cd2f36347b 100644 --- a/src/libraries/System.IO.Pipes/src/Microsoft/Win32/SafeHandles/SafePipeHandle.Unix.cs +++ b/src/libraries/System.IO.Pipes/src/Microsoft/Win32/SafeHandles/SafePipeHandle.Unix.cs @@ -4,10 +4,10 @@ using System; using System.Diagnostics; using System.Net.Sockets; -using System.Reflection; using System.Runtime.InteropServices; -using System.Security; using System.Threading; +using System.Threading.Tasks; +using System.Threading.Tasks.Sources; namespace Microsoft.Win32.SafeHandles { @@ -15,130 +15,523 @@ public sealed partial class SafePipeHandle : SafeHandleZeroOrMinusOneIsInvalid { private const int DefaultInvalidHandle = -1; - // For anonymous pipes, SafePipeHandle.handle is the file descriptor of the pipe. - // For named pipes, SafePipeHandle.handle is a copy of the file descriptor - // extracted from the Socket's SafeHandle. - // This allows operations related to file descriptors to be performed directly on the SafePipeHandle, - // and operations that should go through the Socket to be done via PipeSocket. We keep the - // Socket's SafeHandle alive as long as this SafeHandle is alive. + private NullableBool _isBlocking; + private PollableHandle? _pollHandle; + private ReadOperation? _cachedReadOp; + private WriteOperation? _cachedWriteOp; - private Socket? _pipeSocket; - private SafeHandle? _pipeSocketHandle; - private volatile int _disposed; + private ReadOperation RentReadOperation() + => Interlocked.Exchange(ref _cachedReadOp, null) ?? new ReadOperation(this); - internal SafePipeHandle(Socket namedPipeSocket) : base(ownsHandle: true) + private WriteOperation RentWriteOperation() + => Interlocked.Exchange(ref _cachedWriteOp, null) ?? new WriteOperation(this); + + private void ReturnReadOperation(ReadOperation op) + { + op.Reset(); + Volatile.Write(ref _cachedReadOp, op); + } + + private void ReturnWriteOperation(WriteOperation op) + { + op.Reset(); + Volatile.Write(ref _cachedWriteOp, op); + } + + private bool IsBlocking + { + get + { + NullableBool isBlocking = _isBlocking; + if (isBlocking == NullableBool.Undefined) + { + if (Interop.Sys.Fcntl.GetIsNonBlocking(this, out bool nonBlocking) != 0) + { + throw Interop.GetExceptionForIoErrno(Interop.Sys.GetLastErrorInfo()); + } + + _isBlocking = isBlocking = nonBlocking ? NullableBool.False : NullableBool.True; + } + + return isBlocking == NullableBool.True; + } + } + + private void SetHandleNonBlocking() + { + if (_isBlocking != NullableBool.False) + { + if (Interop.Sys.Fcntl.SetIsNonBlocking(this, 1) != 0) + { + throw Interop.GetExceptionForIoErrno(Interop.Sys.GetLastErrorInfo()); + } + _isBlocking = NullableBool.False; + } + } + + private PollableHandle PollHandle { - SetPipeSocketInterlocked(namedPipeSocket, ownsHandle: true); - base.SetHandle(_pipeSocketHandle!.DangerousGetHandle()); + get + { + if (_pollHandle == null) + { + SetHandleNonBlocking(); + PollableHandle.Create(this, ref _pollHandle); + } + return _pollHandle!; + } } - internal Socket PipeSocket => _pipeSocket ?? CreatePipeSocket(); + internal SafePipeHandle(Socket namedPipeSocket) : base(ownsHandle: true) + { + Debug.Assert(namedPipeSocket != null); - internal SafeHandle? PipeSocketHandle => _pipeSocketHandle; + _isBlocking = namedPipeSocket.Blocking ? NullableBool.True : NullableBool.False; + + // Transfer ownership of the file descriptor from the Socket to this SafeHandle. + SafeHandle socketHandle = namedPipeSocket.SafeHandle; + base.SetHandle(socketHandle.DangerousGetHandle()); + socketHandle.SetHandleAsInvalid(); + namedPipeSocket.Dispose(); + } protected override void Dispose(bool disposing) { - base.Dispose(disposing); // must be called before trying to Dispose the socket - _disposed = 1; - if (disposing && Volatile.Read(ref _pipeSocket) is Socket socket) + if (disposing) { - socket.Dispose(); - _pipeSocket = null; + _pollHandle?.Dispose(); } + base.Dispose(disposing); } protected override bool ReleaseHandle() { - Debug.Assert(!IsInvalid); + return (long)handle >= 0 && Interop.Sys.Close(handle) == 0; + } + + public override bool IsInvalid + { + get { return (long)handle < 0; } + } + + internal new void SetHandle(IntPtr descriptor) + { + base.SetHandle(descriptor); + } + + // Named pipes on Unix are implemented using Unix domain sockets. + // Returns 0 for non-socket handles (getsockopt returns ENOTSOCK). + internal unsafe int GetSocketBufferSize(SocketOptionName optionName) + { + int value; + int optLen = sizeof(int); + Interop.Error error = Interop.Sys.GetSockOpt(this, SocketOptionLevel.Socket, optionName, (byte*)&value, &optLen); + return error == Interop.Error.SUCCESS ? value : 0; + } - if (_pipeSocketHandle != null) + internal unsafe (int BytesRead, Interop.ErrorInfo ErrorInfo) Read(Span buffer) + { + int sequenceNumber = 0; + bool isBlocking = IsBlocking; + + bool doSync = isBlocking || PollHandle.IsReadReady(out sequenceNumber); + if (doSync) { - base.SetHandle((IntPtr)DefaultInvalidHandle); - _pipeSocketHandle.DangerousRelease(); - _pipeSocketHandle = null; - return true; + if (TryCompleteRead(buffer, out var result, out bool pending)) + { + return result; + } + if (isBlocking) + { + // The handle changed to non-blocking due to a concurrent operation. + Debug.Assert(pending); + if (PollHandle.IsReadReady(out sequenceNumber) && TryCompleteRead(buffer, out result, out _)) + { + return result; + } + } } - else + + ReadOperation op = RentReadOperation(); + fixed (byte* bufPtr = &MemoryMarshal.GetReference(buffer)) { - return (long)handle >= 0 ? - Interop.Sys.Close(handle) == 0 : - true; + op.Init(bufPtr, buffer.Length); + + PollOperationSyncResult result = PollHandle.ReadSync(op, sequenceNumber, timeout: -1); + + if (result == PollOperationSyncResult.Completed) + { + var readResult = op.Result; + + ReturnReadOperation(op); + + return readResult; + } + + return (-1, new Interop.ErrorInfo(Interop.Error.ECANCELED)); } } - public override bool IsInvalid + internal ValueTask<(int BytesRead, Interop.ErrorInfo ErrorInfo)> ReadAsync(Memory destination, CancellationToken cancellationToken) { - get { return (long)handle < 0 && _pipeSocket == null; } + if (PollHandle.IsReadReady(out int sequenceNumber) && + TryCompleteRead(destination.Span, out var readResult, out _)) + { + return new ValueTask<(int, Interop.ErrorInfo)>(readResult); + } + + ReadOperation op = RentReadOperation(); + op.Init(destination, cancellationToken); + + PollOperationAsyncResult result = PollHandle.ReadAsync(op, sequenceNumber, cancellationToken); + + if (result == PollOperationAsyncResult.Pending) + { + return new ValueTask<(int, Interop.ErrorInfo)>(op, op.Version); + } + else if (result == PollOperationAsyncResult.Completed) + { + readResult = op.Result; + ReturnReadOperation(op); + return new ValueTask<(int, Interop.ErrorInfo)>(readResult); + } + + return new ValueTask<(int, Interop.ErrorInfo)>((-1, new Interop.ErrorInfo(Interop.Error.ECANCELED))); } - private Socket CreatePipeSocket(bool ownsHandle = true) + internal unsafe Interop.ErrorInfo Write(ReadOnlySpan buffer) { - Socket? socket = null; - if (_disposed == 0) + int sequenceNumber = 0; + bool isBlocking = IsBlocking; + + bool doSync = isBlocking || PollHandle.IsWriteReady(out sequenceNumber); + while (doSync) + { + if (TryCompleteWrite(buffer, out int bytesWritten, out Interop.ErrorInfo errorInfo, out bool pending)) + { + return errorInfo; + } + buffer = buffer.Slice(bytesWritten); + if (isBlocking && pending) + { + // The handle changed to non-blocking due to a concurrent operation. + isBlocking = false; + doSync = PollHandle.IsWriteReady(out sequenceNumber); + } + else + { + // There are bytes remaining for a blocking write. + Debug.Assert(buffer.Length != 0); + doSync = isBlocking; + } + } + + WriteOperation op = RentWriteOperation(); + fixed (byte* bufPtr = &MemoryMarshal.GetReference(buffer)) + { + op.Init(bufPtr, buffer.Length); + + PollOperationSyncResult result = PollHandle.WriteSync(op, sequenceNumber, timeout: -1); + + if (result == PollOperationSyncResult.Completed) + { + Interop.ErrorInfo errorInfo = op.WriteResult; + + ReturnWriteOperation(op); + + return errorInfo; + } + + return new Interop.ErrorInfo(Interop.Error.ECANCELED); + } + } + + internal ValueTask WriteAsync(ReadOnlyMemory source, CancellationToken cancellationToken) + { + int bytesWritten = 0; + if (PollHandle.IsWriteReady(out int sequenceNumber) && + TryCompleteWrite(source.Span, out bytesWritten, out Interop.ErrorInfo writeResult, out _)) + { + return new ValueTask(writeResult); + } + + WriteOperation op = RentWriteOperation(); + op.Init(source.Slice(bytesWritten), cancellationToken); + + PollOperationAsyncResult result = PollHandle.WriteAsync(op, sequenceNumber, cancellationToken); + + if (result == PollOperationAsyncResult.Pending) + { + return new ValueTask(op, op.Version); + } + else if (result == PollOperationAsyncResult.Completed) { - bool refAdded = false; + writeResult = op.WriteResult; + ReturnWriteOperation(op); + return new ValueTask(writeResult); + } + + return new ValueTask(new Interop.ErrorInfo(Interop.Error.ECANCELED)); + } + + private sealed unsafe class ReadOperation : PollTriggeredOperation, IValueTaskSource<(int BytesRead, Interop.ErrorInfo ErrorInfo)> + { + private readonly SafePipeHandle _owner; + internal (int BytesRead, Interop.ErrorInfo ErrorInfo) Result; + private ManualResetValueTaskSourceCore<(int, Interop.ErrorInfo)> _mrvtsc; + private Memory _buffer; + private byte* _syncBuffer; + private int _syncBufferLength; + private CancellationToken _cancellationToken; + + internal ReadOperation(SafePipeHandle owner) + => _owner = owner; + + internal short Version + => _mrvtsc.Version; + + internal void Init(byte* syncBuffer, int syncBufferLength) + { + _syncBuffer = syncBuffer; + _syncBufferLength = syncBufferLength; + } + + internal void Init(Memory buffer, CancellationToken cancellationToken) + { + _buffer = buffer; + _cancellationToken = cancellationToken; + } + + internal void Reset() + { + _buffer = default; + _syncBuffer = null; + _cancellationToken = default; + _mrvtsc.Reset(); + } + + protected override bool TryCompleteOperation(SafeHandle handle) + { + if (_syncBuffer != null) + { + Debug.Assert(_syncBufferLength > 0); + return _owner.TryCompleteRead(_syncBuffer, _syncBufferLength, out Result, out _); + } + + Span span = _buffer.Span; + Debug.Assert(!span.IsEmpty); + + fixed (byte* bufPtr = &MemoryMarshal.GetReference(span)) + { + return _owner.TryCompleteRead(bufPtr, span.Length, out Result, out _); + } + } + + protected override void OnCompleted(PollOperationOnCompletedResult result) + { + if (result == PollOperationOnCompletedResult.Completed) + { + _mrvtsc.SetResult(Result); + } + else if (result == PollOperationOnCompletedResult.Canceled) + { + _mrvtsc.SetException(new OperationCanceledException(_cancellationToken)); + } + else + { + Debug.Assert(result == PollOperationOnCompletedResult.Aborted); + _mrvtsc.SetException(new OperationCanceledException()); + } + } + + ValueTaskSourceStatus IValueTaskSource<(int BytesRead, Interop.ErrorInfo ErrorInfo)>.GetStatus(short token) + => _mrvtsc.GetStatus(token); + + void IValueTaskSource<(int BytesRead, Interop.ErrorInfo ErrorInfo)>.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) + => _mrvtsc.OnCompleted(continuation, state, token, flags); + + (int BytesRead, Interop.ErrorInfo ErrorInfo) IValueTaskSource<(int BytesRead, Interop.ErrorInfo ErrorInfo)>.GetResult(short token) + { + bool canPool = _mrvtsc.GetStatus(token) != ValueTaskSourceStatus.Canceled; try { - DangerousAddRef(ref refAdded); + return _mrvtsc.GetResult(token); + } + finally + { + if (canPool) + { + _owner.ReturnReadOperation(this); + } + } + } + } + + private sealed unsafe class WriteOperation : PollTriggeredOperation, IValueTaskSource + { + private readonly SafePipeHandle _owner; + internal Interop.ErrorInfo WriteResult; + private ManualResetValueTaskSourceCore _mrvtsc; + private ReadOnlyMemory _buffer; + private byte* _syncBuffer; + private int _syncRemaining; + private CancellationToken _cancellationToken; + + internal WriteOperation(SafePipeHandle owner) + => _owner = owner; + + internal short Version + => _mrvtsc.Version; + + internal void Init(byte* syncBuffer, int syncRemaining) + { + _syncBuffer = syncBuffer; + _syncRemaining = syncRemaining; + } + + internal void Init(ReadOnlyMemory buffer, CancellationToken cancellationToken) + { + _buffer = buffer; + _cancellationToken = cancellationToken; + } + + internal void Reset() + { + _buffer = default; + _syncBuffer = null; + _cancellationToken = default; + _mrvtsc.Reset(); + } + + protected override bool TryCompleteOperation(SafeHandle handle) + { + if (_syncBuffer != null) + { + Debug.Assert(_syncRemaining > 0); + + if (_owner.TryCompleteWrite(_syncBuffer, _syncRemaining, out int bytesWritten, out WriteResult, out _)) + { + return true; + } + _syncBuffer += bytesWritten; + _syncRemaining -= bytesWritten; + return false; + } - socket = SetPipeSocketInterlocked(new Socket(new SafeSocketHandle(handle, ownsHandle)), ownsHandle); + ReadOnlySpan span = _buffer.Span; + Debug.Assert(!span.IsEmpty); - // Double check if we haven't Disposed in the meanwhile, and ensure - // the Socket is disposed, in case Dispose() missed the _pipeSocket assignment. - if (_disposed == 1) + fixed (byte* bufPtr = &MemoryMarshal.GetReference(span)) + { + if (_owner.TryCompleteWrite(bufPtr, span.Length, out int bytesWritten, out WriteResult, out _)) { - Volatile.Write(ref _pipeSocket, null); - socket.Dispose(); - socket = null; + return true; } + _buffer = _buffer.Slice(bytesWritten); + return false; + } + } + + protected override void OnCompleted(PollOperationOnCompletedResult result) + { + if (result == PollOperationOnCompletedResult.Completed) + { + _mrvtsc.SetResult(WriteResult); + } + else if (result == PollOperationOnCompletedResult.Canceled) + { + _mrvtsc.SetException(new OperationCanceledException(_cancellationToken)); + } + else + { + Debug.Assert(result == PollOperationOnCompletedResult.Aborted); + _mrvtsc.SetException(new OperationCanceledException()); + } + } + + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) + => _mrvtsc.GetStatus(token); + + void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) + => _mrvtsc.OnCompleted(continuation, state, token, flags); + + Interop.ErrorInfo IValueTaskSource.GetResult(short token) + { + bool canPool = _mrvtsc.GetStatus(token) != ValueTaskSourceStatus.Canceled; + try + { + return _mrvtsc.GetResult(token); } finally { - if (refAdded) + if (canPool) { - DangerousRelease(); + _owner.ReturnWriteOperation(this); } } } - - ObjectDisposedException.ThrowIf(socket is null, this); - return socket; } - private Socket SetPipeSocketInterlocked(Socket socket, bool ownsHandle) + private unsafe bool TryCompleteRead(Span buffer, out (int BytesRead, Interop.ErrorInfo ErrorInfo) result, out bool pending) { - Debug.Assert(socket != null); - - // Multiple threads may try to create the PipeSocket. - Socket? current = Interlocked.CompareExchange(ref _pipeSocket, socket, null); - if (current != null) + fixed (byte* bufPtr = &MemoryMarshal.GetReference(buffer)) { - socket.Dispose(); - return current; + return TryCompleteRead(bufPtr, buffer.Length, out result, out pending); } + } - // If we own the handle, defer ownership to the SocketHandle. - SafeSocketHandle socketHandle = _pipeSocket.SafeHandle; - if (ownsHandle) + private unsafe bool TryCompleteRead(byte* buffer, int length, out (int BytesRead, Interop.ErrorInfo ErrorInfo) result, out bool pending) + { + int bytesRead = Interop.Sys.Read(this, buffer, length); + if (bytesRead < 0) { - _pipeSocketHandle = socketHandle; - - bool ignored = false; - socketHandle.DangerousAddRef(ref ignored); + Interop.ErrorInfo errorInfo = Interop.Sys.GetLastErrorInfo(); + if (IsPending(errorInfo)) + { + pending = true; + result = default; + return false; + } + pending = false; + result = (-1, errorInfo); + return true; } - return socket; + pending = false; + result = (bytesRead, default); + return true; } - internal void SetHandle(IntPtr descriptor, bool ownsHandle = true) + private unsafe bool TryCompleteWrite(ReadOnlySpan buffer, out int bytesWritten, out Interop.ErrorInfo errorInfo, out bool pending) { - base.SetHandle(descriptor); + fixed (byte* bufPtr = &MemoryMarshal.GetReference(buffer)) + { + return TryCompleteWrite(bufPtr, buffer.Length, out bytesWritten, out errorInfo, out pending); + } + } - // Avoid throwing when we own the handle by defering pipe creation. - if (!ownsHandle) + private unsafe bool TryCompleteWrite(byte* buffer, int length, out int bytesWritten, out Interop.ErrorInfo errorInfo, out bool pending) + { + bytesWritten = Interop.Sys.Write(this, buffer, length); + if (bytesWritten < 0) { - _pipeSocket = CreatePipeSocket(ownsHandle); + errorInfo = Interop.Sys.GetLastErrorInfo(); + if (IsPending(errorInfo)) + { + pending = true; + bytesWritten = 0; + return false; + } + pending = false; + return true; } + + pending = false; + errorInfo = default; + return bytesWritten == length; } + + private static bool IsPending(Interop.ErrorInfo errorInfo) + => errorInfo.Error is Interop.Error.EAGAIN or Interop.Error.EWOULDBLOCK; } } diff --git a/src/libraries/System.IO.Pipes/src/System.IO.Pipes.csproj b/src/libraries/System.IO.Pipes/src/System.IO.Pipes.csproj index 280212748fd613..654035ab1e5650 100644 --- a/src/libraries/System.IO.Pipes/src/System.IO.Pipes.csproj +++ b/src/libraries/System.IO.Pipes/src/System.IO.Pipes.csproj @@ -156,12 +156,12 @@ Link="Common\Interop\Unix\Interop.Pipe.cs" /> - + - + + + @@ -183,12 +187,12 @@ + - diff --git a/src/libraries/System.IO.Pipes/src/System/IO/Pipes/NamedPipeClientStream.Unix.cs b/src/libraries/System.IO.Pipes/src/System/IO/Pipes/NamedPipeClientStream.Unix.cs index 4c7df8d63e73a4..bff9fe3953df90 100644 --- a/src/libraries/System.IO.Pipes/src/System/IO/Pipes/NamedPipeClientStream.Unix.cs +++ b/src/libraries/System.IO.Pipes/src/System/IO/Pipes/NamedPipeClientStream.Unix.cs @@ -36,16 +36,13 @@ private bool TryConnect(int _ /* timeout */) // immediately if it isn't. The only delay will be between the time the server // has called Bind and Listen, with the latter immediately following the former. var socket = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.Unspecified); - SafePipeHandle? clientHandle = null; try { socket.Connect(new UnixDomainSocketEndPoint(_normalizedPipePath!)); - clientHandle = new SafePipeHandle(socket); - ConfigureSocket(socket, clientHandle, _direction, 0, 0, _inheritability); + ConfigureSocket(socket, _direction, 0, 0, _inheritability); } catch (SocketException e) { - clientHandle?.Dispose(); socket.Dispose(); switch (e.SocketErrorCode) @@ -62,6 +59,8 @@ private bool TryConnect(int _ /* timeout */) } } + // Transfer ownership of the fd from the Socket to SafePipeHandle. + var clientHandle = new SafePipeHandle(socket); try { ValidateRemotePipeUser(clientHandle); @@ -69,7 +68,6 @@ private bool TryConnect(int _ /* timeout */) catch (Exception) { clientHandle.Dispose(); - socket.Dispose(); throw; } @@ -94,7 +92,7 @@ public override int InBufferSize { CheckPipePropertyOperations(); if (!CanRead) throw new NotSupportedException(SR.NotSupported_UnreadableStream); - return InternalHandle?.PipeSocket.ReceiveBufferSize ?? 0; + return InternalHandle?.GetSocketBufferSize(SocketOptionName.ReceiveBuffer) ?? 0; } } @@ -104,7 +102,7 @@ public override int OutBufferSize { CheckPipePropertyOperations(); if (!CanWrite) throw new NotSupportedException(SR.NotSupported_UnwritableStream); - return InternalHandle?.PipeSocket.SendBufferSize ?? 0; + return InternalHandle?.GetSocketBufferSize(SocketOptionName.SendBuffer) ?? 0; } } diff --git a/src/libraries/System.IO.Pipes/src/System/IO/Pipes/NamedPipeServerStream.Unix.cs b/src/libraries/System.IO.Pipes/src/System/IO/Pipes/NamedPipeServerStream.Unix.cs index e07b6f274d6625..28c8254b287ea6 100644 --- a/src/libraries/System.IO.Pipes/src/System/IO/Pipes/NamedPipeServerStream.Unix.cs +++ b/src/libraries/System.IO.Pipes/src/System/IO/Pipes/NamedPipeServerStream.Unix.cs @@ -100,8 +100,6 @@ async Task WaitForConnectionAsyncCore() private void HandleAcceptedSocket(Socket acceptedSocket) { - var serverHandle = new SafePipeHandle(acceptedSocket); - try { if (IsCurrentUserOnly) @@ -109,7 +107,7 @@ private void HandleAcceptedSocket(Socket acceptedSocket) uint serverEUID = Interop.Sys.GetEUid(); uint peerID; - if (Interop.Sys.GetPeerID(serverHandle, out peerID) == -1) + if (Interop.Sys.GetPeerID(acceptedSocket.SafeHandle, out peerID) == -1) { throw CreateExceptionForLastError(_instance?.PipeName); } @@ -120,15 +118,17 @@ private void HandleAcceptedSocket(Socket acceptedSocket) } } - ConfigureSocket(acceptedSocket, serverHandle, _direction, _inBufferSize, _outBufferSize, _inheritability); + ConfigureSocket(acceptedSocket, _direction, _inBufferSize, _outBufferSize, _inheritability); } catch { - serverHandle.Dispose(); acceptedSocket.Dispose(); throw; } + // Transfer ownership of the fd from the Socket to SafePipeHandle. + var serverHandle = new SafePipeHandle(acceptedSocket); + InitializeHandle(serverHandle, isExposed: false, isAsync: (_options & PipeOptions.Asynchronous) != 0); State = PipeState.Connected; } @@ -158,7 +158,7 @@ public string GetImpersonationUserName() { CheckWriteOperations(); - SafeHandle? handle = InternalHandle?.PipeSocketHandle; + SafeHandle? handle = InternalHandle; if (handle == null) { throw new InvalidOperationException(SR.InvalidOperation_PipeHandleNotSet); @@ -179,7 +179,7 @@ public override int InBufferSize { CheckPipePropertyOperations(); if (!CanRead) throw new NotSupportedException(SR.NotSupported_UnreadableStream); - return InternalHandle?.PipeSocket.ReceiveBufferSize ?? _inBufferSize; + return InternalHandle?.GetSocketBufferSize(SocketOptionName.ReceiveBuffer) ?? _inBufferSize; } } @@ -189,7 +189,7 @@ public override int OutBufferSize { CheckPipePropertyOperations(); if (!CanWrite) throw new NotSupportedException(SR.NotSupported_UnwritableStream); - return InternalHandle?.PipeSocket.SendBufferSize ?? _outBufferSize; + return InternalHandle?.GetSocketBufferSize(SocketOptionName.SendBuffer) ?? _outBufferSize; } } @@ -197,7 +197,7 @@ public override int OutBufferSize public void RunAsClient(PipeStreamImpersonationWorker impersonationWorker) { CheckWriteOperations(); - SafeHandle? handle = InternalHandle?.PipeSocketHandle; + SafeHandle? handle = InternalHandle; if (handle == null) { throw new InvalidOperationException(SR.InvalidOperation_PipeHandleNotSet); diff --git a/src/libraries/System.IO.Pipes/src/System/IO/Pipes/PipeStream.Unix.cs b/src/libraries/System.IO.Pipes/src/System/IO/Pipes/PipeStream.Unix.cs index d6585074f4f635..7268285c39c82a 100644 --- a/src/libraries/System.IO.Pipes/src/System/IO/Pipes/PipeStream.Unix.cs +++ b/src/libraries/System.IO.Pipes/src/System/IO/Pipes/PipeStream.Unix.cs @@ -7,7 +7,6 @@ using System.Net.Sockets; using System.Runtime.InteropServices; using System.Runtime.Versioning; -using System.Security; using System.Threading; using System.Threading.Tasks; using Microsoft.Win32.SafeHandles; @@ -151,7 +150,7 @@ public override Task WriteAsync(byte[] buffer, int offset, int count, Cancellati return Task.CompletedTask; } - return WriteAsyncCore(new ReadOnlyMemory(buffer, offset, count), cancellationToken); + return WriteAsyncCore(new ReadOnlyMemory(buffer, offset, count), cancellationToken).AsTask(); } public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default(CancellationToken)) @@ -173,7 +172,7 @@ public override Task WriteAsync(byte[] buffer, int offset, int count, Cancellati return default; } - return new ValueTask(WriteAsyncCore(buffer, cancellationToken)); + return WriteAsyncCore(buffer, cancellationToken); } public override IAsyncResult BeginWrite(byte[] buffer, int offset, int count, AsyncCallback? callback, object? state) @@ -263,18 +262,9 @@ private int ReadCore(Span buffer) return 0; } - // For a blocking socket, we could simply use the same Read syscall as is done - // for reading an anonymous pipe. However, for a non-blocking socket, Read could - // end up returning EWOULDBLOCK rather than blocking waiting for data. Such a case - // is already handled by Socket.Receive, so we use it here. - try - { - return _handle!.PipeSocket.Receive(buffer, SocketFlags.None); - } - catch (SocketException e) - { - throw GetIOExceptionForSocketException(e); - } + var (bytesRead, errorInfo) = _handle.Read(buffer); + CheckPipeCall(bytesRead, errorInfo); + return bytesRead; } private void WriteCore(ReadOnlySpan buffer) @@ -282,60 +272,23 @@ private void WriteCore(ReadOnlySpan buffer) Debug.Assert(_handle != null); DebugAssertHandleValid(_handle); - // For a blocking socket, we could simply use the same Write syscall as is done - // for writing to anonymous pipe. However, for a non-blocking socket, Write could - // end up returning EWOULDBLOCK rather than blocking waiting for space available. - // Such a case is already handled by Socket.Send, so we use it here. - try - { - while (buffer.Length > 0) - { - int bytesWritten = _handle!.PipeSocket.Send(buffer, SocketFlags.None); - buffer = buffer.Slice(bytesWritten); - } - } - catch (SocketException e) - { - throw GetIOExceptionForSocketException(e); - } + Interop.ErrorInfo errorInfo = _handle.Write(buffer); + CheckPipeCall(errorInfo); } private async ValueTask ReadAsyncCore(Memory destination, CancellationToken cancellationToken) { - try - { - return await InternalHandle!.PipeSocket.ReceiveAsync(destination, SocketFlags.None, cancellationToken).ConfigureAwait(false); - } - catch (SocketException e) - { - throw GetIOExceptionForSocketException(e); - } - } - - private async Task WriteAsyncCore(ReadOnlyMemory source, CancellationToken cancellationToken) - { - try - { - while (source.Length > 0) - { - int bytesWritten = await _handle!.PipeSocket.SendAsync(source, SocketFlags.None, cancellationToken).ConfigureAwait(false); - Debug.Assert(bytesWritten > 0 && bytesWritten <= source.Length); - source = source.Slice(bytesWritten); - } - } - catch (SocketException e) - { - throw GetIOExceptionForSocketException(e); - } + Debug.Assert(_handle != null); + var (bytesRead, errorInfo) = await _handle.ReadAsync(destination, cancellationToken).ConfigureAwait(false); + CheckPipeCall(bytesRead, errorInfo); + return bytesRead; } - private IOException GetIOExceptionForSocketException(SocketException e) + private async ValueTask WriteAsyncCore(ReadOnlyMemory source, CancellationToken cancellationToken) { - if (e.SocketErrorCode == SocketError.Shutdown) // EPIPE - { - State = PipeState.Broken; - } - return new IOException(e.Message, e); + Debug.Assert(_handle != null); + Interop.ErrorInfo errorInfo = await _handle.WriteAsync(source, cancellationToken).ConfigureAwait(false); + CheckPipeCall(errorInfo); } // Blocks until the other end of the pipe has read in all written buffer. @@ -460,15 +413,29 @@ private int CheckPipeCall(int result) { if (result == -1) { - Interop.ErrorInfo errorInfo = Interop.Sys.GetLastErrorInfo(); + CheckPipeCall(result, Interop.Sys.GetLastErrorInfo()); + } + + return result; + } + + private void CheckPipeCall(int result, Interop.ErrorInfo errorInfo) + { + if (result == -1) + { + CheckPipeCall(errorInfo); + } + } + private void CheckPipeCall(Interop.ErrorInfo errorInfo) + { + if (errorInfo.Error != Interop.Error.SUCCESS) + { if (errorInfo.Error == Interop.Error.EPIPE) State = PipeState.Broken; throw Interop.GetExceptionForIoErrno(errorInfo); } - - return result; } private int GetPipeBufferSize() @@ -486,7 +453,7 @@ private int GetPipeBufferSize() } internal static void ConfigureSocket( - Socket s, SafePipeHandle _, + Socket s, PipeDirection direction, int inBufferSize, int outBufferSize, HandleInheritability inheritability) { if (inBufferSize > 0) diff --git a/src/libraries/System.Net.Sockets/src/System.Net.Sockets.csproj b/src/libraries/System.Net.Sockets/src/System.Net.Sockets.csproj index 16413a42a31fdf..7ecb434e5cf2a2 100644 --- a/src/libraries/System.Net.Sockets/src/System.Net.Sockets.csproj +++ b/src/libraries/System.Net.Sockets/src/System.Net.Sockets.csproj @@ -192,18 +192,10 @@ - - - - - - diff --git a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SafeSocketHandle.Unix.cs b/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SafeSocketHandle.Unix.cs index 1432559bae1adf..559ed1987ed8fc 100644 --- a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SafeSocketHandle.Unix.cs +++ b/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SafeSocketHandle.Unix.cs @@ -31,7 +31,7 @@ private enum Flags : byte #endif } - private Flags _flags = Flags.IsSocket | (SocketAsyncEngine.InlineSocketCompletionsEnabled ? Flags.PreferInlineCompletions : 0); + private Flags _flags = Flags.IsSocket | (SocketAsyncContext.InlineSocketCompletionsEnabled ? Flags.PreferInlineCompletions : 0); private void SetFlag(Flags flag, bool value) { @@ -60,7 +60,11 @@ internal bool ExposedHandleOrUntrackedConfiguration internal bool PreferInlineCompletions { get => (_flags & Flags.PreferInlineCompletions) != 0; - set => SetFlag(Flags.PreferInlineCompletions, value); + set + { + SetFlag(Flags.PreferInlineCompletions, value); + AsyncContext.SetInlineCompletions(value); + } } // (ab)use Socket class for performing async I/O on non-socket fds. @@ -98,6 +102,7 @@ internal void TransferTrackedState(SafeSocketHandle target) target.DualMode = DualMode; target.ExposedHandleOrUntrackedConfiguration = ExposedHandleOrUntrackedConfiguration; target.IsSocket = IsSocket; + target.PreferInlineCompletions = PreferInlineCompletions; #if SYSTEM_NET_SOCKETS_APPLE_PLATFROM target.TfoEnabled = TfoEnabled; #endif diff --git a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SafeSocketHandle.cs b/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SafeSocketHandle.cs index 7f653dab759b80..e2da51e3efbb39 100644 --- a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SafeSocketHandle.cs +++ b/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SafeSocketHandle.cs @@ -96,7 +96,7 @@ internal void CloseAsIs(bool abortive) try { #endif - bool shouldClose = TryOwnClose(); + bool shouldClose = !IsInvalid && TryOwnClose(); if (NetEventSource.Log.IsEnabled()) NetEventSource.Info(this, $"abortive={abortive}, shouldClose ={shouldClose}"); diff --git a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketAsyncContext.Unix.cs b/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketAsyncContext.Unix.cs index 14040bcda2304b..4e24c289be52b7 100644 --- a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketAsyncContext.Unix.cs +++ b/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketAsyncContext.Unix.cs @@ -108,30 +108,15 @@ private BufferListSendOperation RentBufferListSendOperation() => Interlocked.Exchange(ref _cachedBufferListSendOperation, null) ?? new BufferListSendOperation(this); - private abstract class AsyncOperation : IThreadPoolWorkItem + internal abstract class AsyncOperation : PollTriggeredOperation { - private enum State - { - Waiting = 0, - Running, - RunningWithPendingCancellation, - Complete, - Canceled - } - - private volatile AsyncOperation.State _state; - #if DEBUG private bool _callbackQueued; // When true, the callback has been queued. #endif public readonly SocketAsyncContext AssociatedContext; - public AsyncOperation Next = null!; // initialized by helper called from ctor public SocketError ErrorCode; public Memory SocketAddress; - public CancellationTokenRegistration CancellationRegistration; - - public ManualResetEventSlim? Event { get; set; } public AsyncOperation(SocketAsyncContext context) { @@ -139,215 +124,38 @@ public AsyncOperation(SocketAsyncContext context) Reset(); } +#pragma warning disable CA1822 public void Reset() { - _state = State.Waiting; - Event = null; - Next = this; #if DEBUG _callbackQueued = false; #endif } +#pragma warning restore CA1822 - public OperationResult TryComplete(SocketAsyncContext context) - { - TraceWithContext(context, "Enter"); - - // Set state to Running, unless we've been canceled - State oldState = Interlocked.CompareExchange(ref _state, State.Running, State.Waiting); - if (oldState == State.Canceled) - { - TraceWithContext(context, "Exit, Previously canceled"); - return OperationResult.Cancelled; - } - - Debug.Assert(oldState == State.Waiting, $"Unexpected operation state: {(State)oldState}"); - - // Try to perform the IO - if (DoTryComplete(context)) - { - Debug.Assert(_state is State.Running or State.RunningWithPendingCancellation, "Unexpected operation state"); - - _state = State.Complete; - - TraceWithContext(context, "Exit, Completed"); - return OperationResult.Completed; - } - - // Set state back to Waiting, unless we were canceled, in which case we have to process cancellation now - State newState; - while (true) - { - State state = _state; - Debug.Assert(state is State.Running or State.RunningWithPendingCancellation, $"Unexpected operation state: {(State)state}"); - - newState = (state == State.Running ? State.Waiting : State.Canceled); - if (state == Interlocked.CompareExchange(ref _state, newState, state)) - { - break; - } - - // Race to update the state. Loop and try again. - } + protected sealed override bool TryCompleteOperation(SafeHandle handle) + => TryCompleteOperation(AssociatedContext); - if (newState == State.Canceled) - { - ProcessCancellation(); - TraceWithContext(context, "Exit, Newly cancelled"); - return OperationResult.Cancelled; - } - - TraceWithContext(context, "Exit, Pending"); - return OperationResult.Pending; - } - - public bool TryCancel() + protected override void OnCompleted(PollOperationOnCompletedResult result) { - Trace("Enter"); - - // Note we could be cancelling because of socket close. Regardless, we don't need the registration anymore. - CancellationRegistration.Dispose(); - - State newState; - while (true) - { - State state = _state; - if (state is State.Complete or State.Canceled or State.RunningWithPendingCancellation) - { - return false; - } - - newState = (state == State.Waiting ? State.Canceled : State.RunningWithPendingCancellation); - if (state == Interlocked.CompareExchange(ref _state, newState, state)) - { - break; - } - - // Race to update the state. Loop and try again. - } - - if (newState == State.RunningWithPendingCancellation) - { - // TryComplete will either succeed, or it will see the pending cancellation and deal with it. - return false; - } - - ProcessCancellation(); - - // Note, we leave the operation in the OperationQueue. - // When we get around to processing it, we'll see it's cancelled and skip it. - return true; - } - - public void ProcessCancellation() - { - Trace("Enter"); - - Debug.Assert(_state == State.Canceled); - - ErrorCode = SocketError.OperationAborted; - - ManualResetEventSlim? e = Event; - if (e != null) - { - e.Set(); - } - else - { #if DEBUG - Debug.Assert(!Interlocked.Exchange(ref _callbackQueued, true), $"Unexpected _callbackQueued: {_callbackQueued}"); + Debug.Assert(!Interlocked.Exchange(ref _callbackQueued, true), $"Unexpected _callbackQueued: {_callbackQueued}"); #endif - // We've marked the operation as canceled, and so should invoke the callback, but - // we can't pool the object, as ProcessQueue may still have a reference to it, due to - // using a pattern whereby it takes the lock to grab an item, but then releases the lock - // to do further processing on the item that's still in the list. - ThreadPool.UnsafeQueueUserWorkItem(o => ((AsyncOperation)o!).InvokeCallback(allowPooling: false), this); - } - } - - public void Dispatch() - { - ManualResetEventSlim? e = Event; - if (e != null) + if (result != PollOperationOnCompletedResult.Completed) { - // Sync operation. Signal waiting thread to continue processing. - e.Set(); + ErrorCode = SocketError.OperationAborted; } - else - { - // Async operation. - Schedule(); - } - } - - public void Schedule() - { - Debug.Assert(Event == null); - // Async operation. Process the IO on the threadpool. - ThreadPool.UnsafeQueueUserWorkItem(this, preferLocal: false); + InvokeCallback(allowPooling: result == PollOperationOnCompletedResult.Completed); } - public void Process() => ((IThreadPoolWorkItem)this).Execute(); - - void IThreadPoolWorkItem.Execute() - { - // ReadOperation and WriteOperation, the only two types derived from - // AsyncOperation, implement IThreadPoolWorkItem.Execute to call - // ProcessAsyncOperation(this) on the appropriate receive or send queue. - // However, this base class needs to be able to queue them without - // additional allocation, so it also implements the interface in order - // to pass the compiler's static checking for the interface, but then - // when the runtime queries for the interface, it'll use the derived - // type's interface implementation. We could instead just make this - // an abstract and have the derived types override it, but that adds - // "Execute" as a public method, which could easily be misunderstood. - // We could also add an abstract method that the base interface implementation - // invokes, but that adds an extra virtual dispatch. - Debug.Fail("Expected derived type to implement IThreadPoolWorkItem"); - throw new InvalidOperationException(); - } - - // Called when op is not in the queue yet, so can't be otherwise executing - public void DoAbort() - { - ErrorCode = SocketError.OperationAborted; - } - - protected abstract bool DoTryComplete(SocketAsyncContext context); + protected abstract bool TryCompleteOperation(SocketAsyncContext context); public abstract void InvokeCallback(bool allowPooling); - [Conditional("SOCKETASYNCCONTEXT_TRACE")] - public void Trace(string message, [CallerMemberName] string? memberName = null) - { - OutputTrace($"{IdOf(this)}.{memberName}: {message}"); - } - - [Conditional("SOCKETASYNCCONTEXT_TRACE")] - public void TraceWithContext(SocketAsyncContext context, string message, [CallerMemberName] string? memberName = null) - { - OutputTrace($"{IdOf(context)}, {IdOf(this)}.{memberName}: {message}"); - } - } - - // These two abstract classes differentiate the operations that go in the - // read queue vs the ones that go in the write queue. - private abstract class ReadOperation : AsyncOperation, IThreadPoolWorkItem - { - public ReadOperation(SocketAsyncContext context) : base(context) { } - - void IThreadPoolWorkItem.Execute() => AssociatedContext.ProcessAsyncReadOperation(this); - } - - private abstract class WriteOperation : AsyncOperation, IThreadPoolWorkItem - { - public WriteOperation(SocketAsyncContext context) : base(context) { } - - void IThreadPoolWorkItem.Execute() => AssociatedContext.ProcessAsyncWriteOperation(this); } - private abstract class SendOperation : WriteOperation + private abstract class SendOperation : AsyncOperation { public SocketFlags Flags; public int BytesTransferred; @@ -368,7 +176,7 @@ private class BufferMemorySendOperation : SendOperation public BufferMemorySendOperation(SocketAsyncContext context) : base(context) { } - protected override bool DoTryComplete(SocketAsyncContext context) + protected override bool TryCompleteOperation(SocketAsyncContext context) { int bufferIndex = 0; return SocketPal.TryCompleteSendTo(context._socket, Buffer.Span, null, ref bufferIndex, ref Offset, ref Count, Flags, SocketAddress.Span, ref BytesTransferred, out ErrorCode); @@ -397,7 +205,7 @@ private sealed class BufferListSendOperation : SendOperation public BufferListSendOperation(SocketAsyncContext context) : base(context) { } - protected override bool DoTryComplete(SocketAsyncContext context) + protected override bool TryCompleteOperation(SocketAsyncContext context) { return SocketPal.TryCompleteSendTo(context._socket, default(ReadOnlySpan), Buffers, ref BufferIndex, ref Offset, ref Count, Flags, SocketAddress.Span, ref BytesTransferred, out ErrorCode); } @@ -424,7 +232,7 @@ private sealed unsafe class BufferPtrSendOperation : SendOperation public BufferPtrSendOperation(SocketAsyncContext context) : base(context) { } - protected override bool DoTryComplete(SocketAsyncContext context) + protected override bool TryCompleteOperation(SocketAsyncContext context) { int bufferIndex = 0; int bufferLength = Offset + Count; // TryCompleteSendTo expects the entire buffer, which it then indexes into with the ref Offset and ref Count arguments @@ -432,7 +240,7 @@ protected override bool DoTryComplete(SocketAsyncContext context) } } - private abstract class ReceiveOperation : ReadOperation + private abstract class ReceiveOperation : AsyncOperation { public SocketFlags Flags; public SocketFlags ReceivedFlags; @@ -453,7 +261,7 @@ private sealed class BufferMemoryReceiveOperation : ReceiveOperation public BufferMemoryReceiveOperation(SocketAsyncContext context) : base(context) { } - protected override bool DoTryComplete(SocketAsyncContext context) + protected override bool TryCompleteOperation(SocketAsyncContext context) { // Zero byte read is performed to know when data is available. // We don't have to call receive, our caller is interested in the event. @@ -508,7 +316,7 @@ private sealed class BufferListReceiveOperation : ReceiveOperation public BufferListReceiveOperation(SocketAsyncContext context) : base(context) { } - protected override bool DoTryComplete(SocketAsyncContext context) + protected override bool TryCompleteOperation(SocketAsyncContext context) { bool completed = SocketPal.TryCompleteReceiveFrom(context._socket, default(Span), Buffers, Flags, SocketAddress.Span, out int socketAddressLen, out BytesTransferred, out ReceivedFlags, out ErrorCode); if (completed && ErrorCode == SocketError.Success) @@ -542,7 +350,7 @@ private sealed unsafe class BufferPtrReceiveOperation : ReceiveOperation public BufferPtrReceiveOperation(SocketAsyncContext context) : base(context) { } - protected override bool DoTryComplete(SocketAsyncContext context) + protected override bool TryCompleteOperation(SocketAsyncContext context) { bool completed = SocketPal.TryCompleteReceiveFrom(context._socket, new Span(BufferPtr, Length), null, Flags, SocketAddress.Span, out int socketAddressLen, out BytesTransferred, out ReceivedFlags, out ErrorCode); if (completed && ErrorCode == SocketError.Success) @@ -553,7 +361,7 @@ protected override bool DoTryComplete(SocketAsyncContext context) } } - private sealed class ReceiveMessageFromOperation : ReadOperation + private sealed class ReceiveMessageFromOperation : AsyncOperation { public Memory Buffer; public SocketFlags Flags; @@ -569,7 +377,7 @@ public ReceiveMessageFromOperation(SocketAsyncContext context) : base(context) { public Action, SocketFlags, IPPacketInformation, SocketError>? Callback { get; set; } - protected override bool DoTryComplete(SocketAsyncContext context) + protected override bool TryCompleteOperation(SocketAsyncContext context) { bool completed = SocketPal.TryCompleteReceiveMessageFrom(context._socket, Buffer.Span, Buffers, Flags, SocketAddress, out int socketAddressLen, IsIPv4, IsIPv6, out BytesTransferred, out ReceivedFlags, out IPPacketInformation, out ErrorCode); if (completed && ErrorCode == SocketError.Success) @@ -583,7 +391,7 @@ public override void InvokeCallback(bool allowPooling) => Callback!(BytesTransferred, SocketAddress, ReceivedFlags, IPPacketInformation, ErrorCode); } - private sealed unsafe class BufferPtrReceiveMessageFromOperation : ReadOperation + private sealed unsafe class BufferPtrReceiveMessageFromOperation : AsyncOperation { public byte* BufferPtr; public int Length; @@ -599,7 +407,7 @@ public BufferPtrReceiveMessageFromOperation(SocketAsyncContext context) : base(c public Action, SocketFlags, IPPacketInformation, SocketError>? Callback { get; set; } - protected override bool DoTryComplete(SocketAsyncContext context) + protected override bool TryCompleteOperation(SocketAsyncContext context) { bool completed = SocketPal.TryCompleteReceiveMessageFrom(context._socket, new Span(BufferPtr, Length), null, Flags, SocketAddress!, out int socketAddressLen, IsIPv4, IsIPv6, out BytesTransferred, out ReceivedFlags, out IPPacketInformation, out ErrorCode); if (completed && ErrorCode == SocketError.Success) @@ -613,7 +421,7 @@ public override void InvokeCallback(bool allowPooling) => Callback!(BytesTransferred, SocketAddress, ReceivedFlags, IPPacketInformation, ErrorCode); } - private sealed class AcceptOperation : ReadOperation + private sealed class AcceptOperation : AsyncOperation { public IntPtr AcceptedFileDescriptor; @@ -621,7 +429,7 @@ public AcceptOperation(SocketAsyncContext context) : base(context) { } public Action, SocketError>? Callback { get; set; } - protected override bool DoTryComplete(SocketAsyncContext context) + protected override bool TryCompleteOperation(SocketAsyncContext context) { bool completed = SocketPal.TryCompleteAccept(context._socket, SocketAddress, out int socketAddressLen, out AcceptedFileDescriptor, out ErrorCode); Debug.Assert(ErrorCode == SocketError.Success || AcceptedFileDescriptor == (IntPtr)(-1), $"Unexpected values: ErrorCode={ErrorCode}, AcceptedFileDescriptor={AcceptedFileDescriptor}"); @@ -652,7 +460,7 @@ private sealed class ConnectOperation : BufferMemorySendOperation { public ConnectOperation(SocketAsyncContext context) : base(context) { } - protected override bool DoTryComplete(SocketAsyncContext context) + protected override bool TryCompleteOperation(SocketAsyncContext context) { bool result = SocketPal.TryCompleteConnect(context._socket, out ErrorCode); context._socket.RegisterConnectResult(ErrorCode); @@ -688,7 +496,7 @@ public override void InvokeCallback(bool allowPooling) } } - private sealed class SendFileOperation : WriteOperation + private sealed class SendFileOperation : AsyncOperation { public SafeFileHandle FileHandle = null!; // always set when constructed public long Offset; @@ -702,644 +510,54 @@ public SendFileOperation(SocketAsyncContext context) : base(context) { } public override void InvokeCallback(bool allowPooling) => Callback!(BytesTransferred, ErrorCode); - protected override bool DoTryComplete(SocketAsyncContext context) => + protected override bool TryCompleteOperation(SocketAsyncContext context) => SocketPal.TryCompleteSendFile(context._socket, FileHandle, ref Offset, ref Count, ref BytesTransferred, out ErrorCode); } - // In debug builds, this struct guards against: - // (1) Unexpected lock reentrancy, which should never happen - // (2) Deadlock, by setting a reasonably large timeout - private readonly struct LockToken : IDisposable - { - private readonly object _lockObject; - - public LockToken(object lockObject) - { - Debug.Assert(lockObject != null); - - _lockObject = lockObject; - - Debug.Assert(!Monitor.IsEntered(_lockObject)); - -#if DEBUG - bool success = Monitor.TryEnter(_lockObject, 10000); - Debug.Assert(success, "Timed out waiting for queue lock"); -#else - Monitor.Enter(_lockObject); -#endif - } - - public void Dispose() - { - Debug.Assert(Monitor.IsEntered(_lockObject)); - Monitor.Exit(_lockObject); - } - } + internal static readonly bool InlineSocketCompletionsEnabled = Environment.GetEnvironmentVariable("DOTNET_SYSTEM_NET_SOCKETS_INLINE_COMPLETIONS") == "1"; - public enum OperationResult - { - Pending = 0, - Completed = 1, - Cancelled = 2 - } + internal readonly SafeSocketHandle _socket; + private PollableHandle? _pollHandle; + private bool _isHandleNonBlocking = OperatingSystem.IsWasi(); // WASI sockets are always non-blocking, because we don't have another thread which could be blocked - private struct OperationQueue - where TOperation : AsyncOperation + // Socket.PreferInlineCompletions is an experimental API with internal access modifier. + // DynamicDependency ensures the setter is available externally using reflection. + [DynamicDependency("set_PreferInlineCompletions", typeof(Socket))] + internal void SetInlineCompletions(bool value) { - // Quick overview: - // - // When attempting to perform an IO operation, the caller first checks IsReady, - // and if true, attempts to perform the operation itself. - // If this returns EWOULDBLOCK, or if the queue was not ready, then the operation - // is enqueued by calling StartAsyncOperation and the state becomes Waiting. - // When an epoll notification is received, we check if the state is Waiting, - // and if so, change the state to Processing and enqueue a workitem to the threadpool - // to try to perform the enqueued operations. - // If an operation is successfully performed, we remove it from the queue, - // enqueue another threadpool workitem to process the next item in the queue (if any), - // and call the user's completion callback. - // If we successfully process all enqueued operations, then the state becomes Ready; - // otherwise, the state becomes Waiting and we wait for another epoll notification. - - private enum QueueState : int - { - Ready = 0, // Indicates that data MAY be available on the socket. - // Queue must be empty. - Waiting = 1, // Indicates that data is definitely not available on the socket. - // Queue must not be empty. - Processing = 2, // Indicates that a thread pool item has been scheduled (and may - // be executing) to process the IO operations in the queue. - // Queue must not be empty. - Stopped = 3, // Indicates that the queue has been stopped because the - // socket has been closed. - // Queue must be empty. - } - - // These fields define the queue state. - - private QueueState _state; // See above - private bool _isNextOperationSynchronous; - private int _sequenceNumber; // This sequence number is updated when we receive an epoll notification. - // It allows us to detect when a new epoll notification has arrived - // since the last time we checked the state of the queue. - // If this happens, we MUST retry the operation, otherwise we risk - // "losing" the notification and causing the operation to pend indefinitely. - private AsyncOperation? _tail; // Queue of pending IO operations to process when data becomes available. - - // The _queueLock is used to ensure atomic access to the queue state above. - // The lock is only ever held briefly, to read and/or update queue state, and - // never around any external call, e.g. OS call or user code invocation. - private object _queueLock; - - private LockToken Lock() => new LockToken(_queueLock); - - public bool IsNextOperationSynchronous_Speculative => _isNextOperationSynchronous; - - public void Init() - { - Debug.Assert(_queueLock == null); - _queueLock = new object(); - - _state = QueueState.Ready; - _sequenceNumber = 0; - } - - // IsReady returns whether an operation can be executed immediately. - // observedSequenceNumber must be passed to StartAsyncOperation. - public bool IsReady(SocketAsyncContext context, out int observedSequenceNumber) + if (SocketAsyncContext.InlineSocketCompletionsEnabled) { - // It is safe to read _state and _sequence without using Lock. - // - The return value is soley based on Volatile.Read of _state. - // - The Volatile.Read of _sequenceNumber ensures we read a value before executing the operation. - // This is needed to retry the operation in StartAsyncOperation in case the _sequenceNumber incremented. - // - Because no Lock is taken, it is possible we observe a sequence number increment before the state - // becomes Ready. When that happens, observedSequenceNumber is decremented, and StartAsyncOperation will - // execute the operation because the sequence number won't match. - - Debug.Assert(sizeof(QueueState) == sizeof(int)); - QueueState state = (QueueState)Volatile.Read(ref Unsafe.As(ref _state)); - observedSequenceNumber = Volatile.Read(ref _sequenceNumber); - - bool isReady = state == QueueState.Ready || state == QueueState.Stopped; - if (!isReady) - { - observedSequenceNumber--; - } - - Trace(context, $"{isReady}"); - - return isReady; - } - - // Return true for pending, false for completed synchronously (including failure and abort) - public bool StartAsyncOperation(SocketAsyncContext context, TOperation operation, int observedSequenceNumber, CancellationToken cancellationToken = default) - { - Trace(context, $"Enter"); - - if (!context.IsRegistered && !context.TryRegister(out Interop.Error error)) - { - HandleFailedRegistration(context, operation, error); - - Trace(context, "Leave, not registered"); - return false; - } - - while (true) - { - bool doAbort = false; - using (Lock()) - { - switch (_state) - { - case QueueState.Ready: - if (observedSequenceNumber != _sequenceNumber) - { - // The queue has become ready again since we previously checked it. - // So, we need to retry the operation before we enqueue it. - Debug.Assert(observedSequenceNumber - _sequenceNumber < 10000, "Very large sequence number increase???"); - observedSequenceNumber = _sequenceNumber; - break; - } - - // Caller tried the operation and got an EWOULDBLOCK, so we need to transition. - _state = QueueState.Waiting; - goto case QueueState.Waiting; - - case QueueState.Waiting: - case QueueState.Processing: - // Enqueue the operation. - Debug.Assert(operation.Next == operation, "Expected operation.Next == operation"); - - if (_tail == null) - { - Debug.Assert(!_isNextOperationSynchronous); - _isNextOperationSynchronous = operation.Event != null; - } - else - { - operation.Next = _tail.Next; - _tail.Next = operation; - } - - _tail = operation; - Trace(context, $"Leave, enqueued {IdOf(operation)}"); - - // Now that the object is enqueued, hook up cancellation. - // Note that it's possible the call to register itself could - // call TryCancel, so we do this after the op is fully enqueued. - if (cancellationToken.CanBeCanceled) - { - operation.CancellationRegistration = cancellationToken.UnsafeRegister(s => ((TOperation)s!).TryCancel(), operation); - } - - return true; - - case QueueState.Stopped: - Debug.Assert(_tail == null); - doAbort = true; - break; - - default: - Environment.FailFast("unexpected queue state"); - break; - } - } - - if (doAbort) - { - operation.DoAbort(); - Trace(context, $"Leave, queue stopped"); - return false; - } - - // Retry the operation. - if (operation.TryComplete(context) != OperationResult.Pending) - { - Trace(context, $"Leave, retry succeeded"); - return false; - } - } - - static void HandleFailedRegistration(SocketAsyncContext context, TOperation operation, Interop.Error error) - { - Debug.Assert(error != Interop.Error.SUCCESS); - - // macOS: kevent returns EPIPE when adding pipe fd for which the other end is closed. - if (error == Interop.Error.EPIPE) - { - // Because the other end close, we expect the operation to complete when we retry it. - // If it doesn't, we fall through and throw an Exception. - if (operation.TryComplete(context) != OperationResult.Pending) - { - return; - } - } - - if (error == Interop.Error.ENOMEM || error == Interop.Error.ENOSPC) - { - throw new OutOfMemoryException(); - } - else - { - throw new InternalException(error); - } - } - } - - public AsyncOperation? ProcessSyncEventOrGetAsyncEvent(SocketAsyncContext context, bool skipAsyncEvents = false) - { - AsyncOperation op; - using (Lock()) - { - Trace(context, $"Enter"); - - switch (_state) - { - case QueueState.Ready: - Debug.Assert(_tail == null, "State == Ready but queue is not empty!"); - _sequenceNumber++; - Trace(context, $"Exit (previously ready)"); - return null; - - case QueueState.Waiting: - Debug.Assert(_tail != null, "State == Waiting but queue is empty!"); - op = _tail.Next; - Debug.Assert(_isNextOperationSynchronous == (op.Event != null)); - if (skipAsyncEvents && !_isNextOperationSynchronous) - { - // Return the operation to indicate that the async operation was not processed, without making - // any state changes because async operations are being skipped - return op; - } - - _state = QueueState.Processing; - // Break out and release lock - break; - - case QueueState.Processing: - Debug.Assert(_tail != null, "State == Processing but queue is empty!"); - _sequenceNumber++; - Trace(context, $"Exit (currently processing)"); - return null; - - case QueueState.Stopped: - Debug.Assert(_tail == null); - Trace(context, $"Exit (stopped)"); - return null; - - default: - Environment.FailFast("unexpected queue state"); - return null; - } - } - - ManualResetEventSlim? e = op.Event; - if (e != null) - { - // Sync operation. Signal waiting thread to continue processing. - e.Set(); - return null; - } - else - { - // Async operation. The caller will figure out how to process the IO. - Debug.Assert(!skipAsyncEvents); - return op; - } - } - - internal void ProcessAsyncOperation(TOperation op) - { - OperationResult result = ProcessQueuedOperation(op); - - Debug.Assert(op.Event == null, "Sync operation encountered in ProcessAsyncOperation"); - - if (result == OperationResult.Completed) - { - // At this point, the operation has completed and it's no longer - // in the queue / no one else has a reference to it. We can invoke - // the callback and let it pool the object if appropriate. This is - // also a good time to unregister from cancellation; we must do - // so before the object is returned to the pool (or else a cancellation - // request for a previous operation could affect a subsequent one) - // and here we know the operation has completed. - op.CancellationRegistration.Dispose(); - op.InvokeCallback(allowPooling: true); - } - } - - public OperationResult ProcessQueuedOperation(TOperation op) - { - SocketAsyncContext context = op.AssociatedContext; - - int observedSequenceNumber; - using (Lock()) - { - Trace(context, $"Enter"); - - if (_state == QueueState.Stopped) - { - Debug.Assert(_tail == null); - Trace(context, $"Exit (stopped)"); - return OperationResult.Cancelled; - } - else - { - Debug.Assert(_state == QueueState.Processing, $"_state={_state} while processing queue!"); - Debug.Assert(_tail != null, "Unexpected empty queue while processing I/O"); - Debug.Assert(op == _tail.Next, "Operation is not at head of queue???"); - observedSequenceNumber = _sequenceNumber; - } - } - - OperationResult result; - while (true) - { - result = op.TryComplete(context); - if (result != OperationResult.Pending) - { - break; - } - - // Check for retry and reset queue state. - - using (Lock()) - { - if (_state == QueueState.Stopped) - { - Debug.Assert(_tail == null); - Trace(context, $"Exit (stopped)"); - return OperationResult.Cancelled; - } - else - { - Debug.Assert(_state == QueueState.Processing, $"_state={_state} while processing queue!"); - - if (observedSequenceNumber != _sequenceNumber) - { - // We received another epoll notification since we previously checked it. - // So, we need to retry the operation. - Debug.Assert(observedSequenceNumber - _sequenceNumber < 10000, "Very large sequence number increase???"); - observedSequenceNumber = _sequenceNumber; - } - else - { - _state = QueueState.Waiting; - Trace(context, $"Exit (received EAGAIN)"); - return OperationResult.Pending; - } - } - } - } - - // Remove the op from the queue and see if there's more to process. - - AsyncOperation? nextOp = null; - using (Lock()) - { - if (_state == QueueState.Stopped) - { - Debug.Assert(_tail == null); - Trace(context, $"Exit (stopped)"); - } - else - { - Debug.Assert(_state == QueueState.Processing, $"_state={_state} while processing queue!"); - Debug.Assert(_tail.Next == op, "Queue modified while processing queue"); - - if (op == _tail) - { - // No more operations to process - _tail = null; - _isNextOperationSynchronous = false; - _state = QueueState.Ready; - _sequenceNumber++; - Trace(context, $"Exit (finished queue)"); - } - else - { - // Pop current operation and advance to next - nextOp = _tail.Next = op.Next; - _isNextOperationSynchronous = nextOp.Event != null; - } - } - } - - nextOp?.Dispatch(); - - Debug.Assert(result != OperationResult.Pending); - return result; + // Ignore value. All completions are inline. + return; } - public void CancelAndContinueProcessing(TOperation op) + if (_pollHandle is not null || value) { - // Note, only sync operations use this method. - Debug.Assert(op.Event != null); - - // Remove operation from queue. - // Note it must be there since it can only be processed and removed by the caller. - AsyncOperation? nextOp = null; - using (Lock()) - { - if (_state == QueueState.Stopped) - { - Debug.Assert(_tail == null); - } - else - { - Debug.Assert(_tail != null, "Unexpected empty queue in CancelAndContinueProcessing"); - - if (_tail.Next == op) - { - // We're the head of the queue - if (op == _tail) - { - // No more operations - _tail = null; - _isNextOperationSynchronous = false; - } - else - { - // Pop current operation and advance to next - _tail.Next = op.Next; - _isNextOperationSynchronous = op.Next.Event != null; - } - - // We're the first op in the queue. - if (_state == QueueState.Processing) - { - // The queue has already handed off execution responsibility to us. - // We need to dispatch to the next op. - if (_tail == null) - { - _state = QueueState.Ready; - _sequenceNumber++; - } - else - { - nextOp = _tail.Next; - } - } - else if (_state == QueueState.Waiting) - { - if (_tail == null) - { - _state = QueueState.Ready; - _sequenceNumber++; - } - } - } - else - { - // We're not the head of the queue. - // Just find this op and remove it. - AsyncOperation current = _tail.Next; - while (current.Next != op) - { - current = current.Next; - } - - if (current.Next == _tail) - { - _tail = current; - } - current.Next = current.Next.Next; - } - } - } - - nextOp?.Dispatch(); + PollHandle.InlineCompletions = value; } + } - // Called when the socket is closed. - public bool StopAndAbort(SocketAsyncContext context) + internal PollableHandle PollHandle + { + get { - bool aborted = false; - - // We should be called exactly once, by SafeSocketHandle. - Debug.Assert(_state != QueueState.Stopped); - - using (Lock()) + if (_pollHandle == null) { - Trace(context, $"Enter"); - - Debug.Assert(_state != QueueState.Stopped); - - _state = QueueState.Stopped; - - if (_tail != null) - { - AsyncOperation op = _tail; - do - { - aborted |= op.TryCancel(); - op = op.Next; - } while (op != _tail); - } - - _tail = null; - _isNextOperationSynchronous = false; - - Trace(context, $"Exit"); + PollableHandle ph = PollableHandle.Create(_socket, ref _pollHandle); + ph.InlineCompletions = InlineSocketCompletionsEnabled; } - - return aborted; - } - - [Conditional("SOCKETASYNCCONTEXT_TRACE")] - public void Trace(SocketAsyncContext context, string message, [CallerMemberName] string? memberName = null) - { - string queueType = - typeof(TOperation) == typeof(ReadOperation) ? "recv" : - typeof(TOperation) == typeof(WriteOperation) ? "send" : - "???"; - - OutputTrace($"{IdOf(context)}-{queueType}.{memberName}: {message}, {_state}-{_sequenceNumber}, {((_tail == null) ? "empty" : "not empty")}"); + return _pollHandle!; } } - internal readonly SafeSocketHandle _socket; - private OperationQueue _receiveQueue; - private OperationQueue _sendQueue; - private SocketAsyncEngine? _asyncEngine; - private bool IsRegistered => _asyncEngine != null; - private bool _isHandleNonBlocking = OperatingSystem.IsWasi(); // WASI sockets are always non-blocking, because we don't have another thread which could be blocked - /// An index into 's table of all contexts that are currently . - internal int GlobalContextIndex = -1; - - private readonly object _registerLock = new object(); - public SocketAsyncContext(SafeSocketHandle socket) { _socket = socket; - - _receiveQueue.Init(); - _sendQueue.Init(); - } - - public bool PreferInlineCompletions - { - // Socket.PreferInlineCompletions is an experimental API with internal access modifier. - // DynamicDependency ensures the setter is available externally using reflection. - [DynamicDependency("set_PreferInlineCompletions", typeof(Socket))] - get => _socket.PreferInlineCompletions; - } - - private bool TryRegister(out Interop.Error error) - { - Debug.Assert(_isHandleNonBlocking); - lock (_registerLock) - { - if (_asyncEngine == null) - { - bool addedRef = false; - try - { - _socket.DangerousAddRef(ref addedRef); - IntPtr handle = _socket.DangerousGetHandle(); - if (SocketAsyncEngine.TryRegisterSocket(handle, this, out SocketAsyncEngine? engine, out error)) - { - Volatile.Write(ref _asyncEngine, engine); - - Trace("Registered"); - return true; - } - else - { - Trace("Registration failed"); - return false; - } - } - finally - { - if (addedRef) - { - _socket.DangerousRelease(); - } - } - } - error = Interop.Error.SUCCESS; - return true; - } } public bool StopAndAbort() { - bool aborted = false; - - // Drain queues - aborted |= _sendQueue.StopAndAbort(this); - aborted |= _receiveQueue.StopAndAbort(this); - - // We don't need to synchronize with Register. - // This method is called when the handle gets released. - // The Register method will throw ODE when it tries to use the handle at this point. - if (IsRegistered) - { - SocketAsyncEngine.UnregisterSocket(this); - } - - return aborted; + return _pollHandle?.AbortAndDispose() ?? false; } public void SetHandleNonBlocking() @@ -1390,63 +608,22 @@ public void SetHandleBlocking() } } - private void PerformSyncOperation(ref OperationQueue queue, TOperation operation, int timeout, int observedSequenceNumber) - where TOperation : AsyncOperation + private void PerformSyncOperation(AsyncOperation operation, bool isRead, int timeout, int observedSequenceNumber) { if (!Socket.OSSupportsThreads) throw new PlatformNotSupportedException(); Debug.Assert(timeout == -1 || timeout > 0, $"Unexpected timeout: {timeout}"); - using (var e = new ManualResetEventSlim(false, 0)) - { - operation.Event = e; - - if (!queue.StartAsyncOperation(this, operation, observedSequenceNumber)) - { - // Completed synchronously - return; - } - - bool timeoutExpired = false; - while (true) - { - long waitStart = Stopwatch.GetTimestamp(); - - if (!e.Wait(timeout)) - { - timeoutExpired = true; - break; - } - - // Reset the event now to avoid lost notifications if the processing is unsuccessful. - e.Reset(); - - // We've been signalled to try to process the operation. - OperationResult result = queue.ProcessQueuedOperation(operation); - if (result == OperationResult.Completed || - result == OperationResult.Cancelled) - { - break; - } - - // Couldn't process the operation. - // Adjust timeout and try again. - if (timeout > 0) - { - timeout -= (int)Stopwatch.GetElapsedTime(waitStart).TotalMilliseconds; - - if (timeout <= 0) - { - timeoutExpired = true; - break; - } - } - } + PollOperationSyncResult result = + isRead ? PollHandle.ReadSync(operation, observedSequenceNumber, timeout) + : PollHandle.WriteSync(operation, observedSequenceNumber, timeout); - if (timeoutExpired) - { - queue.CancelAndContinueProcessing(operation); - operation.ErrorCode = SocketError.TimedOut; - } + if (result == PollOperationSyncResult.TimedOut) + { + operation.ErrorCode = SocketError.TimedOut; + } + else if (result == PollOperationSyncResult.Aborted) + { + operation.ErrorCode = SocketError.OperationAborted; } } @@ -1463,17 +640,13 @@ private bool ShouldRetrySyncOperation(out SocketError errorCode) return false; } - private void ProcessAsyncReadOperation(ReadOperation op) => _receiveQueue.ProcessAsyncOperation(op); - - private void ProcessAsyncWriteOperation(WriteOperation op) => _sendQueue.ProcessAsyncOperation(op); - public SocketError Accept(Memory socketAddress, out int socketAddressLen, out IntPtr acceptedFd) { Debug.Assert(socketAddress.Length > 0, $"Unexpected socketAddressLen: {socketAddress.Length}"); SocketError errorCode; int observedSequenceNumber; - if (_receiveQueue.IsReady(this, out observedSequenceNumber) && + if (PollHandle.IsReadReady(out observedSequenceNumber) && SocketPal.TryCompleteAccept(_socket, socketAddress, out socketAddressLen, out acceptedFd, out errorCode)) { Debug.Assert(errorCode == SocketError.Success || acceptedFd == (IntPtr)(-1), $"Unexpected values: errorCode={errorCode}, acceptedFd={acceptedFd}"); @@ -1485,7 +658,7 @@ public SocketError Accept(Memory socketAddress, out int socketAddressLen, SocketAddress = socketAddress, }; - PerformSyncOperation(ref _receiveQueue, operation, -1, observedSequenceNumber); + PerformSyncOperation(operation, isRead: true, -1, observedSequenceNumber); socketAddressLen = operation.SocketAddress.Length; acceptedFd = operation.AcceptedFileDescriptor; @@ -1501,7 +674,7 @@ public SocketError AcceptAsync(Memory socketAddress, out int socketAddress SocketError errorCode; int observedSequenceNumber; - if (_receiveQueue.IsReady(this, out observedSequenceNumber) && + if (PollHandle.IsReadReady(out observedSequenceNumber) && SocketPal.TryCompleteAccept(_socket, socketAddress, out socketAddressLen, out acceptedFd, out errorCode)) { Debug.Assert(errorCode == SocketError.Success || acceptedFd == (IntPtr)(-1), $"Unexpected values: errorCode={errorCode}, acceptedFd={acceptedFd}"); @@ -1513,11 +686,13 @@ public SocketError AcceptAsync(Memory socketAddress, out int socketAddress operation.Callback = callback; operation.SocketAddress = socketAddress; - if (!_receiveQueue.StartAsyncOperation(this, operation, observedSequenceNumber, cancellationToken)) + PollOperationAsyncResult result = PollHandle.ReadAsync(operation, observedSequenceNumber, cancellationToken); + + if (result == PollOperationAsyncResult.Completed) { + errorCode = operation.ErrorCode; socketAddressLen = operation.SocketAddress.Length; acceptedFd = operation.AcceptedFileDescriptor; - errorCode = operation.ErrorCode; ReturnOperation(operation); return errorCode; @@ -1525,7 +700,7 @@ public SocketError AcceptAsync(Memory socketAddress, out int socketAddress acceptedFd = (IntPtr)(-1); socketAddressLen = 0; - return SocketError.IOPending; + return GetSocketErrorForNonCompleted(result); } public SocketError Connect(Memory socketAddress) @@ -1539,7 +714,7 @@ public SocketError Connect(Memory socketAddress) // Thus, always call TryStartConnect regardless of readiness. SocketError errorCode; int observedSequenceNumber; - _sendQueue.IsReady(this, out observedSequenceNumber); + PollHandle.IsWriteReady(out observedSequenceNumber); if (SocketPal.TryStartConnect(_socket, socketAddress, out errorCode) || !ShouldRetrySyncOperation(out errorCode)) { @@ -1552,7 +727,7 @@ public SocketError Connect(Memory socketAddress) SocketAddress = socketAddress, }; - PerformSyncOperation(ref _sendQueue, operation, -1, observedSequenceNumber); + PerformSyncOperation(operation, isRead: false, -1, observedSequenceNumber); return operation.ErrorCode; } @@ -1569,7 +744,7 @@ public SocketError ConnectAsync(Memory socketAddress, Action socketAddress, Action buffer, SocketFlags flags, int timeout, out int bytesReceived) @@ -1642,7 +820,7 @@ public SocketError ReceiveFrom(Memory buffer, ref SocketFlags flags, Memor SocketFlags receivedFlags; SocketError errorCode; int observedSequenceNumber; - if (_receiveQueue.IsReady(this, out observedSequenceNumber) && + if (PollHandle.IsReadReady(out observedSequenceNumber) && (SocketPal.TryCompleteReceiveFrom(_socket, buffer.Span, flags, socketAddress.Span, out socketAddressLen, out bytesReceived, out receivedFlags, out errorCode) || !ShouldRetrySyncOperation(out errorCode))) { @@ -1658,7 +836,7 @@ public SocketError ReceiveFrom(Memory buffer, ref SocketFlags flags, Memor SocketAddress = socketAddress, }; - PerformSyncOperation(ref _receiveQueue, operation, timeout, observedSequenceNumber); + PerformSyncOperation(operation, isRead: true, timeout, observedSequenceNumber); flags = operation.ReceivedFlags; bytesReceived = operation.BytesTransferred; @@ -1673,7 +851,7 @@ public unsafe SocketError ReceiveFrom(Span buffer, ref SocketFlags flags, SocketFlags receivedFlags; SocketError errorCode; int observedSequenceNumber; - if (_receiveQueue.IsReady(this, out observedSequenceNumber) && + if (PollHandle.IsReadReady(out observedSequenceNumber) && (SocketPal.TryCompleteReceiveFrom(_socket, buffer, flags, socketAddress.Span, out socketAddressLen, out bytesReceived, out receivedFlags, out errorCode) || !ShouldRetrySyncOperation(out errorCode))) { @@ -1691,7 +869,7 @@ public unsafe SocketError ReceiveFrom(Span buffer, ref SocketFlags flags, SocketAddress = socketAddress, }; - PerformSyncOperation(ref _receiveQueue, operation, timeout, observedSequenceNumber); + PerformSyncOperation(operation, isRead: true, timeout, observedSequenceNumber); flags = operation.ReceivedFlags; bytesReceived = operation.BytesTransferred; @@ -1706,7 +884,7 @@ public SocketError ReceiveAsync(Memory buffer, SocketFlags flags, out int SocketError errorCode; int observedSequenceNumber; - if (_receiveQueue.IsReady(this, out observedSequenceNumber) && + if (PollHandle.IsReadReady(out observedSequenceNumber) && SocketPal.TryCompleteReceive(_socket, buffer.Span, flags, out bytesReceived, out errorCode)) { return errorCode; @@ -1719,17 +897,19 @@ public SocketError ReceiveAsync(Memory buffer, SocketFlags flags, out int operation.Flags = flags; operation.SocketAddress = default; - if (!_receiveQueue.StartAsyncOperation(this, operation, observedSequenceNumber, cancellationToken)) + PollOperationAsyncResult result = PollHandle.ReadAsync(operation, observedSequenceNumber, cancellationToken); + + if (result == PollOperationAsyncResult.Completed) { - bytesReceived = operation.BytesTransferred; errorCode = operation.ErrorCode; + bytesReceived = operation.BytesTransferred; ReturnOperation(operation); return errorCode; } bytesReceived = 0; - return SocketError.IOPending; + return GetSocketErrorForNonCompleted(result); } public SocketError ReceiveFromAsync(Memory buffer, SocketFlags flags, Memory socketAddress, out int socketAddressLen, out int bytesReceived, out SocketFlags receivedFlags, Action, SocketFlags, SocketError> callback, CancellationToken cancellationToken = default) @@ -1738,7 +918,7 @@ public SocketError ReceiveFromAsync(Memory buffer, SocketFlags flags, Memo SocketError errorCode; int observedSequenceNumber; - if (_receiveQueue.IsReady(this, out observedSequenceNumber) && + if (PollHandle.IsReadReady(out observedSequenceNumber) && SocketPal.TryCompleteReceiveFrom(_socket, buffer.Span, flags, socketAddress.Span, out socketAddressLen, out bytesReceived, out receivedFlags, out errorCode)) { return errorCode; @@ -1751,11 +931,13 @@ public SocketError ReceiveFromAsync(Memory buffer, SocketFlags flags, Memo operation.Flags = flags; operation.SocketAddress = socketAddress; - if (!_receiveQueue.StartAsyncOperation(this, operation, observedSequenceNumber, cancellationToken)) + PollOperationAsyncResult result = PollHandle.ReadAsync(operation, observedSequenceNumber, cancellationToken); + + if (result == PollOperationAsyncResult.Completed) { + errorCode = operation.ErrorCode; receivedFlags = operation.ReceivedFlags; bytesReceived = operation.BytesTransferred; - errorCode = operation.ErrorCode; socketAddressLen = operation.SocketAddress.Length; ReturnOperation(operation); @@ -1765,7 +947,7 @@ public SocketError ReceiveFromAsync(Memory buffer, SocketFlags flags, Memo bytesReceived = 0; socketAddressLen = 0; receivedFlags = SocketFlags.None; - return SocketError.IOPending; + return GetSocketErrorForNonCompleted(result); } public SocketError Receive(IList> buffers, SocketFlags flags, int timeout, out int bytesReceived) @@ -1787,7 +969,7 @@ public SocketError ReceiveFrom(IList> buffers, ref SocketFlag SocketFlags receivedFlags; SocketError errorCode; int observedSequenceNumber; - if (_receiveQueue.IsReady(this, out observedSequenceNumber) && + if (PollHandle.IsReadReady(out observedSequenceNumber) && (SocketPal.TryCompleteReceiveFrom(_socket, buffers, flags, socketAddress.Span, out socketAddressLen, out bytesReceived, out receivedFlags, out errorCode) || !ShouldRetrySyncOperation(out errorCode))) { @@ -1802,7 +984,7 @@ public SocketError ReceiveFrom(IList> buffers, ref SocketFlag SocketAddress = socketAddress, }; - PerformSyncOperation(ref _receiveQueue, operation, timeout, observedSequenceNumber); + PerformSyncOperation(operation, isRead: true, timeout, observedSequenceNumber); socketAddressLen = operation.SocketAddress.Length; flags = operation.ReceivedFlags; @@ -1816,7 +998,7 @@ public SocketError ReceiveFromAsync(IList> buffers, SocketFla SocketError errorCode; int observedSequenceNumber; - if (_receiveQueue.IsReady(this, out observedSequenceNumber) && + if (PollHandle.IsReadReady(out observedSequenceNumber) && SocketPal.TryCompleteReceiveFrom(_socket, buffers, flags, socketAddress.Span, out socketAddressLen, out bytesReceived, out receivedFlags, out errorCode)) { // Synchronous success or failure @@ -1829,12 +1011,14 @@ public SocketError ReceiveFromAsync(IList> buffers, SocketFla operation.Flags = flags; operation.SocketAddress = socketAddress; - if (!_receiveQueue.StartAsyncOperation(this, operation, observedSequenceNumber)) + PollOperationAsyncResult result = PollHandle.ReadAsync(operation, observedSequenceNumber, default); + + if (result == PollOperationAsyncResult.Completed) { + errorCode = operation.ErrorCode; socketAddressLen = operation.SocketAddress.Length; receivedFlags = operation.ReceivedFlags; bytesReceived = operation.BytesTransferred; - errorCode = operation.ErrorCode; ReturnOperation(operation); return errorCode; @@ -1843,7 +1027,7 @@ public SocketError ReceiveFromAsync(IList> buffers, SocketFla receivedFlags = SocketFlags.None; socketAddressLen = 0; bytesReceived = 0; - return SocketError.IOPending; + return GetSocketErrorForNonCompleted(result); } public SocketError ReceiveMessageFrom( @@ -1856,7 +1040,7 @@ public SocketError ReceiveMessageFrom( SocketFlags receivedFlags; SocketError errorCode; int observedSequenceNumber; - if (_receiveQueue.IsReady(this, out observedSequenceNumber) && + if (PollHandle.IsReadReady(out observedSequenceNumber) && (SocketPal.TryCompleteReceiveMessageFrom(_socket, buffer.Span, null, flags, socketAddress, out socketAddressLen, isIPv4, isIPv6, out bytesReceived, out receivedFlags, out ipPacketInformation, out errorCode) || !ShouldRetrySyncOperation(out errorCode))) { @@ -1874,7 +1058,7 @@ public SocketError ReceiveMessageFrom( IsIPv6 = isIPv6, }; - PerformSyncOperation(ref _receiveQueue, operation, timeout, observedSequenceNumber); + PerformSyncOperation(operation, isRead: true, timeout, observedSequenceNumber); socketAddressLen = operation.SocketAddress.Length; flags = operation.ReceivedFlags; @@ -1893,7 +1077,7 @@ public unsafe SocketError ReceiveMessageFrom( SocketFlags receivedFlags; SocketError errorCode; int observedSequenceNumber; - if (_receiveQueue.IsReady(this, out observedSequenceNumber) && + if (PollHandle.IsReadReady(out observedSequenceNumber) && (SocketPal.TryCompleteReceiveMessageFrom(_socket, buffer, null, flags, socketAddress, out socketAddressLen, isIPv4, isIPv6, out bytesReceived, out receivedFlags, out ipPacketInformation, out errorCode) || !ShouldRetrySyncOperation(out errorCode))) { @@ -1913,7 +1097,7 @@ public unsafe SocketError ReceiveMessageFrom( IsIPv6 = isIPv6, }; - PerformSyncOperation(ref _receiveQueue, operation, timeout, observedSequenceNumber); + PerformSyncOperation(operation, isRead: true, timeout, observedSequenceNumber); socketAddressLen = operation.SocketAddress.Length; flags = operation.ReceivedFlags; @@ -1929,7 +1113,7 @@ public SocketError ReceiveMessageFromAsync(Memory buffer, IList buffer, IList buffer, SocketFlags flags, int timeout, out int bytesSent) => @@ -1985,7 +1172,7 @@ public SocketError SendTo(byte[] buffer, int offset, int count, SocketFlags flag bytesSent = 0; SocketError errorCode; int observedSequenceNumber; - if (_sendQueue.IsReady(this, out observedSequenceNumber) && + if (PollHandle.IsWriteReady(out observedSequenceNumber) && (SocketPal.TryCompleteSendTo(_socket, buffer, ref offset, ref count, flags, socketAddress.Span, ref bytesSent, out errorCode) || !ShouldRetrySyncOperation(out errorCode))) { @@ -2002,7 +1189,7 @@ public SocketError SendTo(byte[] buffer, int offset, int count, SocketFlags flag BytesTransferred = bytesSent }; - PerformSyncOperation(ref _sendQueue, operation, timeout, observedSequenceNumber); + PerformSyncOperation(operation, isRead: false, timeout, observedSequenceNumber); bytesSent = operation.BytesTransferred; return operation.ErrorCode; @@ -2018,7 +1205,7 @@ public unsafe SocketError SendTo(ReadOnlySpan buffer, SocketFlags flags, M SocketError errorCode; int bufferIndexIgnored = 0, offset = 0, count = buffer.Length; int observedSequenceNumber; - if (_sendQueue.IsReady(this, out observedSequenceNumber) && + if (PollHandle.IsWriteReady(out observedSequenceNumber) && (SocketPal.TryCompleteSendTo(_socket, buffer, null, ref bufferIndexIgnored, ref offset, ref count, flags, socketAddress.Span, ref bytesSent, out errorCode) || !ShouldRetrySyncOperation(out errorCode))) { @@ -2037,7 +1224,7 @@ public unsafe SocketError SendTo(ReadOnlySpan buffer, SocketFlags flags, M BytesTransferred = bytesSent }; - PerformSyncOperation(ref _sendQueue, operation, timeout, observedSequenceNumber); + PerformSyncOperation(operation, isRead: false, timeout, observedSequenceNumber); bytesSent = operation.BytesTransferred; return operation.ErrorCode; @@ -2050,7 +1237,7 @@ public SocketError SendToAsync(Memory buffer, int offset, int count, Socke SocketError errorCode; int observedSequenceNumber; - if (_sendQueue.IsReady(this, out observedSequenceNumber) && + if (PollHandle.IsWriteReady(out observedSequenceNumber) && SocketPal.TryCompleteSendTo(_socket, buffer.Span, ref offset, ref count, flags, socketAddress.Span, ref bytesSent, out errorCode)) { return errorCode; @@ -2065,16 +1252,18 @@ public SocketError SendToAsync(Memory buffer, int offset, int count, Socke operation.SocketAddress = socketAddress; operation.BytesTransferred = bytesSent; - if (!_sendQueue.StartAsyncOperation(this, operation, observedSequenceNumber, cancellationToken)) + PollOperationAsyncResult result = PollHandle.WriteAsync(operation, observedSequenceNumber, cancellationToken); + + if (result == PollOperationAsyncResult.Completed) { - bytesSent = operation.BytesTransferred; errorCode = operation.ErrorCode; + bytesSent = operation.BytesTransferred; ReturnOperation(operation); return errorCode; } - return SocketError.IOPending; + return GetSocketErrorForNonCompleted(result); } public SocketError Send(IList> buffers, SocketFlags flags, int timeout, out int bytesSent) @@ -2098,7 +1287,7 @@ public SocketError SendTo(IList> buffers, SocketFlags flags, int offset = 0; SocketError errorCode; int observedSequenceNumber; - if (_sendQueue.IsReady(this, out observedSequenceNumber) && + if (PollHandle.IsWriteReady(out observedSequenceNumber) && (SocketPal.TryCompleteSendTo(_socket, buffers, ref bufferIndex, ref offset, flags, socketAddress.Span, ref bytesSent, out errorCode) || !ShouldRetrySyncOperation(out errorCode))) { @@ -2115,7 +1304,7 @@ public SocketError SendTo(IList> buffers, SocketFlags flags, BytesTransferred = bytesSent }; - PerformSyncOperation(ref _sendQueue, operation, timeout, observedSequenceNumber); + PerformSyncOperation(operation, isRead: false, timeout, observedSequenceNumber); bytesSent = operation.BytesTransferred; return operation.ErrorCode; @@ -2130,7 +1319,7 @@ public SocketError SendToAsync(IList> buffers, SocketFlags fl int offset = 0; SocketError errorCode; int observedSequenceNumber; - if (_sendQueue.IsReady(this, out observedSequenceNumber) && + if (PollHandle.IsWriteReady(out observedSequenceNumber) && SocketPal.TryCompleteSendTo(_socket, buffers, ref bufferIndex, ref offset, flags, socketAddress.Span, ref bytesSent, out errorCode)) { return errorCode; @@ -2145,16 +1334,18 @@ public SocketError SendToAsync(IList> buffers, SocketFlags fl operation.SocketAddress = socketAddress; operation.BytesTransferred = bytesSent; - if (!_sendQueue.StartAsyncOperation(this, operation, observedSequenceNumber)) + PollOperationAsyncResult result = PollHandle.WriteAsync(operation, observedSequenceNumber, default); + + if (result == PollOperationAsyncResult.Completed) { - bytesSent = operation.BytesTransferred; errorCode = operation.ErrorCode; + bytesSent = operation.BytesTransferred; ReturnOperation(operation); return errorCode; } - return SocketError.IOPending; + return GetSocketErrorForNonCompleted(result); } public SocketError SendFile(SafeFileHandle fileHandle, long offset, long count, int timeout, out long bytesSent) @@ -2166,7 +1357,7 @@ public SocketError SendFile(SafeFileHandle fileHandle, long offset, long count, bytesSent = 0; SocketError errorCode; int observedSequenceNumber; - if (_sendQueue.IsReady(this, out observedSequenceNumber) && + if (PollHandle.IsWriteReady(out observedSequenceNumber) && (SocketPal.TryCompleteSendFile(_socket, fileHandle, ref offset, ref count, ref bytesSent, out errorCode) || !ShouldRetrySyncOperation(out errorCode))) { @@ -2181,7 +1372,7 @@ public SocketError SendFile(SafeFileHandle fileHandle, long offset, long count, BytesTransferred = bytesSent }; - PerformSyncOperation(ref _sendQueue, operation, timeout, observedSequenceNumber); + PerformSyncOperation(operation, isRead: false, timeout, observedSequenceNumber); bytesSent = operation.BytesTransferred; return operation.ErrorCode; @@ -2194,7 +1385,7 @@ public SocketError SendFileAsync(SafeFileHandle fileHandle, long offset, long co bytesSent = 0; SocketError errorCode; int observedSequenceNumber; - if (_sendQueue.IsReady(this, out observedSequenceNumber) && + if (PollHandle.IsWriteReady(out observedSequenceNumber) && SocketPal.TryCompleteSendFile(_socket, fileHandle, ref offset, ref count, ref bytesSent, out errorCode)) { return errorCode; @@ -2209,123 +1400,27 @@ public SocketError SendFileAsync(SafeFileHandle fileHandle, long offset, long co BytesTransferred = bytesSent }; - if (!_sendQueue.StartAsyncOperation(this, operation, observedSequenceNumber, cancellationToken)) - { - bytesSent = operation.BytesTransferred; - return operation.ErrorCode; - } - - return SocketError.IOPending; - } - - // Called on the epoll thread, speculatively tries to process synchronous events and errors for synchronous events, and - // returns any remaining events that remain to be processed. Taking a lock for each operation queue to deterministically - // handle synchronous events on the epoll thread seems to significantly reduce throughput in benchmarks. On the other - // hand, the speculative checks make it nondeterministic, where it would be possible for the epoll thread to think that - // the next operation in a queue is not synchronous when it is (due to a race, old caches, etc.) and cause the event to - // be scheduled instead. It's not functionally incorrect to schedule the release of a synchronous operation, just it may - // lead to thread pool starvation issues if the synchronous operations are blocking thread pool threads (typically not - // advised) and more threads are not immediately available to run work items that would release those operations. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Interop.Sys.SocketEvents HandleSyncEventsSpeculatively(Interop.Sys.SocketEvents events) - { - if ((events & Interop.Sys.SocketEvents.Error) != 0) - { - // Set the Read and Write flags; the processing for these events - // will pick up the error. - events ^= Interop.Sys.SocketEvents.Error; - events |= Interop.Sys.SocketEvents.Read | Interop.Sys.SocketEvents.Write; - } - - if ((events & Interop.Sys.SocketEvents.Read) != 0 && - _receiveQueue.IsNextOperationSynchronous_Speculative && - _receiveQueue.ProcessSyncEventOrGetAsyncEvent(this, skipAsyncEvents: true) == null) - { - events ^= Interop.Sys.SocketEvents.Read; - } - - if ((events & Interop.Sys.SocketEvents.Write) != 0 && - _sendQueue.IsNextOperationSynchronous_Speculative && - _sendQueue.ProcessSyncEventOrGetAsyncEvent(this, skipAsyncEvents: true) == null) - { - events ^= Interop.Sys.SocketEvents.Write; - } - - return events; - } - - // Called on the epoll thread. - public void HandleEventsInline(Interop.Sys.SocketEvents events) - { - if ((events & Interop.Sys.SocketEvents.Error) != 0) - { - // Set the Read and Write flags; the processing for these events - // will pick up the error. - events ^= Interop.Sys.SocketEvents.Error; - events |= Interop.Sys.SocketEvents.Read | Interop.Sys.SocketEvents.Write; - } + PollOperationAsyncResult result = PollHandle.WriteAsync(operation, observedSequenceNumber, cancellationToken); - if ((events & Interop.Sys.SocketEvents.Read) != 0) + if (result == PollOperationAsyncResult.Completed) { - AsyncOperation? receiveOperation = _receiveQueue.ProcessSyncEventOrGetAsyncEvent(this); - receiveOperation?.Process(); + errorCode = operation.ErrorCode; + bytesSent = operation.BytesTransferred; + return errorCode; } - if ((events & Interop.Sys.SocketEvents.Write) != 0) - { - AsyncOperation? sendOperation = _sendQueue.ProcessSyncEventOrGetAsyncEvent(this); - sendOperation?.Process(); - } + return GetSocketErrorForNonCompleted(result); } - // Called on ThreadPool thread. - public void HandleEvents(Interop.Sys.SocketEvents events) + private static SocketError GetSocketErrorForNonCompleted(PollOperationAsyncResult result) { - Debug.Assert((events & Interop.Sys.SocketEvents.Error) == 0); - - AsyncOperation? receiveOperation = - (events & Interop.Sys.SocketEvents.Read) != 0 ? _receiveQueue.ProcessSyncEventOrGetAsyncEvent(this) : null; - AsyncOperation? sendOperation = - (events & Interop.Sys.SocketEvents.Write) != 0 ? _sendQueue.ProcessSyncEventOrGetAsyncEvent(this) : null; - - // This method is called from a thread pool thread. When we have only one operation to process, process it - // synchronously to avoid an extra thread pool work item. When we have two operations to process, processing both - // synchronously may delay the second operation, so schedule one onto the thread pool and process the other - // synchronously. There might be better ways of doing this. - if (sendOperation == null) - { - receiveOperation?.Process(); - } - else - { - receiveOperation?.Schedule(); - sendOperation.Process(); - } + Debug.Assert(result is PollOperationAsyncResult.Pending or PollOperationAsyncResult.Aborted); + return result == PollOperationAsyncResult.Pending ? SocketError.IOPending : SocketError.OperationAborted; } // // Tracing stuff // - // To enabled tracing: - // (1) Add reference to System.Console in the csproj - // (2) #define SOCKETASYNCCONTEXT_TRACE - - [Conditional("SOCKETASYNCCONTEXT_TRACE")] - public void Trace(string message, [CallerMemberName] string? memberName = null) - { - OutputTrace($"{IdOf(this)}.{memberName}: {message}"); - } - - [Conditional("SOCKETASYNCCONTEXT_TRACE")] - public static void OutputTrace(string s) - { - // CONSIDER: Change to NetEventSource -#if SOCKETASYNCCONTEXT_TRACE - Console.WriteLine(s); -#endif - } - - public static string IdOf(object o) => o == null ? "(null)" : $"{o.GetType().Name}#{o.GetHashCode():X2}"; } } diff --git a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketAsyncContext.Wasi.cs b/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketAsyncContext.Wasi.cs deleted file mode 100644 index fbe89b6fa64c00..00000000000000 --- a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketAsyncContext.Wasi.cs +++ /dev/null @@ -1,19 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Threading; -using Microsoft.Win32.SafeHandles; -using System.Runtime.Versioning; - -namespace System.Net.Sockets -{ - internal sealed partial class SocketAsyncContext - { - public CancellationTokenSource unregisterPollHook = new(); - } -} diff --git a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketAsyncEngine.Unix.cs b/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketAsyncEngine.Unix.cs deleted file mode 100644 index ae9b6c9095e43f..00000000000000 --- a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketAsyncEngine.Unix.cs +++ /dev/null @@ -1,373 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Threading; - -namespace System.Net.Sockets -{ - internal sealed unsafe class SocketAsyncEngine : IThreadPoolWorkItem - { - private const int EventBufferCount = -#if DEBUG - 32; -#else - 1024; -#endif - - // Socket continuations are dispatched to the ThreadPool from the event thread. - // This avoids continuations blocking the event handling. - // Setting PreferInlineCompletions allows continuations to run directly on the event thread. - // PreferInlineCompletions defaults to false and can be set to true using the DOTNET_SYSTEM_NET_SOCKETS_INLINE_COMPLETIONS envvar. - internal static readonly bool InlineSocketCompletionsEnabled = Environment.GetEnvironmentVariable("DOTNET_SYSTEM_NET_SOCKETS_INLINE_COMPLETIONS") == "1"; - - private static int GetEngineCount() - { - // The responsibility of SocketAsyncEngine is to get notifications from epoll|kqueue - // and schedule corresponding work items to ThreadPool (socket reads and writes). - // - // Using TechEmpower benchmarks that generate a LOT of SMALL socket reads and writes under a VERY HIGH load - // we have observed that a single engine is capable of keeping busy up to thirty x64 and eight ARM64 CPU Cores. - // - // The vast majority of real-life scenarios is never going to generate such a huge load (hundreds of thousands of requests per second) - // and having a single producer should be almost always enough. - // - // We want to be sure that we can handle extreme loads and that's why we have decided to use these values. - // - // It's impossible to predict all possible scenarios so we have added a possibility to configure this value using environment variables. - if (uint.TryParse(Environment.GetEnvironmentVariable("DOTNET_SYSTEM_NET_SOCKETS_THREAD_COUNT"), out uint count)) - { - return (int)count; - } - - // When inlining continuations, we default to ProcessorCount to make sure event threads cannot be a bottleneck. - if (InlineSocketCompletionsEnabled) - { - return Environment.ProcessorCount; - } - - Architecture architecture = RuntimeInformation.ProcessArchitecture; - int coresPerEngine = architecture == Architecture.Arm64 || architecture == Architecture.Arm - ? 8 - : 30; - - return Math.Max(1, (int)Math.Round(Environment.ProcessorCount / (double)coresPerEngine)); - } - - private static readonly SocketAsyncEngine[] s_engines = CreateEngines(); - private static int s_allocateFromEngine = -1; - - private static SocketAsyncEngine[] CreateEngines() - { - int engineCount = GetEngineCount(); - - var engines = new SocketAsyncEngine[engineCount]; - - for (int i = 0; i < engineCount; i++) - { - engines[i] = new SocketAsyncEngine(); - } - - return engines; - } - - /// - /// Each is assigned an index into this table while registered with a . - /// The index is used as the to quickly map events to s. - /// It is also stored in so that we can efficiently remove it when unregistering the socket. - /// - private static SocketAsyncContext?[] s_registeredContexts = []; - private static readonly Queue s_registeredContextsFreeList = []; - - private readonly IntPtr _port; - private readonly Interop.Sys.SocketEvent* _buffer; - - // - // Queue of events generated by EventLoop() that would be processed by the thread pool - // - private readonly ConcurrentQueue _eventQueue = new ConcurrentQueue(); - - // This flag is used for communication between item enqueuing and workers that process the items. - // There are two states of this flag: - // 0: has no guarantees - // 1: means a worker will check work queues and ensure that - // any work items inserted in work queue before setting the flag - // are picked up. - // Note: The state must be cleared by the worker thread _before_ - // checking. Otherwise there is a window between finding no work - // and resetting the flag, when the flag is in a wrong state. - // A new work item may be added right before the flag is reset - // without asking for a worker, while the last worker is quitting. - private int _hasOutstandingThreadRequest; - - // - // Registers the Socket with a SocketAsyncEngine, and returns the associated engine. - // - public static bool TryRegisterSocket(IntPtr socketHandle, SocketAsyncContext context, out SocketAsyncEngine? engine, out Interop.Error error) - { - int engineIndex = Math.Abs(Interlocked.Increment(ref s_allocateFromEngine) % s_engines.Length); - SocketAsyncEngine nextEngine = s_engines[engineIndex]; - bool registered = nextEngine.TryRegisterCore(socketHandle, context, out error); - engine = registered ? nextEngine : null; - return registered; - } - - private bool TryRegisterCore(IntPtr socketHandle, SocketAsyncContext context, out Interop.Error error) - { - Debug.Assert(context.GlobalContextIndex == -1); - - lock (s_registeredContextsFreeList) - { - if (!s_registeredContextsFreeList.TryDequeue(out int index)) - { - int previousLength = s_registeredContexts.Length; - int newLength = Math.Max(4, 2 * previousLength); - - Array.Resize(ref s_registeredContexts, newLength); - - for (int i = previousLength + 1; i < newLength; i++) - { - s_registeredContextsFreeList.Enqueue(i); - } - - index = previousLength; - } - - Debug.Assert(s_registeredContexts[index] is null); - - s_registeredContexts[index] = context; - context.GlobalContextIndex = index; - } - - error = Interop.Sys.TryChangeSocketEventRegistration(_port, socketHandle, Interop.Sys.SocketEvents.None, - Interop.Sys.SocketEvents.Read | Interop.Sys.SocketEvents.Write, context.GlobalContextIndex); - if (error == Interop.Error.SUCCESS) - { - return true; - } - - UnregisterSocket(context); - return false; - } - - public static void UnregisterSocket(SocketAsyncContext context) - { - Debug.Assert(context.GlobalContextIndex >= 0); - Debug.Assert(ReferenceEquals(s_registeredContexts[context.GlobalContextIndex], context)); - - lock (s_registeredContextsFreeList) - { - s_registeredContexts[context.GlobalContextIndex] = null; - s_registeredContextsFreeList.Enqueue(context.GlobalContextIndex); - } - - context.GlobalContextIndex = -1; - } - - private SocketAsyncEngine() - { - _port = (IntPtr)(-1); - try - { - // - // Create the event port and buffer - // - Interop.Error err; - fixed (IntPtr* portPtr = &_port) - { - err = Interop.Sys.CreateSocketEventPort(portPtr); - if (err != Interop.Error.SUCCESS) - { - throw new InternalException(err); - } - } - - fixed (Interop.Sys.SocketEvent** bufferPtr = &_buffer) - { - err = Interop.Sys.CreateSocketEventBuffer(EventBufferCount, bufferPtr); - if (err != Interop.Error.SUCCESS) - { - throw new InternalException(err); - } - } - - var thread = new Thread(static s => ((SocketAsyncEngine)s!).EventLoop()) - { - IsBackground = true, - Name = ".NET Sockets" - }; - thread.UnsafeStart(this); - } - catch - { - FreeNativeResources(); - throw; - } - } - - private void EventLoop() - { - try - { - SocketEventHandler handler = new SocketEventHandler(this); - while (true) - { - int numEvents = EventBufferCount; - Interop.Error err = Interop.Sys.WaitForSocketEvents(_port, handler.Buffer, &numEvents); - if (err != Interop.Error.SUCCESS) - { - throw new InternalException(err); - } - - // The native shim is responsible for ensuring this condition. - Debug.Assert(numEvents > 0, $"Unexpected numEvents: {numEvents}"); - - if (handler.HandleSocketEvents(numEvents)) - { - EnsureWorkerScheduled(); - } - } - } - catch (Exception e) - { - Environment.FailFast("Exception thrown from SocketAsyncEngine event loop: " + e.ToString(), e); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void EnsureWorkerScheduled() - { - // Only one worker is requested at a time to mitigate Thundering Herd problem. - if (Interlocked.Exchange(ref _hasOutstandingThreadRequest, 1) == 0) - { - ThreadPool.UnsafeQueueUserWorkItem(this, preferLocal: false); - } - } - - void IThreadPoolWorkItem.Execute() - { - // We are asking for one worker at a time, thus the state should be 1. - Debug.Assert(_hasOutstandingThreadRequest == 1); - _hasOutstandingThreadRequest = 0; - - // Checking for items must happen after resetting the processing state. - Interlocked.MemoryBarrier(); - - ConcurrentQueue eventQueue = _eventQueue; - if (!eventQueue.TryDequeue(out SocketIOEvent ev)) - { - return; - } - - // The batch that is currently in the queue could have asked only for one worker. - // We are going to process a workitem, which may take unknown time or even block. - // In a worst case the current workitem will indirectly depend on progress of other - // items and that would lead to a deadlock if no one else checks the queue. - // We must ensure at least one more worker is coming if the queue is not empty. - if (!eventQueue.IsEmpty) - { - EnsureWorkerScheduled(); - } - - int startTimeMs = Environment.TickCount; - do - { - ev.Context.HandleEvents(ev.Events); - - // If there is a constant stream of new events, and/or if user callbacks take long to process an event, this - // work item may run for a long time. If work items of this type are using up all of the thread pool threads, - // collectively they may starve other types of work items from running. Before dequeuing and processing another - // event, check the elapsed time since the start of the work item and yield the thread after some time has - // elapsed to allow the thread pool to run other work items. - // - // The threshold chosen below was based on trying various thresholds and in trying to keep the latency of - // running another work item low when these work items are using up all of the thread pool worker threads. In - // such cases, the latency would be something like threshold / proc count. Smaller thresholds were tried and - // using Stopwatch instead (like 1 ms, 5 ms, etc.), from quick tests they appeared to have a slightly greater - // impact on throughput compared to the threshold chosen below, though it is slight enough that it may not - // matter much. Higher thresholds didn't seem to have any noticeable effect. - } while (Environment.TickCount - startTimeMs < 15 && eventQueue.TryDequeue(out ev)); - } - - private void FreeNativeResources() - { - if (_buffer != null) - { - Interop.Sys.FreeSocketEventBuffer(_buffer); - } - if (_port != (IntPtr)(-1)) - { - Interop.Sys.CloseSocketEventPort(_port); - } - } - - // The JIT is allowed to arbitrarily extend the lifetime of locals, which may retain SocketAsyncContext references, - // indirectly preventing Socket instances to be finalized, despite being no longer referenced by user code. - // To avoid this, the event handling logic is delegated to a non-inlined processing method. - // See discussion: https://github.com/dotnet/runtime/issues/37064 - // SocketEventHandler holds an on-stack cache of SocketAsyncEngine members needed by the handler method. - private readonly struct SocketEventHandler - { - public Interop.Sys.SocketEvent* Buffer { get; } - - private readonly ConcurrentQueue _eventQueue; - - public SocketEventHandler(SocketAsyncEngine engine) - { - Buffer = engine._buffer; - _eventQueue = engine._eventQueue; - } - - [MethodImpl(MethodImplOptions.NoInlining)] - public bool HandleSocketEvents(int numEvents) - { - bool enqueuedEvent = false; - foreach (var socketEvent in new ReadOnlySpan(Buffer, numEvents)) - { - Debug.Assert((uint)socketEvent.Data < (uint)s_registeredContexts.Length); - - // The context may be null if the socket was unregistered right before the event was processed. - // The slot in s_registeredContexts may have been reused by a different context, in which case the - // incorrect socket will notice that no information is available yet and harmlessly retry, waiting for new events. - SocketAsyncContext? context = s_registeredContexts[(uint)socketEvent.Data]; - - if (context is not null) - { - if (context.PreferInlineCompletions) - { - context.HandleEventsInline(socketEvent.Events); - } - else - { - Interop.Sys.SocketEvents events = context.HandleSyncEventsSpeculatively(socketEvent.Events); - - if (events != Interop.Sys.SocketEvents.None) - { - _eventQueue.Enqueue(new SocketIOEvent(context, events)); - enqueuedEvent = true; - } - } - } - } - - return enqueuedEvent; - } - } - - private readonly struct SocketIOEvent - { - public SocketAsyncContext Context { get; } - public Interop.Sys.SocketEvents Events { get; } - - public SocketIOEvent(SocketAsyncContext context, Interop.Sys.SocketEvents events) - { - Context = context; - Events = events; - } - } - } -} diff --git a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketAsyncEngine.Wasi.cs b/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketAsyncEngine.Wasi.cs deleted file mode 100644 index 0a39feb2699364..00000000000000 --- a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketAsyncEngine.Wasi.cs +++ /dev/null @@ -1,154 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.Net.Sockets; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Threading; -using System.Threading.Tasks; -using static Interop; -using static Interop.Sys; - -namespace System.Net.Sockets -{ - internal sealed unsafe class SocketAsyncEngine - { - internal const bool InlineSocketCompletionsEnabled = true; - private static readonly SocketAsyncEngine s_engine = new SocketAsyncEngine(); - - public static bool TryRegisterSocket(IntPtr socketHandle, SocketAsyncContext context, out SocketAsyncEngine? engine, out Interop.Error error) - { - engine = s_engine; - - nint entryPtr = default; - error = Interop.Sys.GetWasiSocketDescriptor(socketHandle, &entryPtr); - if (error != Interop.Error.SUCCESS) - { - return false; - } - - RegisterWasiPollHook(context, BeforePollHook, HandleSocketEvent, context.unregisterPollHook.Token); - - return true; - } - - public static void UnregisterSocket(SocketAsyncContext context) - { - context.unregisterPollHook.Cancel(); - } - - // this method is invading private implementation details of wasi-libc - // we could get rid of it when https://github.com/WebAssembly/wasi-libc/issues/542 is resolved - // or after WASIp3 promises are implemented, whatever comes first - public static IList BeforePollHook(object? state) - { - var context = (SocketAsyncContext)state!; - if (context._socket.IsClosed) - { - return []; - } - - List pollableHandles = new(); - // fail fast if the handle is not found in the descriptor table - // probably because the socket was closed and the entry was removed, without unregistering the poll hook - nint entryPtr = default; - IntPtr socketHandle = context._socket.DangerousGetHandle(); - var error = Interop.Sys.GetWasiSocketDescriptor(socketHandle, &entryPtr); - if (error != Interop.Error.SUCCESS) - { - Environment.FailFast("Can't resolve libc descriptor for socket handle " + socketHandle); - } - - var entry = (descriptor_table_entry_t*)entryPtr; - switch (entry->tag) - { - case descriptor_table_entry_tag.DESCRIPTOR_TABLE_ENTRY_TCP_SOCKET: - { - tcp_socket_t* socket = &(entry->entry.tcp_socket); - switch (socket->state.tag) - { - case tcp_socket_state_tag.TCP_SOCKET_STATE_CONNECTING: - case tcp_socket_state_tag.TCP_SOCKET_STATE_LISTENING: - pollableHandles.Add(socket->socket_pollable.handle); - break; - case tcp_socket_state_tag.TCP_SOCKET_STATE_CONNECTED: - pollableHandles.Add(socket->state.state.connected.input_pollable.handle); - pollableHandles.Add(socket->state.state.connected.output_pollable.handle); - break; - case tcp_socket_state_tag.TCP_SOCKET_STATE_CONNECT_FAILED: - context.HandleEventsInline(Sys.SocketEvents.Error); - break; - case tcp_socket_state_tag.TCP_SOCKET_STATE_UNBOUND: - case tcp_socket_state_tag.TCP_SOCKET_STATE_BOUND: - break; - default: - throw new NotImplementedException("TCP:" + socket->state.tag); - } - break; - } - case descriptor_table_entry_tag.DESCRIPTOR_TABLE_ENTRY_UDP_SOCKET: - { - udp_socket_t* socket = &(entry->entry.udp_socket); - switch (socket->state.tag) - { - case udp_socket_state_tag.UDP_SOCKET_STATE_UNBOUND: - case udp_socket_state_tag.UDP_SOCKET_STATE_BOUND_NOSTREAMS: - // TODO ? pollableHandles.Add(socket->socket_pollable.handle); - context.HandleEventsInline(Sys.SocketEvents.Read | Sys.SocketEvents.Write); - break; - case udp_socket_state_tag.UDP_SOCKET_STATE_BOUND_STREAMING: - case udp_socket_state_tag.UDP_SOCKET_STATE_CONNECTED: - { - udp_socket_streams_t* streams; - if (socket->state.tag == udp_socket_state_tag.UDP_SOCKET_STATE_BOUND_STREAMING) - { - streams = &(socket->state.state.bound_streaming.streams); - } - else - { - streams = &(socket->state.state.connected.streams); - } - pollableHandles.Add(streams->incoming_pollable.handle); - pollableHandles.Add(streams->outgoing_pollable.handle); - break; - } - - default: - throw new NotImplementedException("UDP" + socket->state.tag); - } - break; - } - default: - throw new NotImplementedException("TYPE" + entry->tag); - } - return pollableHandles; - } - - public static void HandleSocketEvent(object? state) - { - SocketAsyncContext ctx = (SocketAsyncContext)state!; - try - { - using (ExecutionContext.SuppressFlow()) - { - ctx.HandleEventsInline(Sys.SocketEvents.Write | Sys.SocketEvents.Read); - } - } - catch (Exception e) - { - Environment.FailFast("Exception thrown from SocketAsyncEngine event loop: " + e.ToString(), e); - } - } - - private static void RegisterWasiPollHook(object? state, Func> beforePollHook, Action onResolveCallback, CancellationToken cancellationToken) - { - CallRegisterWasiPollHook((Thread)null!, state, beforePollHook, onResolveCallback, cancellationToken); - - [UnsafeAccessor(UnsafeAccessorKind.StaticMethod, Name = "RegisterWasiPollHook")] - static extern void CallRegisterWasiPollHook(Thread t, object? state, Func> beforePollHook, Action onResolveCallback, CancellationToken cancellationToken); - } - } -} diff --git a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketPal.Wasi.cs b/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketPal.Wasi.cs deleted file mode 100644 index 3fd6dee783e01c..00000000000000 --- a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketPal.Wasi.cs +++ /dev/null @@ -1,227 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Collections; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Win32.SafeHandles; - -// types here are clone of private implementation details of wasi-libc -// we could get rid of it when https://github.com/WebAssembly/wasi-libc/issues/542 is resolved -// or after WASIp3 promises are implemented, whatever comes first - -namespace System.Net.Sockets -{ - [StructLayout(LayoutKind.Sequential)] - internal struct tcp_own_tcp_socket_t - { - public int handle; - } - - [StructLayout(LayoutKind.Sequential)] - internal struct udp_own_udp_socket_t - { - public int handle; - } - - [StructLayout(LayoutKind.Sequential)] - internal struct udp_own_incoming_datagram_stream_t - { - public int handle; - } - - [StructLayout(LayoutKind.Sequential)] - internal struct udp_own_outgoing_datagram_stream_t - { - public int handle; - } - - [StructLayout(LayoutKind.Sequential)] - internal struct streams_own_input_stream_t - { - public int handle; - } - - [StructLayout(LayoutKind.Sequential)] - internal struct poll_own_pollable_t - { - public int handle; - } - - [StructLayout(LayoutKind.Sequential)] - internal struct streams_own_output_stream_t - { - public int handle; - } - - [StructLayout(LayoutKind.Sequential)] - internal struct tcp_socket_state_unbound_t - { - public int dummy; - } - - [StructLayout(LayoutKind.Sequential)] - internal struct tcp_socket_state_bound_t - { - public int dummy; - } - - [StructLayout(LayoutKind.Sequential)] - internal struct tcp_socket_state_connecting_t - { - public int dummy; - } - - [StructLayout(LayoutKind.Sequential)] - internal struct tcp_socket_state_listening_t - { - public int dummy; - } - - [StructLayout(LayoutKind.Sequential)] - internal struct tcp_socket_state_connected_t - { - public streams_own_input_stream_t input; - public poll_own_pollable_t input_pollable; - public streams_own_output_stream_t output; - public poll_own_pollable_t output_pollable; - } - - [StructLayout(LayoutKind.Sequential)] - internal struct tcp_socket_state_connect_failed_t - { - public byte error_code; - } - - internal enum tcp_socket_state_tag - { - TCP_SOCKET_STATE_UNBOUND, - TCP_SOCKET_STATE_BOUND, - TCP_SOCKET_STATE_CONNECTING, - TCP_SOCKET_STATE_CONNECTED, - TCP_SOCKET_STATE_CONNECT_FAILED, - TCP_SOCKET_STATE_LISTENING, - } - - [StructLayout(LayoutKind.Explicit)] - internal struct tcp_socket_state_union - { - [FieldOffset(0)] public tcp_socket_state_unbound_t unbound; - [FieldOffset(0)] public tcp_socket_state_bound_t bound; - [FieldOffset(0)] public tcp_socket_state_connecting_t connecting; - [FieldOffset(0)] public tcp_socket_state_connected_t connected; - [FieldOffset(0)] public tcp_socket_state_connect_failed_t connect_failed; - [FieldOffset(0)] public tcp_socket_state_listening_t listening; - } - - [StructLayout(LayoutKind.Sequential)] - internal struct tcp_socket_state_t - { - public tcp_socket_state_tag tag; - public tcp_socket_state_union state; - } - - [StructLayout(LayoutKind.Sequential)] - internal struct tcp_socket_t - { - public tcp_own_tcp_socket_t socket; - public poll_own_pollable_t socket_pollable; - public bool blocking; - public bool fake_nodelay; - public bool fake_reuseaddr; - public byte family; - public tcp_socket_state_t state; - } - - [StructLayout(LayoutKind.Sequential)] - internal struct udp_socket_streams_t - { - public udp_own_incoming_datagram_stream_t incoming; - public poll_own_pollable_t incoming_pollable; - public udp_own_outgoing_datagram_stream_t outgoing; - public poll_own_pollable_t outgoing_pollable; - } - - [StructLayout(LayoutKind.Sequential)] - internal struct udp_socket_state_unbound_t - { - public int dummy; - } - - [StructLayout(LayoutKind.Sequential)] - internal struct udp_socket_state_bound_nostreams_t - { - public int dummy; - } - - [StructLayout(LayoutKind.Sequential)] - internal struct udp_socket_state_bound_streaming_t - { - public udp_socket_streams_t streams; - } - - [StructLayout(LayoutKind.Sequential)] - internal struct udp_socket_state_connected_t - { - public udp_socket_streams_t streams; - } - - internal enum udp_socket_state_tag - { - UDP_SOCKET_STATE_UNBOUND, - UDP_SOCKET_STATE_BOUND_NOSTREAMS, - UDP_SOCKET_STATE_BOUND_STREAMING, - UDP_SOCKET_STATE_CONNECTED, - } - - [StructLayout(LayoutKind.Explicit)] - internal struct udp_socket_state_union - { - [FieldOffset(0)] public udp_socket_state_unbound_t unbound; - [FieldOffset(0)] public udp_socket_state_bound_nostreams_t bound_nostreams; - [FieldOffset(0)] public udp_socket_state_bound_streaming_t bound_streaming; - [FieldOffset(0)] public udp_socket_state_connected_t connected; - } - - [StructLayout(LayoutKind.Sequential)] - internal struct udp_socket_state_t - { - public udp_socket_state_tag tag; - public udp_socket_state_union state; - } - - [StructLayout(LayoutKind.Sequential)] - internal struct udp_socket_t - { - public udp_own_udp_socket_t socket; - public poll_own_pollable_t socket_pollable; - public bool blocking; - public byte family; - public udp_socket_state_t state; - } - - internal enum descriptor_table_entry_tag - { - DESCRIPTOR_TABLE_ENTRY_TCP_SOCKET, - DESCRIPTOR_TABLE_ENTRY_UDP_SOCKET, - } - - [StructLayout(LayoutKind.Explicit)] - internal struct descriptor_table_entry_union - { - [FieldOffset(0)] public tcp_socket_t tcp_socket; - [FieldOffset(0)] public udp_socket_t udp_socket; - } - - [StructLayout(LayoutKind.Sequential)] - internal struct descriptor_table_entry_t - { - public descriptor_table_entry_tag tag; - public descriptor_table_entry_union entry; - } -} diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.ThreadPoolValueTaskSource.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.ThreadPoolValueTaskSource.cs deleted file mode 100644 index 9fff83001662bd..00000000000000 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.ThreadPoolValueTaskSource.cs +++ /dev/null @@ -1,235 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.IO.Strategies; -using System.Runtime.InteropServices; -using System.Threading; -using System.Threading.Tasks; -using System.Threading.Tasks.Sources; - -namespace Microsoft.Win32.SafeHandles -{ - public sealed partial class SafeFileHandle : SafeHandleZeroOrMinusOneIsInvalid - { - private ThreadPoolValueTaskSource? _reusableThreadPoolValueTaskSource; // reusable ThreadPoolValueTaskSource that is currently NOT being used - - // Rent the reusable ThreadPoolValueTaskSource, or create a new one to use if we couldn't get one (which - // should only happen on first use or if the SafeFileHandle is being used concurrently). - internal ThreadPoolValueTaskSource GetThreadPoolValueTaskSource() => - Interlocked.Exchange(ref _reusableThreadPoolValueTaskSource, null) ?? new ThreadPoolValueTaskSource(this); - - /// - /// A reusable implementation that - /// queues asynchronous operations to - /// be completed synchronously on the thread pool. - /// - internal sealed class ThreadPoolValueTaskSource : IThreadPoolWorkItem, IValueTaskSource, IValueTaskSource, IValueTaskSource - { - private readonly SafeFileHandle _fileHandle; - private ManualResetValueTaskSourceCore _source; - private Operation _operation = Operation.None; - private ExecutionContext? _context; - private OSFileStreamStrategy? _strategy; - - // These fields store the parameters for the operation. - // The first two are common for all kinds of operations. - private long _fileOffset; - private CancellationToken _cancellationToken; - // Used by simple reads and writes. Will be unsafely cast to a memory when performing a read. - private ReadOnlyMemory _singleSegment; - private IReadOnlyList>? _readScatterBuffers; - private IReadOnlyList>? _writeGatherBuffers; - - internal ThreadPoolValueTaskSource(SafeFileHandle fileHandle) - { - _fileHandle = fileHandle; - } - - [Conditional("DEBUG")] - private void ValidateInvariants() - { - Operation op = _operation; - Debug.Assert(op == Operation.None, $"An operation was queued before the previous {op}'s completion."); - } - - public ValueTaskSourceStatus GetStatus(short token) => - _source.GetStatus(token); - - public void OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) => - _source.OnCompleted(continuation, state, token, flags); - - void IValueTaskSource.GetResult(short token) => GetResult(token); - int IValueTaskSource.GetResult(short token) => (int)GetResult(token); - public long GetResult(short token) - { - try - { - return _source.GetResult(token); - } - finally - { - _source.Reset(); - Volatile.Write(ref _fileHandle._reusableThreadPoolValueTaskSource, this); - } - } - - private void ExecuteInternal() - { - Debug.Assert(_operation >= Operation.Read && _operation <= Operation.WriteGather); - - long result = 0; - Exception? exception = null; - try - { - // This is the operation's last chance to be canceled. - if (_cancellationToken.IsCancellationRequested) - { - exception = new OperationCanceledException(_cancellationToken); - } - else - { - switch (_operation) - { - case Operation.Read: - Memory writableSingleSegment = MemoryMarshal.AsMemory(_singleSegment); - result = RandomAccess.ReadAtOffset(_fileHandle, writableSingleSegment.Span, _fileOffset); - break; - case Operation.Write: - RandomAccess.WriteAtOffset(_fileHandle, _singleSegment.Span, _fileOffset); - break; - case Operation.ReadScatter: - Debug.Assert(_readScatterBuffers != null); - result = RandomAccess.ReadScatterAtOffset(_fileHandle, _readScatterBuffers, _fileOffset); - break; - case Operation.WriteGather: - Debug.Assert(_writeGatherBuffers != null); - RandomAccess.WriteGatherAtOffset(_fileHandle, _writeGatherBuffers, _fileOffset); - break; - } - } - } - catch (Exception e) - { - exception = e; - } - finally - { - if (_strategy is not null) - { - // WriteAtOffset returns void, so we need to fix position only in case of an exception - if (exception is not null) - { - _strategy.OnIncompleteOperation(_singleSegment.Length, 0); - } - else if (_operation == Operation.Read && result != _singleSegment.Length) - { - _strategy.OnIncompleteOperation(_singleSegment.Length, (int)result); - } - } - - _operation = Operation.None; - _context = null; - _strategy = null; - _cancellationToken = default; - _singleSegment = default; - _readScatterBuffers = null; - _writeGatherBuffers = null; - } - - if (exception == null) - { - _source.SetResult(result); - } - else - { - _source.SetException(exception); - } - } - - void IThreadPoolWorkItem.Execute() - { - if (_context == null || _context.IsDefault) - { - ExecuteInternal(); - } - else - { - ExecutionContext.RunForThreadPoolUnsafe(_context, static x => x.ExecuteInternal(), this); - } - } - - private void QueueToThreadPool() - { - _context = ExecutionContext.Capture(); - ThreadPool.UnsafeQueueUserWorkItem(this, preferLocal: true); - } - - public ValueTask QueueRead(Memory buffer, long fileOffset, CancellationToken cancellationToken, OSFileStreamStrategy? strategy) - { - ValidateInvariants(); - - _operation = Operation.Read; - _singleSegment = buffer; - _fileOffset = fileOffset; - _cancellationToken = cancellationToken; - _strategy = strategy; - QueueToThreadPool(); - - return new ValueTask(this, _source.Version); - } - - public ValueTask QueueWrite(ReadOnlyMemory buffer, long fileOffset, CancellationToken cancellationToken, OSFileStreamStrategy? strategy) - { - ValidateInvariants(); - - _operation = Operation.Write; - _singleSegment = buffer; - _fileOffset = fileOffset; - _cancellationToken = cancellationToken; - _strategy = strategy; - QueueToThreadPool(); - - return new ValueTask(this, _source.Version); - } - - public ValueTask QueueReadScatter(IReadOnlyList> buffers, long fileOffset, CancellationToken cancellationToken) - { - ValidateInvariants(); - - _operation = Operation.ReadScatter; - _readScatterBuffers = buffers; - _fileOffset = fileOffset; - _cancellationToken = cancellationToken; - QueueToThreadPool(); - - return new ValueTask(this, _source.Version); - } - - public ValueTask QueueWriteGather(IReadOnlyList> buffers, long fileOffset, CancellationToken cancellationToken) - { - ValidateInvariants(); - - _operation = Operation.WriteGather; - _writeGatherBuffers = buffers; - _fileOffset = fileOffset; - _cancellationToken = cancellationToken; - QueueToThreadPool(); - - return new ValueTask(this, _source.Version); - } - - private enum Operation : byte - { - None, - Read, - Write, - ReadScatter, - WriteGather - } - } - } -} diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs index f65aaf042bf891..c201db223de623 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs @@ -2,15 +2,25 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Buffers; +using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.IO.Strategies; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Threading; +using System.Threading.Tasks; +using System.Threading.Tasks.Sources; namespace Microsoft.Win32.SafeHandles { public sealed partial class SafeFileHandle : SafeHandleZeroOrMinusOneIsInvalid { + // IovStackThreshold matches Linux's UIO_FASTIOV, which is the number of 'struct iovec' + // that get stackalloced in the Linux kernel. + private const int IovStackThreshold = 8; + private const UnixFileMode PermissionMask = UnixFileMode.UserRead | UnixFileMode.UserWrite | @@ -40,9 +50,12 @@ public sealed partial class SafeFileHandle : SafeHandleZeroOrMinusOneIsInvalid // not using bool? as it's not thread safe private NullableBool _canSeek /* = NullableBool.Undefined */; private NullableBool _supportsRandomAccess /* = NullableBool.Undefined */; - private NullableBool _isAsync /* = NullableBool.Undefined */; private bool _deleteOnClose; private bool _isLocked; + private NullableBool _isBlocking; + private PollableHandle? _pollHandle; + private ReadOperation? _cachedReadOp; + private WriteOperation? _cachedWriteOp; public SafeFileHandle() : this(ownsHandle: true) { @@ -54,24 +67,92 @@ private SafeFileHandle(bool ownsHandle) SetHandle(new IntPtr(-1)); } - public bool IsAsync + private SafeFileHandle(FileHandleType type, bool nonBlocking) + : this(ownsHandle: true) + { + _cachedFileType = (int)type; + _isBlocking = nonBlocking ? NullableBool.False : NullableBool.True; + } + + public bool IsAsync => !IsBlocking; + + // RegularFile and BlockDevices don't support non-blocking and do support random access. + // Perform read/write operations on the ThreadPool so that multiple can happen in parallel. + // Pipe and Socket support non-blocking and do not support random access. + // Character devices may support non-blocking and may support random access. + private bool SupportsNonBlocking + => Type is FileHandleType.Socket or FileHandleType.Pipe or FileHandleType.CharacterDevice; + private bool UseThreadPoolForAsync + => !SupportsNonBlocking; + + private bool IsBlocking { get { - NullableBool isAsync = _isAsync; - if (isAsync == NullableBool.Undefined && !IsClosed) + NullableBool isBlocking = _isBlocking; + if (isBlocking == NullableBool.Undefined) { - if (Interop.Sys.Fcntl.GetIsNonBlocking(this, out bool isNonBlocking) != 0) + if (!SupportsNonBlocking) + { + _isBlocking = NullableBool.True; + return true; + } + + if (Interop.Sys.Fcntl.GetIsNonBlocking(this, out bool nonBlocking) != 0) { - throw Interop.GetExceptionForIoErrno(Interop.Sys.GetLastErrorInfo(), Path); + throw Interop.GetExceptionForIoErrno(Interop.Sys.GetLastErrorInfo()); } - _isAsync = isAsync = isNonBlocking ? NullableBool.True : NullableBool.False; + _isBlocking = isBlocking = nonBlocking ? NullableBool.False : NullableBool.True; + } + + return isBlocking == NullableBool.True; + } + } + + private void SetHandleNonBlocking() + { + Debug.Assert(SupportsNonBlocking); + + if (_isBlocking != NullableBool.False) + { + if (Interop.Sys.Fcntl.SetIsNonBlocking(this, 1) != 0) + { + throw Interop.GetExceptionForIoErrno(Interop.Sys.GetLastErrorInfo()); } + _isBlocking = NullableBool.False; + } + } - return isAsync == NullableBool.True; + private PollableHandle PollHandle + { + get + { + if (_pollHandle == null) + { + SetHandleNonBlocking(); + PollableHandle.Create(this, ref _pollHandle); + } + return _pollHandle!; } - private set => _isAsync = value ? NullableBool.True : NullableBool.False; + } + + private ReadOperation RentReadOperation() + => Interlocked.Exchange(ref _cachedReadOp, null) ?? new ReadOperation(this); + + private WriteOperation RentWriteOperation() + => Interlocked.Exchange(ref _cachedWriteOp, null) ?? new WriteOperation(this); + + private void ReturnReadOperation(ReadOperation op) + { + op.Reset(); + Volatile.Write(ref _cachedReadOp, op); + } + + private void ReturnWriteOperation(WriteOperation op) + { + op.Reset(); + Volatile.Write(ref _cachedWriteOp, op); } internal bool CanSeek => !IsClosed && GetCanSeek(); @@ -142,6 +223,15 @@ private static SafeFileHandle Open(string path, Interop.Sys.OpenFlags flags, int return handle; } + protected override void Dispose(bool disposing) + { + if (disposing) + { + _pollHandle?.Dispose(); + } + base.Dispose(disposing); + } + protected override bool ReleaseHandle() { // If DeleteOnClose was requested when constructed, delete the file now. @@ -183,8 +273,8 @@ public override bool IsInvalid public static partial void CreateAnonymousPipe(out SafeFileHandle readHandle, out SafeFileHandle writeHandle, bool asyncRead, bool asyncWrite) { // Allocate the handles first, so in case of OOM we don't leak any handles. - SafeFileHandle tempReadHandle = new(); - SafeFileHandle tempWriteHandle = new(); + SafeFileHandle tempReadHandle = new(FileHandleType.Pipe, nonBlocking: asyncRead); + SafeFileHandle tempWriteHandle = new(FileHandleType.Pipe, nonBlocking: asyncWrite); Interop.Sys.PipeFlags flags = Interop.Sys.PipeFlags.O_CLOEXEC; if (asyncRead) @@ -214,10 +304,7 @@ public static partial void CreateAnonymousPipe(out SafeFileHandle readHandle, ou } tempReadHandle.SetHandle(readFd); - tempReadHandle.IsAsync = asyncRead; - tempWriteHandle.SetHandle(writeFd); - tempWriteHandle.IsAsync = asyncWrite; readHandle = tempReadHandle; writeHandle = tempWriteHandle; @@ -408,8 +495,6 @@ private bool Init(string path, FileMode mode, FileAccess access, FileShare share filePermissions = ((UnixFileMode)status.Mode) & PermissionMask; } - IsAsync = false; // Unix does not support O_NONBLOCK for regular files. - // Lock the file if requested via FileShare. This is only advisory locking. FileShare.None implies an exclusive // lock on the file and all other modes use a shared lock. While this is not as granular as Windows, not mandatory, // and not atomic with file opening, it's better than nothing. @@ -553,7 +638,8 @@ private bool GetCanSeek() NullableBool canSeek = _canSeek; if (canSeek == NullableBool.Undefined) { - _canSeek = canSeek = Interop.Sys.LSeek(this, 0, Interop.Sys.SeekWhence.SEEK_CUR) >= 0 ? NullableBool.True : NullableBool.False; + bool unseekable = Type is FileHandleType.Socket or FileHandleType.Pipe; + _canSeek = canSeek = !unseekable && Interop.Sys.LSeek(this, 0, Interop.Sys.SeekWhence.SEEK_CUR) >= 0 ? NullableBool.True : NullableBool.False; } return canSeek == NullableBool.True; @@ -590,5 +676,1205 @@ internal long GetFileLength() FileStreamHelpers.CheckFileCall(result, Path); return status.Size; } + + internal unsafe int Read(long offset, Span buffer) + { + int sequenceNumber = 0; + bool isBlocking = IsBlocking; + + bool doSync = isBlocking || PollHandle.IsReadReady(out sequenceNumber); + if (doSync) + { + fixed (byte* bufPtr = &MemoryMarshal.GetReference(buffer)) + { + if (TryCompleteReadAt(offset, bufPtr, buffer.Length, out var result, out bool pending)) + { + return CheckFileCall(result.BytesRead, result.ErrorInfo); + } + if (isBlocking) + { + Debug.Assert(pending); + if (PollHandle.IsReadReady(out sequenceNumber) && TryCompleteReadAt(offset, bufPtr, buffer.Length, out result, out _)) + { + return CheckFileCall(result.BytesRead, result.ErrorInfo); + } + } + } + } + + ReadOperation op = RentReadOperation(); + fixed (byte* bufPtr = &MemoryMarshal.GetReference(buffer)) + { + op.Init(offset, bufPtr, buffer.Length); + + PollOperationSyncResult result = PollHandle.ReadSync(op, sequenceNumber, timeout: -1); + + if (result == PollOperationSyncResult.Completed) + { + int readResult = (int)op.BytesRead; + Exception? exception = op.Exception; + + ReturnReadOperation(op); + + if (exception != null) + { + throw exception; + } + return readResult; + } + + throw new OperationCanceledException(); + } + } + + internal unsafe ValueTask ReadAsync(long offset, Memory destination, CancellationToken cancellationToken, OSFileStreamStrategy? strategy = null) + => UseThreadPoolForAsync + ? ReadAsyncThreadPool(offset, destination, cancellationToken, strategy) + : ReadAsyncPollable(offset, destination, cancellationToken, strategy); + + private ValueTask ReadAsyncThreadPool(long offset, Memory destination, CancellationToken cancellationToken, OSFileStreamStrategy? strategy) + { + ReadOperation op = RentReadOperation(); + op.Init(offset, destination, cancellationToken, strategy); + op.QueueToThreadPool(); + return new ValueTask(op, op.Version); + } + + private unsafe ValueTask ReadAsyncPollable(long offset, Memory destination, CancellationToken cancellationToken, OSFileStreamStrategy? strategy) + { + int sequenceNumber; + if (PollHandle.IsReadReady(out sequenceNumber)) + { + fixed (byte* bufPtr = &MemoryMarshal.GetReference(destination.Span)) + { + if (TryCompleteReadAt(offset, bufPtr, destination.Length, out var readResult, out _)) + { + UpdateFileStreamForAsyncRead(strategy, destination, readResult.BytesRead); + return new ValueTask(CheckFileCall(readResult.BytesRead, readResult.ErrorInfo)); + } + } + } + + ReadOperation op = RentReadOperation(); + op.Init(offset, destination, cancellationToken, strategy); + + PollOperationAsyncResult result = PollHandle.ReadAsync(op, sequenceNumber, cancellationToken); + + if (result == PollOperationAsyncResult.Pending) + { + return new ValueTask(op, op.Version); + } + else if (result == PollOperationAsyncResult.Completed) + { + int completedResult = (int)op.BytesRead; + Exception? exception = op.Exception; + UpdateFileStreamForAsyncRead(strategy, destination, completedResult); + + ReturnReadOperation(op); + + if (exception != null) + { + throw exception; + } + return new ValueTask(completedResult); + } + + throw new OperationCanceledException(); + } + + internal unsafe void Write(long offset, ReadOnlySpan buffer) + { + if (buffer.IsEmpty) + { + return; + } + + int sequenceNumber = 0; + bool isBlocking = IsBlocking; + + bool doSync = isBlocking || PollHandle.IsWriteReady(out sequenceNumber); + while (doSync) + { + fixed (byte* bufPtr = &MemoryMarshal.GetReference(buffer)) + { + if (TryCompleteWriteAt(offset, bufPtr, buffer.Length, out int bytesWritten, out Interop.ErrorInfo errorInfo, out bool pending)) + { + CheckFileCall(errorInfo); + return; + } + buffer = buffer.Slice(bytesWritten); + offset += bytesWritten; + if (isBlocking && pending) + { + isBlocking = false; + doSync = PollHandle.IsWriteReady(out sequenceNumber); + } + else + { + Debug.Assert(buffer.Length != 0); + doSync = isBlocking; + } + } + } + + WriteOperation op = RentWriteOperation(); + fixed (byte* bufPtr = &MemoryMarshal.GetReference(buffer)) + { + op.Init(offset, bufPtr, buffer.Length); + + PollOperationSyncResult result = PollHandle.WriteSync(op, sequenceNumber, timeout: -1); + + if (result == PollOperationSyncResult.Completed) + { + Exception? exception = op.Exception; + + ReturnWriteOperation(op); + + if (exception != null) + { + throw exception; + } + return; + } + + throw new OperationCanceledException(); + } + } + + internal unsafe ValueTask WriteAsync(long offset, ReadOnlyMemory source, CancellationToken cancellationToken, OSFileStreamStrategy? strategy = null) + => UseThreadPoolForAsync + ? WriteAsyncThreadPool(offset, source, cancellationToken, strategy) + : WriteAsyncPollable(offset, source, cancellationToken, strategy); + + private ValueTask WriteAsyncThreadPool(long offset, ReadOnlyMemory source, CancellationToken cancellationToken, OSFileStreamStrategy? strategy) + { + if (source.IsEmpty) + { + return default; + } + + WriteOperation op = RentWriteOperation(); + op.Init(offset, source, cancellationToken, strategy); + op.QueueToThreadPool(); + return new ValueTask(op, op.Version); + } + + private unsafe ValueTask WriteAsyncPollable(long offset, ReadOnlyMemory source, CancellationToken cancellationToken, OSFileStreamStrategy? strategy) + { + if (source.IsEmpty) + { + return default; + } + + int bytesWritten = 0; + int sequenceNumber; + if (PollHandle.IsWriteReady(out sequenceNumber)) + { + fixed (byte* bufPtr = &MemoryMarshal.GetReference(source.Span)) + { + if (TryCompleteWriteAt(offset, bufPtr, source.Length, out bytesWritten, out Interop.ErrorInfo writeResult, out _)) + { + UpdateFileStreamForAsyncWrite(strategy, source.Slice(Math.Max(bytesWritten, 0))); + CheckFileCall(writeResult); + return default; + } + } + } + + WriteOperation op = RentWriteOperation(); + op.Init(offset + bytesWritten, source.Slice(bytesWritten), cancellationToken, strategy); + + PollOperationAsyncResult result = PollHandle.WriteAsync(op, sequenceNumber, cancellationToken); + + if (result == PollOperationAsyncResult.Pending) + { + return new ValueTask(op, op.Version); + } + else if (result == PollOperationAsyncResult.Completed) + { + Exception? exception = op.Exception; + UpdateFileStreamForAsyncWrite(strategy, op.Remaining); + + ReturnWriteOperation(op); + + if (exception != null) + { + throw exception; + } + return default; + } + + throw new OperationCanceledException(); + } + + internal long Read(long offset, IReadOnlyList> buffers) + { + int sequenceNumber = 0; + bool isBlocking = IsBlocking; + + bool doSync = isBlocking || PollHandle.IsReadReady(out sequenceNumber); + if (doSync) + { + if (TryCompleteReadAt(offset, buffers, out var result, out bool pending)) + { + return CheckFileCall(result.BytesRead, result.ErrorInfo); + } + if (isBlocking) + { + Debug.Assert(pending); + if (PollHandle.IsReadReady(out sequenceNumber) && TryCompleteReadAt(offset, buffers, out result, out _)) + { + return CheckFileCall(result.BytesRead, result.ErrorInfo); + } + } + } + + ReadOperation op = RentReadOperation(); + op.Init(offset, buffers); + + PollOperationSyncResult result2 = PollHandle.ReadSync(op, sequenceNumber, timeout: -1); + + if (result2 == PollOperationSyncResult.Completed) + { + long readResult = op.BytesRead; + Exception? exception = op.Exception; + + ReturnReadOperation(op); + + if (exception != null) + { + throw exception; + } + return readResult; + } + + throw new OperationCanceledException(); + } + + internal ValueTask ReadAsync(long offset, IReadOnlyList> buffers, CancellationToken cancellationToken) + => UseThreadPoolForAsync + ? ReadAsyncThreadPool(offset, buffers, cancellationToken) + : ReadAsyncPollable(offset, buffers, cancellationToken); + + private ValueTask ReadAsyncThreadPool(long offset, IReadOnlyList> buffers, CancellationToken cancellationToken) + { + ReadOperation op = RentReadOperation(); + op.Init(offset, buffers, cancellationToken); + op.QueueToThreadPool(); + return new ValueTask(op, op.Version); + } + + private ValueTask ReadAsyncPollable(long offset, IReadOnlyList> buffers, CancellationToken cancellationToken) + { + int sequenceNumber; + if (PollHandle.IsReadReady(out sequenceNumber) && + TryCompleteReadAt(offset, buffers, out var readResult, out _)) + { + return new ValueTask(CheckFileCall(readResult.BytesRead, readResult.ErrorInfo)); + } + + ReadOperation op = RentReadOperation(); + op.Init(offset, buffers, cancellationToken); + + PollOperationAsyncResult result = PollHandle.ReadAsync(op, sequenceNumber, cancellationToken); + + if (result == PollOperationAsyncResult.Pending) + { + return new ValueTask(op, op.Version); + } + else if (result == PollOperationAsyncResult.Completed) + { + long completedResult = op.BytesRead; + Exception? exception = op.Exception; + + ReturnReadOperation(op); + + if (exception != null) + { + throw exception; + } + return new ValueTask(completedResult); + } + + throw new OperationCanceledException(); + } + + internal void Write(long offset, IReadOnlyList> buffers) + { + int bufferIndex = 0; + int bufferOffset = 0; + int sequenceNumber = 0; + bool isBlocking = IsBlocking; + + bool doSync = isBlocking || PollHandle.IsWriteReady(out sequenceNumber); + while (doSync) + { + if (TryCompleteWriteAt(ref offset, buffers, ref bufferIndex, ref bufferOffset, out Interop.ErrorInfo errorInfo, out bool pending)) + { + CheckFileCall(errorInfo); + return; + } + if (isBlocking && pending) + { + isBlocking = false; + doSync = PollHandle.IsWriteReady(out sequenceNumber); + } + else + { + doSync = isBlocking; + } + } + + WriteOperation op = RentWriteOperation(); + op.Init(offset, buffers, bufferIndex, bufferOffset); + + PollOperationSyncResult result = PollHandle.WriteSync(op, sequenceNumber, timeout: -1); + + if (result == PollOperationSyncResult.Completed) + { + Exception? exception = op.Exception; + + ReturnWriteOperation(op); + + if (exception != null) + { + throw exception; + } + return; + } + + throw new OperationCanceledException(); + } + + internal ValueTask WriteAsync(long offset, IReadOnlyList> buffers, CancellationToken cancellationToken) + => UseThreadPoolForAsync + ? WriteAsyncThreadPool(offset, buffers, cancellationToken) + : WriteAsyncPollable(offset, buffers, cancellationToken); + + private ValueTask WriteAsyncThreadPool(long offset, IReadOnlyList> buffers, CancellationToken cancellationToken) + { + WriteOperation op = RentWriteOperation(); + op.Init(offset, buffers, 0, 0, cancellationToken); + op.QueueToThreadPool(); + return new ValueTask(op, op.Version); + } + + private ValueTask WriteAsyncPollable(long offset, IReadOnlyList> buffers, CancellationToken cancellationToken) + { + int bufferIndex = 0; + int bufferOffset = 0; + int sequenceNumber; + if (PollHandle.IsWriteReady(out sequenceNumber)) + { + if (TryCompleteWriteAt(ref offset, buffers, ref bufferIndex, ref bufferOffset, out Interop.ErrorInfo writeResult, out _)) + { + CheckFileCall(writeResult); + return default; + } + } + + WriteOperation op = RentWriteOperation(); + op.Init(offset, buffers, bufferIndex, bufferOffset, cancellationToken); + + PollOperationAsyncResult result = PollHandle.WriteAsync(op, sequenceNumber, cancellationToken); + + if (result == PollOperationAsyncResult.Pending) + { + return new ValueTask(op, op.Version); + } + else if (result == PollOperationAsyncResult.Completed) + { + Exception? exception = op.Exception; + + ReturnWriteOperation(op); + + if (exception != null) + { + throw exception; + } + return default; + } + + throw new OperationCanceledException(); + } + + private sealed unsafe class ReadOperation : PollTriggeredOperation, IValueTaskSource, IValueTaskSource + { + private readonly SafeFileHandle _owner; + private ManualResetValueTaskSourceCore _mrvtsc; + private Memory _buffer; + private IReadOnlyList>? _buffers; + private byte* _syncBuffer; + private int _syncBufferLength; + private long _offset; + private bool _runOnThreadPool; + private ExecutionContext? _executionContext; + private CancellationToken _cancellationToken; + private OSFileStreamStrategy? _strategy; + + internal long BytesRead; + internal Exception? Exception; + + internal ReadOperation(SafeFileHandle owner) + => _owner = owner; + + internal short Version + => _mrvtsc.Version; + + internal void Init(long offset, byte* syncBuffer, int syncBufferLength) + { + _offset = offset; + _syncBuffer = syncBuffer; + _syncBufferLength = syncBufferLength; + } + + internal void Init(long offset, Memory buffer, CancellationToken cancellationToken, OSFileStreamStrategy? strategy = null) + { + _offset = offset; + _buffer = buffer; + _cancellationToken = cancellationToken; + _strategy = strategy; + } + + internal void Init(long offset, IReadOnlyList> buffers) + { + _offset = offset; + _buffers = buffers; + } + + internal void Init(long offset, IReadOnlyList> buffers, CancellationToken cancellationToken) + { + _offset = offset; + _buffers = buffers; + _cancellationToken = cancellationToken; + } + + internal void QueueToThreadPool() + { + _runOnThreadPool = true; + _executionContext = ExecutionContext.Capture(); + bool refAdded = false; + _owner.DangerousAddRef(ref refAdded); + try + { + ThreadPool.UnsafeQueueUserWorkItem(this, preferLocal: true); + } + catch + { + _owner.DangerousRelease(); + throw; + } + } + + private void ExecuteOnThreadPool() + { + bool completed = TryCompleteOperation(_owner); + Debug.Assert(completed); + _owner.DangerousRelease(); + OnCompleted(PollOperationOnCompletedResult.Completed); + } + + protected override void ExecuteThreadPoolWorkItem() + { + if (!_runOnThreadPool) + { + base.ExecuteThreadPoolWorkItem(); + return; + } + + if (_executionContext == null || _executionContext.IsDefault) + { + ExecuteOnThreadPool(); + } + else + { + ExecutionContext.RunForThreadPoolUnsafe(_executionContext, static x => x.ExecuteOnThreadPool(), this); + } + } + + internal void Reset() + { + _buffer = default; + _buffers = null; + _syncBuffer = null; + _offset = 0; + _runOnThreadPool = false; + _executionContext = null; + _cancellationToken = default; + _strategy = null; + Exception = null; + _mrvtsc.Reset(); + } + + protected internal override bool TryCompleteOperation(SafeHandle handle) + { + (int BytesRead, Interop.ErrorInfo ErrorInfo) readResult; + + if (_buffers != null) + { + if (!_owner.TryCompleteReadAt(_offset, _buffers, out readResult, out _)) + { + return false; + } + } + else if (_syncBuffer != null) + { + Debug.Assert(_syncBufferLength > 0); + if (!_owner.TryCompleteReadAt(_offset, _syncBuffer, _syncBufferLength, out readResult, out _)) + { + return false; + } + } + else + { + Span span = _buffer.Span; + Debug.Assert(!span.IsEmpty); + + fixed (byte* bufPtr = &MemoryMarshal.GetReference(span)) + { + if (!_owner.TryCompleteReadAt(_offset, bufPtr, span.Length, out readResult, out _)) + { + return false; + } + } + } + + BytesRead = readResult.BytesRead; + if (readResult.BytesRead == -1) + { + Exception = Interop.GetExceptionForIoErrno(readResult.ErrorInfo, _owner.Path); + } + return true; + } + + protected internal override void OnCompleted(PollOperationOnCompletedResult result) + { + UpdateFileStreamForAsyncRead(_strategy, _buffer, (int)BytesRead); + + if (result == PollOperationOnCompletedResult.Completed) + { + if (Exception != null) + { + _mrvtsc.SetException(Exception); + } + else + { + _mrvtsc.SetResult(BytesRead); + } + } + else if (result == PollOperationOnCompletedResult.Canceled) + { + _mrvtsc.SetException(new OperationCanceledException(_cancellationToken)); + } + else + { + Debug.Assert(result == PollOperationOnCompletedResult.Aborted); + _mrvtsc.SetException(new OperationCanceledException()); + } + } + + private long GetResultAndPool(short token) + { + bool canPool = _mrvtsc.GetStatus(token) != ValueTaskSourceStatus.Canceled; + try + { + return _mrvtsc.GetResult(token); + } + finally + { + if (canPool) + { + _owner.ReturnReadOperation(this); + } + } + } + + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) + => _mrvtsc.GetStatus(token); + + void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) + => _mrvtsc.OnCompleted(continuation, state, token, flags); + + int IValueTaskSource.GetResult(short token) + => (int)GetResultAndPool(token); + + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) + => _mrvtsc.GetStatus(token); + + void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) + => _mrvtsc.OnCompleted(continuation, state, token, flags); + + long IValueTaskSource.GetResult(short token) + => GetResultAndPool(token); + } + + private sealed unsafe class WriteOperation : PollTriggeredOperation, IValueTaskSource + { + private readonly SafeFileHandle _owner; + private ManualResetValueTaskSourceCore _mrvtsc; + private ReadOnlyMemory _buffer; + private IReadOnlyList>? _buffers; + private byte* _syncBuffer; + private int _syncRemaining; + private long _offset; + private int _bufferIndex; + private int _bufferOffset; + private bool _runOnThreadPool; + private ExecutionContext? _executionContext; + private CancellationToken _cancellationToken; + private OSFileStreamStrategy? _strategy; + + internal Exception? Exception; + + internal WriteOperation(SafeFileHandle owner) + => _owner = owner; + + internal short Version + => _mrvtsc.Version; + + internal ReadOnlyMemory Remaining + => _buffer; + + internal void Init(long offset, byte* syncBuffer, int syncRemaining) + { + _offset = offset; + _syncBuffer = syncBuffer; + _syncRemaining = syncRemaining; + } + + internal void Init(long offset, ReadOnlyMemory buffer, CancellationToken cancellationToken, OSFileStreamStrategy? strategy = null) + { + _offset = offset; + _buffer = buffer; + _cancellationToken = cancellationToken; + _strategy = strategy; + } + + internal void Init(long offset, IReadOnlyList> buffers, int bufferIndex, int bufferOffset) + { + _offset = offset; + _buffers = buffers; + _bufferIndex = bufferIndex; + _bufferOffset = bufferOffset; + } + + internal void Init(long offset, IReadOnlyList> buffers, int bufferIndex, int bufferOffset, CancellationToken cancellationToken) + { + _offset = offset; + _buffers = buffers; + _bufferIndex = bufferIndex; + _bufferOffset = bufferOffset; + _cancellationToken = cancellationToken; + } + + internal void QueueToThreadPool() + { + _runOnThreadPool = true; + _executionContext = ExecutionContext.Capture(); + bool refAdded = false; + _owner.DangerousAddRef(ref refAdded); + try + { + ThreadPool.UnsafeQueueUserWorkItem(this, preferLocal: true); + } + catch + { + _owner.DangerousRelease(); + throw; + } + } + + private void ExecuteOnThreadPool() + { + while (!TryCompleteOperation(_owner)) + { + } + _owner.DangerousRelease(); + OnCompleted(PollOperationOnCompletedResult.Completed); + } + + protected override void ExecuteThreadPoolWorkItem() + { + if (!_runOnThreadPool) + { + base.ExecuteThreadPoolWorkItem(); + return; + } + + if (_executionContext == null || _executionContext.IsDefault) + { + ExecuteOnThreadPool(); + } + else + { + ExecutionContext.RunForThreadPoolUnsafe(_executionContext, static x => x.ExecuteOnThreadPool(), this); + } + } + + internal void Reset() + { + _buffer = default; + _buffers = null; + _syncBuffer = null; + _offset = 0; + _bufferIndex = 0; + _bufferOffset = 0; + _runOnThreadPool = false; + _executionContext = null; + _cancellationToken = default; + _strategy = null; + Exception = null; + _mrvtsc.Reset(); + } + + protected internal override bool TryCompleteOperation(SafeHandle handle) + { + if (_buffers != null) + { + if (_owner.TryCompleteWriteAt(ref _offset, _buffers, ref _bufferIndex, ref _bufferOffset, out Interop.ErrorInfo errorInfo, out bool pending)) + { + if (errorInfo.Error != Interop.Error.SUCCESS) + { + Exception = Interop.GetExceptionForIoErrno(errorInfo, _owner.Path); + } + return true; + } + Debug.Assert(!_runOnThreadPool || !pending, "ThreadPool only used with non-blocking"); + return false; + } + + if (_syncBuffer != null) + { + Debug.Assert(_syncRemaining > 0); + + if (_owner.TryCompleteWriteAt(_offset, _syncBuffer, _syncRemaining, out int bytesWritten, out Interop.ErrorInfo errorInfo, out bool pending)) + { + if (errorInfo.Error != Interop.Error.SUCCESS) + { + Exception = Interop.GetExceptionForIoErrno(errorInfo, _owner.Path); + } + return true; + } + _syncBuffer += bytesWritten; + _syncRemaining -= bytesWritten; + _offset += bytesWritten; + Debug.Assert(!_runOnThreadPool || !pending, "ThreadPool only used with non-blocking"); + return false; + } + + ReadOnlySpan span = _buffer.Span; + Debug.Assert(!span.IsEmpty); + + fixed (byte* bufPtr = &MemoryMarshal.GetReference(span)) + { + if (_owner.TryCompleteWriteAt(_offset, bufPtr, span.Length, out int bytesWritten, out Interop.ErrorInfo errorInfo, out bool pending)) + { + if (errorInfo.Error != Interop.Error.SUCCESS) + { + Exception = Interop.GetExceptionForIoErrno(errorInfo, _owner.Path); + } + else + { + _buffer = default; // Clear for UpdateFileStreamForAsyncWrite. + } + return true; + } + _buffer = _buffer.Slice(bytesWritten); + _offset += bytesWritten; + Debug.Assert(!_runOnThreadPool || !pending, "ThreadPool only used with non-blocking"); + return false; + } + } + + protected internal override void OnCompleted(PollOperationOnCompletedResult result) + { + UpdateFileStreamForAsyncWrite(_strategy, _buffer); + + if (result == PollOperationOnCompletedResult.Completed) + { + if (Exception != null) + { + _mrvtsc.SetException(Exception); + } + else + { + _mrvtsc.SetResult(default); + } + } + else if (result == PollOperationOnCompletedResult.Canceled) + { + _mrvtsc.SetException(new OperationCanceledException(_cancellationToken)); + } + else + { + Debug.Assert(result == PollOperationOnCompletedResult.Aborted); + _mrvtsc.SetException(new OperationCanceledException()); + } + } + + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) + => _mrvtsc.GetStatus(token); + + void IValueTaskSource.OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) + => _mrvtsc.OnCompleted(continuation, state, token, flags); + + void IValueTaskSource.GetResult(short token) + { + bool canPool = _mrvtsc.GetStatus(token) != ValueTaskSourceStatus.Canceled; + try + { + _mrvtsc.GetResult(token); + } + finally + { + if (canPool) + { + _owner.ReturnWriteOperation(this); + } + } + } + } + + private unsafe bool TryCompleteReadAt(long offset, IReadOnlyList> buffers, out (int BytesRead, Interop.ErrorInfo ErrorInfo) result, out bool pending) + { + if (SupportsRandomAccess) + { + if (TryCompleteReadAt(useOffset: true, offset, buffers, out result, out pending)) + { + if (result.BytesRead == -1 && ShouldFallBackToNonOffsetSyscall(result.ErrorInfo)) + { + SupportsRandomAccess = false; + } + else + { + return true; + } + } + else + { + return false; + } + } + + return TryCompleteReadAt(useOffset: false, offset, buffers, out result, out pending); + } + + private unsafe bool TryCompleteReadAt(bool useOffset, long offset, IReadOnlyList> buffers, out (int BytesRead, Interop.ErrorInfo ErrorInfo) result, out bool pending) + { + int count = buffers.Count; + MemoryHandle[] memHandles = new MemoryHandle[count]; + Span vectors = count <= IovStackThreshold + ? stackalloc Interop.Sys.IOVector[IovStackThreshold].Slice(0, count) + : new Interop.Sys.IOVector[count]; + + try + { + for (int i = 0; i < count; i++) + { + Memory buffer = buffers[i]; + MemoryHandle mh = buffer.Pin(); + vectors[i] = new Interop.Sys.IOVector { Base = (byte*)mh.Pointer, Count = (UIntPtr)buffer.Length }; + memHandles[i] = mh; + } + + fixed (Interop.Sys.IOVector* pinnedVectors = &MemoryMarshal.GetReference(vectors)) + { + long bytesRead = useOffset + ? Interop.Sys.PReadV(this, pinnedVectors, count, offset) + : Interop.Sys.ReadV(this, pinnedVectors, count); + if (bytesRead < 0) + { + Interop.ErrorInfo errorInfo = Interop.Sys.GetLastErrorInfo(); + if (IsPending(errorInfo)) + { + pending = true; + result = default; + return false; + } + pending = false; + result = (-1, errorInfo); + return true; + } + + pending = false; + result = ((int)bytesRead, default); + return true; + } + } + finally + { + foreach (MemoryHandle mh in memHandles) + { + mh.Dispose(); + } + } + } + + private unsafe bool TryCompleteReadAt(long offset, byte* buffer, int length, out (int BytesRead, Interop.ErrorInfo ErrorInfo) result, out bool pending) + { + if (SupportsRandomAccess) + { + if (TryCompleteReadAt(useOffset: true, offset, buffer, length, out result, out pending)) + { + if (result.BytesRead == -1 && ShouldFallBackToNonOffsetSyscall(result.ErrorInfo)) + { + SupportsRandomAccess = false; + } + else + { + return true; + } + } + else + { + return false; + } + } + + return TryCompleteReadAt(useOffset: false, offset, buffer, length, out result, out pending); + } + + private unsafe bool TryCompleteReadAt(bool useOffset, long offset, byte* buffer, int length, out (int BytesRead, Interop.ErrorInfo ErrorInfo) result, out bool pending) + { + int bytesRead = useOffset + ? Interop.Sys.PRead(this, buffer, length, offset) + : Interop.Sys.Read(this, buffer, length); + if (bytesRead < 0) + { + Interop.ErrorInfo errorInfo = Interop.Sys.GetLastErrorInfo(); + if (IsPending(errorInfo)) + { + pending = true; + result = default; + return false; + } + pending = false; + result = (-1, errorInfo); + return true; + } + + pending = false; + result = (bytesRead, default); + return true; + } + + private unsafe bool TryCompleteWriteAt(ref long offset, IReadOnlyList> buffers, ref int bufferIndex, ref int bufferOffset, out Interop.ErrorInfo errorInfo, out bool pending) + { + if (SupportsRandomAccess) + { + if (TryCompleteWriteAt(useOffset: true, ref offset, buffers, ref bufferIndex, ref bufferOffset, out errorInfo, out pending)) + { + if (errorInfo.Error != Interop.Error.SUCCESS && ShouldFallBackToNonOffsetSyscall(errorInfo)) + { + SupportsRandomAccess = false; + } + else + { + return true; + } + } + else + { + return false; + } + } + + return TryCompleteWriteAt(useOffset: false, ref offset, buffers, ref bufferIndex, ref bufferOffset, out errorInfo, out pending); + } + + private unsafe bool TryCompleteWriteAt(bool useOffset, ref long offset, IReadOnlyList> buffers, ref int bufferIndex, ref int bufferOffset, out Interop.ErrorInfo errorInfo, out bool pending) + { + while (bufferIndex < buffers.Count && buffers[bufferIndex].Length <= bufferOffset) + { + bufferIndex++; + bufferOffset = 0; + } + + if (bufferIndex >= buffers.Count) + { + errorInfo = default; + pending = false; + return true; + } + + int remaining = buffers.Count - bufferIndex; + MemoryHandle[] memHandles = new MemoryHandle[remaining]; + Span vectors = remaining <= IovStackThreshold + ? stackalloc Interop.Sys.IOVector[IovStackThreshold].Slice(0, remaining) + : new Interop.Sys.IOVector[remaining]; + + try + { + long totalToWrite = 0; + for (int i = 0; i < remaining; i++) + { + ReadOnlyMemory buf = buffers[bufferIndex + i]; + MemoryHandle mh = buf.Pin(); + byte* ptr = (byte*)mh.Pointer; + int len = buf.Length; + if (i == 0 && bufferOffset > 0) + { + ptr += bufferOffset; + len -= bufferOffset; + } + vectors[i] = new Interop.Sys.IOVector { Base = ptr, Count = (UIntPtr)len }; + memHandles[i] = mh; + totalToWrite += len; + } + + fixed (Interop.Sys.IOVector* pinnedVectors = &MemoryMarshal.GetReference(vectors)) + { + long bytesWritten = useOffset + ? Interop.Sys.PWriteV(this, pinnedVectors, remaining, offset) + : Interop.Sys.WriteV(this, pinnedVectors, remaining); + if (bytesWritten < 0) + { + errorInfo = Interop.Sys.GetLastErrorInfo(); + if (IsPending(errorInfo)) + { + pending = true; + return false; + } + pending = false; + return true; + } + + pending = false; + errorInfo = default; + offset += bytesWritten; + + if (bytesWritten == totalToWrite) + { + bufferIndex = buffers.Count; + bufferOffset = 0; + return true; + } + + long written = bytesWritten; + while (written > 0) + { + int currentLen = buffers[bufferIndex].Length - bufferOffset; + if (written >= currentLen) + { + written -= currentLen; + bufferIndex++; + bufferOffset = 0; + } + else + { + bufferOffset += (int)written; + written = 0; + } + } + return false; + } + } + finally + { + foreach (MemoryHandle mh in memHandles) + { + mh.Dispose(); + } + } + } + + private unsafe bool TryCompleteWriteAt(long offset, byte* buffer, int length, out int bytesWritten, out Interop.ErrorInfo errorInfo, out bool pending) + { + if (SupportsRandomAccess) + { + if (TryCompleteWriteAt(useOffset: true, offset, buffer, length, out bytesWritten, out errorInfo, out pending)) + { + if (errorInfo.Error != Interop.Error.SUCCESS && ShouldFallBackToNonOffsetSyscall(errorInfo)) + { + SupportsRandomAccess = false; + } + else + { + return true; + } + } + else + { + return false; + } + } + + return TryCompleteWriteAt(useOffset: false, offset, buffer, length, out bytesWritten, out errorInfo, out pending); + } + + private unsafe bool TryCompleteWriteAt(bool useOffset, long offset, byte* buffer, int length, out int bytesWritten, out Interop.ErrorInfo errorInfo, out bool pending) + { + int toWrite = GetNumberOfBytesToWrite(length); + bytesWritten = useOffset + ? Interop.Sys.PWrite(this, buffer, toWrite, offset) + : Interop.Sys.Write(this, buffer, toWrite); + if (bytesWritten < 0) + { + errorInfo = Interop.Sys.GetLastErrorInfo(); + if (IsPending(errorInfo)) + { + pending = true; + bytesWritten = 0; + return false; + } + pending = false; + return true; + } + + pending = false; + errorInfo = default; + return bytesWritten == length; + } + + private static void UpdateFileStreamForAsyncRead(OSFileStreamStrategy? strategy, Memory destination, int bytesRead) + { + if (strategy is not null) + { + int bytesRemaining = destination.Length - Math.Max(bytesRead, 0); + strategy.OnIncompleteOperation(bytesRemaining, 0); + } + } + + private static void UpdateFileStreamForAsyncWrite(OSFileStreamStrategy? strategy, ReadOnlyMemory remaining) + { + strategy?.OnIncompleteOperation(remaining.Length, 0); + } + + private int CheckFileCall(int result, Interop.ErrorInfo errorInfo) + { + if (result == -1) + { + throw Interop.GetExceptionForIoErrno(errorInfo, Path); + } + return result; + } + + private void CheckFileCall(Interop.ErrorInfo errorInfo) + { + if (errorInfo.Error != Interop.Error.SUCCESS) + { + throw Interop.GetExceptionForIoErrno(errorInfo, Path); + } + } + + private static bool ShouldFallBackToNonOffsetSyscall(Interop.ErrorInfo errorInfo) + => errorInfo.Error is Interop.Error.ENXIO or Interop.Error.ESPIPE; + + private static bool IsPending(Interop.ErrorInfo errorInfo) + => errorInfo.Error is Interop.Error.EAGAIN or Interop.Error.EWOULDBLOCK; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int GetNumberOfBytesToWrite(int byteCount) + { +#if DEBUG + // In debug only, to assist with testing, simulate writing fewer than the requested number of bytes. + if (byteCount > 1 && // ensure we don't turn the write into a zero-byte write + byteCount < 512) // avoid on larger buffers that might have a length used to meet an alignment requirement + { + byteCount /= 2; + } +#endif + return byteCount; + } } } diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index 2c48c4cba7ffd1..4c9d8630375d38 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -2641,7 +2641,6 @@ Common\Interop\Android\Interop.Libraries.cs - @@ -2903,6 +2902,29 @@ + + + + + + + + + + + + + + + Common\Interop\Unix\System.Native\Interop.HandleEvents.cs + + + + + + Common\Interop\Wasi\System.Native\Interop.HandleEvents.cs + + diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs index c16a258a0d701e..73cf094a025b19 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs @@ -1,12 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Buffers; using System.Collections.Generic; -using System.Diagnostics; using System.IO.Strategies; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; using Microsoft.Win32.SafeHandles; @@ -15,262 +11,31 @@ namespace System.IO { public static partial class RandomAccess { - // IovStackThreshold matches Linux's UIO_FASTIOV, which is the number of 'struct iovec' - // that get stackalloced in the Linux kernel. - private const int IovStackThreshold = 8; - internal static unsafe void SetFileLength(SafeFileHandle handle, long length) => FileStreamHelpers.CheckFileCall(Interop.Sys.FTruncate(handle, length), handle.Path); internal static unsafe int ReadAtOffset(SafeFileHandle handle, Span buffer, long fileOffset) - { - fixed (byte* bufPtr = &MemoryMarshal.GetReference(buffer)) - { - // The Windows implementation uses ReadFile, which ignores the offset if the handle - // isn't seekable. We do the same manually with PRead vs Read, in order to enable - // the function to be used by FileStream for all the same situations. - int result = -1; - if (handle.IsAsync) - { - result = Interop.Sys.ReadFromNonblocking(handle, bufPtr, buffer.Length); - } - else if (handle.SupportsRandomAccess) - { - // Try pread for seekable files. - result = Interop.Sys.PRead(handle, bufPtr, buffer.Length, fileOffset); - - if (result == -1 && ShouldFallBackToNonOffsetSyscall(Interop.Sys.GetLastErrorInfo())) - { - handle.SupportsRandomAccess = false; // Fall through to non-offset Read below. - } - } - - if (!handle.IsAsync && !handle.SupportsRandomAccess) - { - result = Interop.Sys.Read(handle, bufPtr, buffer.Length); - } - - FileStreamHelpers.CheckFileCall(result, handle.Path); - return result; - } - } + => handle.Read(fileOffset, buffer); internal static unsafe long ReadScatterAtOffset(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset) - { - MemoryHandle[] handles = new MemoryHandle[buffers.Count]; - Span vectors = buffers.Count <= IovStackThreshold ? stackalloc Interop.Sys.IOVector[IovStackThreshold] : new Interop.Sys.IOVector[buffers.Count]; - - long result = -1; - try - { - int buffersCount = buffers.Count; - for (int i = 0; i < buffersCount; i++) - { - Memory buffer = buffers[i]; - MemoryHandle memoryHandle = buffer.Pin(); - vectors[i] = new Interop.Sys.IOVector { Base = (byte*)memoryHandle.Pointer, Count = (UIntPtr)buffer.Length }; - handles[i] = memoryHandle; - } - - fixed (Interop.Sys.IOVector* pinnedVectors = &MemoryMarshal.GetReference(vectors)) - { - if (handle.SupportsRandomAccess) - { - result = Interop.Sys.PReadV(handle, pinnedVectors, buffers.Count, fileOffset); - - if (result == -1 && ShouldFallBackToNonOffsetSyscall(Interop.Sys.GetLastErrorInfo())) - { - handle.SupportsRandomAccess = false; // Fall through to non-offset ReadV below. - } - } - - if (!handle.SupportsRandomAccess) - { - result = Interop.Sys.ReadV(handle, pinnedVectors, buffers.Count); - } - } - } - finally - { - foreach (MemoryHandle memoryHandle in handles) - { - memoryHandle.Dispose(); - } - } - - return FileStreamHelpers.CheckFileCall(result, handle.Path); - } + => handle.Read(fileOffset, buffers); internal static ValueTask ReadAtOffsetAsync(SafeFileHandle handle, Memory buffer, long fileOffset, CancellationToken cancellationToken, OSFileStreamStrategy? strategy = null) - => handle.GetThreadPoolValueTaskSource().QueueRead(buffer, fileOffset, cancellationToken, strategy); + => handle.ReadAsync(fileOffset, buffer, cancellationToken, strategy); private static ValueTask ReadScatterAtOffsetAsync(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset, CancellationToken cancellationToken) - => handle.GetThreadPoolValueTaskSource().QueueReadScatter(buffers, fileOffset, cancellationToken); + => handle.ReadAsync(fileOffset, buffers, cancellationToken); internal static unsafe void WriteAtOffset(SafeFileHandle handle, ReadOnlySpan buffer, long fileOffset) - { - while (!buffer.IsEmpty) - { - fixed (byte* bufPtr = &MemoryMarshal.GetReference(buffer)) - { - // The Windows implementation uses WriteFile, which ignores the offset if the handle - // isn't seekable. We do the same manually with PWrite vs Write, in order to enable - // the function to be used by FileStream for all the same situations. - int bytesToWrite = GetNumberOfBytesToWrite(buffer.Length); - int bytesWritten = -1; - if (handle.IsAsync) - { - bytesWritten = Interop.Sys.WriteToNonblocking(handle, bufPtr, bytesToWrite); - } - else if (handle.SupportsRandomAccess) - { - bytesWritten = Interop.Sys.PWrite(handle, bufPtr, bytesToWrite, fileOffset); - - if (bytesWritten == -1 && ShouldFallBackToNonOffsetSyscall(Interop.Sys.GetLastErrorInfo())) - { - handle.SupportsRandomAccess = false; // Fall through to non-offset Write below. - } - } - - if (!handle.IsAsync && !handle.SupportsRandomAccess) - { - bytesWritten = Interop.Sys.Write(handle, bufPtr, bytesToWrite); - } - - FileStreamHelpers.CheckFileCall(bytesWritten, handle.Path); - if (bytesWritten == buffer.Length) - { - break; - } - - // The write completed successfully but for fewer bytes than requested. - // We need to try again for the remainder. - buffer = buffer.Slice(bytesWritten); - fileOffset += bytesWritten; - } - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int GetNumberOfBytesToWrite(int byteCount) - { -#if DEBUG - // In debug only, to assist with testing, simulate writing fewer than the requested number of bytes. - if (byteCount > 1 && // ensure we don't turn the read into a zero-byte read - byteCount < 512) // avoid on larger buffers that might have a length used to meet an alignment requirement - { - byteCount /= 2; - } -#endif - return byteCount; - } + => handle.Write(fileOffset, buffer); internal static unsafe void WriteGatherAtOffset(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset) - { - int buffersCount = buffers.Count; - if (buffersCount == 0) - { - return; - } - - var handles = new MemoryHandle[buffersCount]; - Span vectors = buffersCount <= IovStackThreshold ? - stackalloc Interop.Sys.IOVector[IovStackThreshold].Slice(0, buffersCount) : - new Interop.Sys.IOVector[buffersCount]; - - try - { - long totalBytesToWrite = 0; - for (int i = 0; i < buffersCount; i++) - { - ReadOnlyMemory buffer = buffers[i]; - totalBytesToWrite += buffer.Length; - - MemoryHandle memoryHandle = buffer.Pin(); - vectors[i] = new Interop.Sys.IOVector { Base = (byte*)memoryHandle.Pointer, Count = (UIntPtr)buffer.Length }; - handles[i] = memoryHandle; - } - - int buffersOffset = 0; - while (totalBytesToWrite > 0) - { - long bytesWritten = -1; - Span left = vectors.Slice(buffersOffset); - fixed (Interop.Sys.IOVector* pinnedVectors = &MemoryMarshal.GetReference(left)) - { - if (handle.SupportsRandomAccess) - { - bytesWritten = Interop.Sys.PWriteV(handle, pinnedVectors, left.Length, fileOffset); - - if (bytesWritten == -1 && ShouldFallBackToNonOffsetSyscall(Interop.Sys.GetLastErrorInfo())) - { - handle.SupportsRandomAccess = false; // Fall through to non-offset WriteV below. - } - } - - if (!handle.SupportsRandomAccess) - { - bytesWritten = Interop.Sys.WriteV(handle, pinnedVectors, left.Length); - } - } - - FileStreamHelpers.CheckFileCall(bytesWritten, handle.Path); - if (bytesWritten == totalBytesToWrite) - { - break; - } - - // The write completed successfully but for fewer bytes than requested. - // We need to perform next write where the previous one has finished. - fileOffset += bytesWritten; - totalBytesToWrite -= bytesWritten; - // We need to try again for the remainder. - while (buffersOffset < buffersCount && bytesWritten > 0) - { - int n = (int)vectors[buffersOffset].Count; - if (n <= bytesWritten) - { - bytesWritten -= n; - buffersOffset++; - } - else - { - // A partial read: the vector needs to point to the new offset. - // But that offset needs to be relative to the previous attempt. - // Example: we have a single buffer with 30 bytes and the first read returned 10. - // The next read should try to read the remaining 20 bytes, but in case it also reads just 10, - // the third attempt should read last 10 bytes (not 20 again). - Interop.Sys.IOVector current = vectors[buffersOffset]; - vectors[buffersOffset] = new Interop.Sys.IOVector - { - Base = current.Base + (int)(bytesWritten), - Count = current.Count - (UIntPtr)(bytesWritten) - }; - break; - } - } - } - } - finally - { - foreach (MemoryHandle memoryHandle in handles) - { - memoryHandle.Dispose(); - } - } - } + => handle.Write(fileOffset, buffers); internal static ValueTask WriteAtOffsetAsync(SafeFileHandle handle, ReadOnlyMemory buffer, long fileOffset, CancellationToken cancellationToken, OSFileStreamStrategy? strategy = null) - => handle.GetThreadPoolValueTaskSource().QueueWrite(buffer, fileOffset, cancellationToken, strategy); + => handle.WriteAsync(fileOffset, buffer, cancellationToken, strategy); private static ValueTask WriteGatherAtOffsetAsync(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset, CancellationToken cancellationToken) - => handle.GetThreadPoolValueTaskSource().QueueWriteGather(buffers, fileOffset, cancellationToken); - - /// - /// Checks the last error after a failed pread/pwrite/preadv/pwritev call - /// and returns true if the error indicates a non-seekable file type (ENXIO or ESPIPE). - /// - private static bool ShouldFallBackToNonOffsetSyscall(Interop.ErrorInfo lastError) - => lastError.Error is Interop.Error.ENXIO or Interop.Error.ESPIPE; + => handle.WriteAsync(fileOffset, buffers, cancellationToken); } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/PollOperationResults.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/PollOperationResults.cs new file mode 100644 index 00000000000000..f882af1fd02c06 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/PollOperationResults.cs @@ -0,0 +1,47 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace System.Threading +{ + /// + /// Describes the result of an asynchronous I/O operation. + /// + public enum PollOperationAsyncResult + { + /// The operation is pending. + Pending = 0, + + /// The operation completed successfully. + Completed = 1, + /// The operation was aborted. + Aborted = 2, + } + + /// + /// Describes the result of a completed asynchronous I/O operation. + /// + public enum PollOperationOnCompletedResult + { + /// The operation completed successfully. + Completed = 1, + /// The operation was aborted (handle closed). + Aborted = 2, + + /// The operation was canceled (CancellationToken triggered). + Canceled = 3, + } + + /// + /// Describes the result of a synchronous I/O operation. + /// + public enum PollOperationSyncResult + { + /// The operation completed successfully. + Completed = 1, + /// The operation was aborted (handle closed). + Aborted = 2, + + /// The operation timed out. + TimedOut = 4, + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/PollThread.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/PollThread.cs new file mode 100644 index 00000000000000..f9b090a994aac1 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/PollThread.cs @@ -0,0 +1,313 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace System.Threading +{ + internal sealed unsafe class PollThread : IThreadPoolWorkItem + { + private const int EventBufferCount = +#if DEBUG + 32; +#else + 1024; +#endif + + private static int GetEngineCount() + { + if (uint.TryParse(Environment.GetEnvironmentVariable("DOTNET_SYSTEM_NET_SOCKETS_THREAD_COUNT"), out uint count)) + { + return (int)count; + } + + bool inlineSocketCompletionsEnabled = Environment.GetEnvironmentVariable("DOTNET_SYSTEM_NET_SOCKETS_INLINE_COMPLETIONS") == "1"; + if (inlineSocketCompletionsEnabled) + { + return Environment.ProcessorCount; + } + + Architecture architecture = RuntimeInformation.ProcessArchitecture; + int coresPerEngine = architecture == Architecture.Arm64 || architecture == Architecture.Arm + ? 8 + : 30; + + return Math.Max(1, (int)Math.Round(Environment.ProcessorCount / (double)coresPerEngine)); + } + + private static readonly PollThread[] s_engines = CreateEngines(); + private static int s_allocateFromEngine = -1; + + private static PollThread[] CreateEngines() + { + int engineCount = GetEngineCount(); + + var engines = new PollThread[engineCount]; + + for (int i = 0; i < engineCount; i++) + { + engines[i] = new PollThread(); + } + + return engines; + } + + /// + /// Each is assigned an index into this table while registered with a . + /// The index is used as the to quickly map events to s. + /// It is also stored in so that we can efficiently remove it when unregistering. + /// + private static PollableHandle?[] s_registeredHandles = []; + private static readonly Queue s_registeredHandlesFreeList = []; + + private readonly IntPtr _port; + private readonly Interop.Sys.HandleEvent* _buffer; + + // + // Queue of events generated by EventLoop() that would be processed by the thread pool + // + private readonly ConcurrentQueue _eventQueue = new ConcurrentQueue(); + + private int _hasOutstandingThreadRequest; + + // + // Registers a PollableHandle with a PollThread. + // + public static bool TryRegister(IntPtr socketHandle, PollableHandle pollHandle, out Interop.Error error) + { + int engineIndex = Math.Abs(Interlocked.Increment(ref s_allocateFromEngine) % s_engines.Length); + PollThread nextEngine = s_engines[engineIndex]; + return nextEngine.TryRegisterCore(socketHandle, pollHandle, out error); + } + + private bool TryRegisterCore(IntPtr socketHandle, PollableHandle pollHandle, out Interop.Error error) + { + Debug.Assert(pollHandle.ContextIndex == -1); + + lock (s_registeredHandlesFreeList) + { + if (!s_registeredHandlesFreeList.TryDequeue(out int index)) + { + int previousLength = s_registeredHandles.Length; + int newLength = Math.Max(4, 2 * previousLength); + + Array.Resize(ref s_registeredHandles, newLength); + + for (int i = previousLength + 1; i < newLength; i++) + { + s_registeredHandlesFreeList.Enqueue(i); + } + + index = previousLength; + } + + Debug.Assert(s_registeredHandles[index] is null); + + s_registeredHandles[index] = pollHandle; + pollHandle.ContextIndex = index; + } + + error = Interop.Sys.TryChangeHandleEventRegistration(_port, socketHandle, Interop.Sys.HandleEvents.None, + Interop.Sys.HandleEvents.Read | Interop.Sys.HandleEvents.Write, pollHandle.ContextIndex); + if (error == Interop.Error.SUCCESS) + { + return true; + } + + Unregister(pollHandle); + return false; + } + + public static void Unregister(PollableHandle pollHandle) + { + Debug.Assert(pollHandle.ContextIndex >= 0); + Debug.Assert(ReferenceEquals(s_registeredHandles[pollHandle.ContextIndex], pollHandle)); + + lock (s_registeredHandlesFreeList) + { + s_registeredHandles[pollHandle.ContextIndex] = null; + s_registeredHandlesFreeList.Enqueue(pollHandle.ContextIndex); + } + + pollHandle.ContextIndex = -1; + } + + private PollThread() + { + _port = (IntPtr)(-1); + try + { + // + // Create the event port and buffer + // + Interop.Error err; + fixed (IntPtr* portPtr = &_port) + { + err = Interop.Sys.CreateHandleEventPort(portPtr); + if (err != Interop.Error.SUCCESS) + { + throw new InvalidOperationException($"Unexpected error: {err}"); + } + } + + fixed (Interop.Sys.HandleEvent** bufferPtr = &_buffer) + { + err = Interop.Sys.CreateHandleEventBuffer(EventBufferCount, bufferPtr); + if (err != Interop.Error.SUCCESS) + { + throw new InvalidOperationException($"Unexpected error: {err}"); + } + } + + var thread = new Thread(static s => ((PollThread)s!).EventLoop()) + { + IsBackground = true, + Name = ".NET I/O Events" + }; + thread.UnsafeStart(this); + } + catch + { + FreeNativeResources(); + throw; + } + } + + private void EventLoop() + { + try + { + PollEventHandler handler = new PollEventHandler(this); + while (true) + { + int numEvents = EventBufferCount; + Interop.Error err = Interop.Sys.WaitForHandleEvents(_port, handler.Buffer, &numEvents); + if (err != Interop.Error.SUCCESS) + { + throw new InvalidOperationException($"Unexpected error: {err}"); + } + + // The native shim is responsible for ensuring this condition. + Debug.Assert(numEvents > 0, $"Unexpected numEvents: {numEvents}"); + + if (handler.HandleEvents(numEvents)) + { + EnsureWorkerScheduled(); + } + } + } + catch (Exception e) + { + Environment.FailFast("Exception thrown from PollThread event loop: " + e.ToString(), e); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void EnsureWorkerScheduled() + { + if (Interlocked.Exchange(ref _hasOutstandingThreadRequest, 1) == 0) + { + ThreadPool.UnsafeQueueUserWorkItem(this, preferLocal: false); + } + } + + void IThreadPoolWorkItem.Execute() + { + Debug.Assert(_hasOutstandingThreadRequest == 1); + _hasOutstandingThreadRequest = 0; + + Interlocked.MemoryBarrier(); + + ConcurrentQueue eventQueue = _eventQueue; + if (!eventQueue.TryDequeue(out PollIOEvent ev)) + { + return; + } + + if (!eventQueue.IsEmpty) + { + EnsureWorkerScheduled(); + } + + int startTimeMs = Environment.TickCount; + do + { + ev.PollHandle.HandleEventsOnThreadPool(ev.Events); + } while (Environment.TickCount - startTimeMs < 15 && eventQueue.TryDequeue(out ev)); + } + + private void FreeNativeResources() + { + if (_buffer != null) + { + Interop.Sys.FreeHandleEventBuffer(_buffer); + } + if (_port != (IntPtr)(-1)) + { + Interop.Sys.CloseHandleEventPort(_port); + } + } + + // See discussion: https://github.com/dotnet/runtime/issues/37064 + private readonly struct PollEventHandler + { + public Interop.Sys.HandleEvent* Buffer { get; } + + private readonly ConcurrentQueue _eventQueue; + + public PollEventHandler(PollThread engine) + { + Buffer = engine._buffer; + _eventQueue = engine._eventQueue; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public bool HandleEvents(int numEvents) + { + bool enqueuedEvent = false; + foreach (var handleEvent in new ReadOnlySpan(Buffer, numEvents)) + { + Debug.Assert((uint)handleEvent.Data < (uint)s_registeredHandles.Length); + + PollableHandle? pollHandle = s_registeredHandles[(uint)handleEvent.Data]; + + if (pollHandle is not null) + { + if (pollHandle.InlineCompletions) + { + pollHandle.HandleEventsInline(handleEvent.Events); + } + else + { + Interop.Sys.HandleEvents events = pollHandle.ProcessInlineSpeculatively(handleEvent.Events); + + if (events != Interop.Sys.HandleEvents.None) + { + _eventQueue.Enqueue(new PollIOEvent(pollHandle, events)); + enqueuedEvent = true; + } + } + } + } + + return enqueuedEvent; + } + } + + private readonly struct PollIOEvent + { + public PollableHandle PollHandle { get; } + public Interop.Sys.HandleEvents Events { get; } + + public PollIOEvent(PollableHandle pollHandle, Interop.Sys.HandleEvents events) + { + PollHandle = pollHandle; + Events = events; + } + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/PollTriggeredOperation.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/PollTriggeredOperation.PlatformNotSupported.cs new file mode 100644 index 00000000000000..853b7ffd746b39 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/PollTriggeredOperation.PlatformNotSupported.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.InteropServices; + +namespace System.Threading +{ + public abstract class PollTriggeredOperation : IThreadPoolWorkItem + { + void IThreadPoolWorkItem.Execute() { } + + protected virtual void ExecuteThreadPoolWorkItem() { } + + protected internal abstract bool TryCompleteOperation(SafeHandle handle); + protected internal abstract void OnCompleted(PollOperationOnCompletedResult result); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/PollTriggeredOperation.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/PollTriggeredOperation.cs new file mode 100644 index 00000000000000..4660e0ecfa4d08 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/PollTriggeredOperation.cs @@ -0,0 +1,237 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Runtime.InteropServices; + +namespace System.Threading +{ + /// + /// Represents an I/O operation that is triggered by an OS readiness notification. + /// + public abstract class PollTriggeredOperation : IThreadPoolWorkItem + { + private enum State + { + Waiting = 0, + Running, + RunningWithPendingCancellation, + RunningWithPendingAbort, + Complete, + Canceled, + Aborted + } + + private volatile State _state; + + // Node fields — set by Init before enqueuing. + internal PollTriggeredOperation? Next; + internal CancellationTokenRegistration CancellationRegistration; + internal bool IsReadOperation; + internal ManualResetEventSlim? SyncEvent; + private PollableHandle? _owner; + + /// + /// Whether this node should be treated as inline by the poll thread. + /// Sync nodes are always inline (they just signal the ManualResetEvent). + /// Async nodes are inline when the owner's is set. + /// + internal bool CanInline => SyncEvent != null || _owner!.InlineCompletions; + + internal void Init(PollableHandle owner, bool isReadOperation) + { + Debug.Assert(_state == State.Waiting, $"Unexpected operation state: {_state}"); + _owner = owner; + IsReadOperation = isReadOperation; + } + + /// + /// Clears fields that hold references to external objects so the operation + /// can be safely returned to a pool without rooting the queue or the handle. + /// + internal void ResetForReuse() + { + _state = State.Waiting; + Next = null; + _owner = null; + // CancellationRegistration is already disposed. + } + + /// + /// Dispatches readiness for this node. + /// Sync nodes signal the waiting thread; inline nodes process on the poll thread; + /// other nodes get scheduled on the thread pool. + /// + internal void DispatchProcess() + { + if (SyncEvent != null) + { + // Sync operation. Signal waiting thread to try the I/O. + SyncEvent.Set(); + } + else if (CanInline) + { + if (IsReadOperation) + _owner!.ProcessRead(this); + else + _owner!.ProcessWrite(this); + } + else + { + ProcessOnThreadPool(); + } + } + + internal void ProcessOnThreadPool() + { + ThreadPool.UnsafeQueueUserWorkItem(this, preferLocal: false); + } + + void IThreadPoolWorkItem.Execute() + => ExecuteThreadPoolWorkItem(); + + protected virtual void ExecuteThreadPoolWorkItem() + { + if (IsReadOperation) + _owner!.ProcessRead(this); + else + _owner!.ProcessWrite(this); + } + + /// + /// Aborts this node in the queue (called by Dispose). + /// For sync nodes, signals the ManualResetEvent so the waiting thread wakes up + /// and sees QueueState.Stopped in TryCompleteQueued. + /// For async nodes, goes through the state machine to cancel and invoke the callback. + /// + internal bool Abort() + { + if (SyncEvent != null) + { + // Wake up the thread. It will see the QueueState.Stopped in ProcessQueuedOperation. + SyncEvent.Set(); + return true; + } + return TryCancel(abort: true); + } + + internal PollOperationAsyncResult TryExecute() + { + // Set state to Running, unless we've been canceled. + State oldState = Interlocked.CompareExchange(ref _state, State.Running, State.Waiting); + if (oldState is State.Canceled or State.Aborted) + { + return PollOperationAsyncResult.Aborted; + } + + Debug.Assert(oldState == State.Waiting, $"Unexpected operation state: {oldState}"); + + // Try to perform the I/O. + if (TryCompleteOperation(_owner!.Handle)) + { + Debug.Assert(_state is State.Running or State.RunningWithPendingCancellation or State.RunningWithPendingAbort, $"Unexpected operation state: {_state}"); + + _state = State.Complete; + + return PollOperationAsyncResult.Completed; + } + + // Set state back to Waiting, unless we were canceled/aborted, + // in which case we have to process that now. + State newState; + while (true) + { + State state = _state; + Debug.Assert(state is State.Running or State.RunningWithPendingCancellation or State.RunningWithPendingAbort, $"Unexpected operation state: {state}"); + + newState = state switch + { + State.Running => State.Waiting, + State.RunningWithPendingCancellation => State.Canceled, + State.RunningWithPendingAbort => State.Aborted, + _ => state // unreachable + }; + if (state == Interlocked.CompareExchange(ref _state, newState, state)) + { + break; + } + + // Race to update the state. Loop and try again. + } + + if (newState is State.Canceled or State.Aborted) + { + NotifyOnThreadPool(); + return PollOperationAsyncResult.Aborted; + } + + return PollOperationAsyncResult.Pending; + } + + /// + /// Attempts to cancel or abort the operation. + /// Races with via the state machine and invokes the + /// continuation callback on success. + /// + /// true to abort (handle closed); false to cancel (CancellationToken). + /// true if this call performed the cancellation; false if the operation already completed or is mid-execution. + internal bool TryCancel(bool abort) + { + State intendedState = abort ? State.Aborted : State.Canceled; + + State newState; + while (true) + { + State state = _state; + if (state is State.Complete or State.Canceled or State.Aborted + or State.RunningWithPendingCancellation or State.RunningWithPendingAbort) + { + return false; + } + + newState = state == State.Waiting + ? intendedState + : (abort ? State.RunningWithPendingAbort : State.RunningWithPendingCancellation); + if (state == Interlocked.CompareExchange(ref _state, newState, state)) + { + break; + } + + // Race to update the state. Loop and try again. + } + + if (newState is State.RunningWithPendingCancellation or State.RunningWithPendingAbort) + { + // TryExecute will either succeed, or it will see the pending cancellation and deal with it. + return false; + } + + NotifyOnThreadPool(); + + // Note, we leave the operation in the OperationQueue. + // When we get around to processing it, we'll see it's cancelled and skip it. + return true; + } + + private void NotifyOnThreadPool() + { + ThreadPool.UnsafeQueueUserWorkItem(static s => + { + var op = (PollTriggeredOperation)s!; + var result = op._state == State.Canceled ? PollOperationOnCompletedResult.Canceled : PollOperationOnCompletedResult.Aborted; + op.OnCompleted(result); + }, this); + } + + /// + /// Performs the operation. Returns if it completed, returns if it is pending on the handle to become ready again. + /// + protected internal abstract bool TryCompleteOperation(SafeHandle handle); + + /// + /// Called when the operation completes asynchronously. + /// + /// The operation may be reused for another operation when is . + protected internal abstract void OnCompleted(PollOperationOnCompletedResult result); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/PollableHandle.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/PollableHandle.PlatformNotSupported.cs new file mode 100644 index 00000000000000..028f9ca89109ec --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/PollableHandle.PlatformNotSupported.cs @@ -0,0 +1,51 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.InteropServices; + +namespace System.Threading +{ + public sealed partial class PollableHandle : IDisposable + { + private PollableHandle() { } + + public bool InlineCompletions { get; set; } + + public static PollableHandle Create(SafeHandle handle, ref PollableHandle? field) + { + throw new PlatformNotSupportedException(); + } + + public bool IsReadReady(out int observedSequenceNumber) { throw new PlatformNotSupportedException(); } + public bool IsWriteReady(out int observedSequenceNumber) { throw new PlatformNotSupportedException(); } + + public PollOperationAsyncResult ReadAsync(PollTriggeredOperation operation, int observedSequenceNumber, CancellationToken cancellationToken) + { + throw new PlatformNotSupportedException(); + } + + public PollOperationAsyncResult WriteAsync(PollTriggeredOperation operation, int observedSequenceNumber, CancellationToken cancellationToken) + { + throw new PlatformNotSupportedException(); + } + + public PollOperationSyncResult ReadSync(PollTriggeredOperation operation, int observedSequenceNumber, int timeout) + { + throw new PlatformNotSupportedException(); + } + + public PollOperationSyncResult WriteSync(PollTriggeredOperation operation, int observedSequenceNumber, int timeout) + { + throw new PlatformNotSupportedException(); + } + + public bool AbortAndDispose() + { + throw new PlatformNotSupportedException(); + } + + public void Dispose() + { + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/PollableHandle.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/PollableHandle.Unix.cs new file mode 100644 index 00000000000000..d2902435885590 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/PollableHandle.Unix.cs @@ -0,0 +1,85 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Runtime.InteropServices; + +namespace System.Threading +{ + public sealed partial class PollableHandle + { + internal bool IsRegistered + { + get; + private set; + } + + private bool TryRegisterWithPollThread(out Interop.Error error) + { + // Multiple callers may try to register concurrently. + using (_writeQueue.Lock()) // Lock is used for IsDisposed. + { + if (IsDisposed || IsRegistered) + { + // Already registered/disposed. + error = Interop.Error.SUCCESS; + return true; + } + + bool addedRef = false; + try + { + Handle.DangerousAddRef(ref addedRef); + IntPtr rawHandle = Handle.DangerousGetHandle(); + if (PollThread.TryRegister(rawHandle, this, out error)) + { + IsRegistered = true; + return true; // Succesfully registered. + } + else + { + return false; // Registration failed. + } + } + finally + { + if (addedRef) + { + Handle.DangerousRelease(); + } + } + } + } + + private bool Register(PollTriggeredOperation node) + { + if (TryRegisterWithPollThread(out Interop.Error error)) + { + // Registration was a success. Operation will be triggered by poll thread. + return true; + } + + // macOS: kevent returns EPIPE when adding pipe fd for which the other end is closed. + if (error == Interop.Error.EPIPE && node.TryCompleteOperation(Handle)) + { + return false; + } + + if (error == Interop.Error.ENOMEM || error == Interop.Error.ENOSPC) + { + throw new OutOfMemoryException(); + } + + throw new InvalidOperationException($"Unexpected error: {error}"); + } + + private void Unregister() + { + Debug.Assert(IsDisposed); + if (IsRegistered) + { + PollThread.Unregister(this); + } + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/PollableHandle.Wasi.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/PollableHandle.Wasi.cs new file mode 100644 index 00000000000000..26ebee9381f65e --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/PollableHandle.Wasi.cs @@ -0,0 +1,385 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.InteropServices; +using static Interop; +using static Interop.Sys; + +namespace System.Threading +{ + // types here are clone of private implementation details of wasi-libc + // we could get rid of it when https://github.com/WebAssembly/wasi-libc/issues/542 is resolved + // or after WASIp3 promises are implemented, whatever comes first + + [StructLayout(LayoutKind.Sequential)] + internal struct tcp_own_tcp_socket_t + { + public int handle; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct udp_own_udp_socket_t + { + public int handle; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct udp_own_incoming_datagram_stream_t + { + public int handle; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct udp_own_outgoing_datagram_stream_t + { + public int handle; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct streams_own_input_stream_t + { + public int handle; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct poll_own_pollable_t + { + public int handle; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct streams_own_output_stream_t + { + public int handle; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct tcp_socket_state_unbound_t + { + public int dummy; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct tcp_socket_state_bound_t + { + public int dummy; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct tcp_socket_state_connecting_t + { + public int dummy; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct tcp_socket_state_listening_t + { + public int dummy; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct tcp_socket_state_connected_t + { + public streams_own_input_stream_t input; + public poll_own_pollable_t input_pollable; + public streams_own_output_stream_t output; + public poll_own_pollable_t output_pollable; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct tcp_socket_state_connect_failed_t + { + public byte error_code; + } + + internal enum tcp_socket_state_tag + { + TCP_SOCKET_STATE_UNBOUND, + TCP_SOCKET_STATE_BOUND, + TCP_SOCKET_STATE_CONNECTING, + TCP_SOCKET_STATE_CONNECTED, + TCP_SOCKET_STATE_CONNECT_FAILED, + TCP_SOCKET_STATE_LISTENING, + } + + [StructLayout(LayoutKind.Explicit)] + internal struct tcp_socket_state_union + { + [FieldOffset(0)] public tcp_socket_state_unbound_t unbound; + [FieldOffset(0)] public tcp_socket_state_bound_t bound; + [FieldOffset(0)] public tcp_socket_state_connecting_t connecting; + [FieldOffset(0)] public tcp_socket_state_connected_t connected; + [FieldOffset(0)] public tcp_socket_state_connect_failed_t connect_failed; + [FieldOffset(0)] public tcp_socket_state_listening_t listening; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct tcp_socket_state_t + { + public tcp_socket_state_tag tag; + public tcp_socket_state_union state; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct tcp_socket_t + { + public tcp_own_tcp_socket_t socket; + public poll_own_pollable_t socket_pollable; + public bool blocking; + public bool fake_nodelay; + public bool fake_reuseaddr; + public byte family; + public tcp_socket_state_t state; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct udp_socket_streams_t + { + public udp_own_incoming_datagram_stream_t incoming; + public poll_own_pollable_t incoming_pollable; + public udp_own_outgoing_datagram_stream_t outgoing; + public poll_own_pollable_t outgoing_pollable; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct udp_socket_state_unbound_t + { + public int dummy; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct udp_socket_state_bound_nostreams_t + { + public int dummy; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct udp_socket_state_bound_streaming_t + { + public udp_socket_streams_t streams; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct udp_socket_state_connected_t + { + public udp_socket_streams_t streams; + } + + internal enum udp_socket_state_tag + { + UDP_SOCKET_STATE_UNBOUND, + UDP_SOCKET_STATE_BOUND_NOSTREAMS, + UDP_SOCKET_STATE_BOUND_STREAMING, + UDP_SOCKET_STATE_CONNECTED, + } + + [StructLayout(LayoutKind.Explicit)] + internal struct udp_socket_state_union + { + [FieldOffset(0)] public udp_socket_state_unbound_t unbound; + [FieldOffset(0)] public udp_socket_state_bound_nostreams_t bound_nostreams; + [FieldOffset(0)] public udp_socket_state_bound_streaming_t bound_streaming; + [FieldOffset(0)] public udp_socket_state_connected_t connected; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct udp_socket_state_t + { + public udp_socket_state_tag tag; + public udp_socket_state_union state; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct udp_socket_t + { + public udp_own_udp_socket_t socket; + public poll_own_pollable_t socket_pollable; + public bool blocking; + public byte family; + public udp_socket_state_t state; + } + + internal enum descriptor_table_entry_tag + { + DESCRIPTOR_TABLE_ENTRY_TCP_SOCKET, + DESCRIPTOR_TABLE_ENTRY_UDP_SOCKET, + } + + [StructLayout(LayoutKind.Explicit)] + internal struct descriptor_table_entry_union + { + [FieldOffset(0)] public tcp_socket_t tcp_socket; + [FieldOffset(0)] public udp_socket_t udp_socket; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct descriptor_table_entry_t + { + public descriptor_table_entry_tag tag; + public descriptor_table_entry_union entry; + } + + public sealed partial class PollableHandle + { + internal bool IsRegistered + { + get; + private set; + } + + private CancellationTokenSource? _unregisterPollHook; + + private bool TryRegisterWithPollThread(out Interop.Error error) + { + // Multiple callers may try to register concurrently. + using (_writeQueue.Lock()) // Lock is used for IsDisposed. + { + if (IsDisposed || IsRegistered) + { + // Already registered/disposed. + error = Interop.Error.SUCCESS; + return true; + } + + nint entryPtr = default; + IntPtr socketHandle = Handle.DangerousGetHandle(); + unsafe { error = Interop.Sys.GetWasiSocketDescriptor(socketHandle, &entryPtr); } + if (error != Interop.Error.SUCCESS) + { + return false; // Registration failed. + } + + _unregisterPollHook = new CancellationTokenSource(); + Thread.RegisterWasiPollHook(this, BeforePollHook, HandlePollEvent, _unregisterPollHook.Token); + + IsRegistered = true; + return true; // Succesfully registered. + } + } + + private bool Register(PollTriggeredOperation _) + { + if (TryRegisterWithPollThread(out Interop.Error error)) + { + // Registration was a success. Operation will be triggered by poll thread. + return true; + } + + if (error == Interop.Error.ENOMEM || error == Interop.Error.ENOSPC) + { + throw new OutOfMemoryException(); + } + + throw new InvalidOperationException($"Unexpected error: {error}"); + } + + private void Unregister() + { + Debug.Assert(IsDisposed); + if (IsRegistered) + { + _unregisterPollHook?.Cancel(); + } + } + + private static unsafe IList BeforePollHook(object? state) + { + var pollHandle = (PollableHandle)state!; + if (pollHandle.Handle.IsClosed) + { + return []; + } + + List pollableHandles = new(); + nint entryPtr = default; + IntPtr socketHandle = pollHandle.Handle.DangerousGetHandle(); + var error = Interop.Sys.GetWasiSocketDescriptor(socketHandle, &entryPtr); + if (error != Interop.Error.SUCCESS) + { + Environment.FailFast("Can't resolve libc descriptor for socket handle " + socketHandle); + } + + var entry = (descriptor_table_entry_t*)entryPtr; + switch (entry->tag) + { + case descriptor_table_entry_tag.DESCRIPTOR_TABLE_ENTRY_TCP_SOCKET: + { + tcp_socket_t* socket = &(entry->entry.tcp_socket); + switch (socket->state.tag) + { + case tcp_socket_state_tag.TCP_SOCKET_STATE_CONNECTING: + case tcp_socket_state_tag.TCP_SOCKET_STATE_LISTENING: + pollableHandles.Add(socket->socket_pollable.handle); + break; + case tcp_socket_state_tag.TCP_SOCKET_STATE_CONNECTED: + pollableHandles.Add(socket->state.state.connected.input_pollable.handle); + pollableHandles.Add(socket->state.state.connected.output_pollable.handle); + break; + case tcp_socket_state_tag.TCP_SOCKET_STATE_CONNECT_FAILED: + pollHandle.HandleEventsInline(Sys.HandleEvents.Error); + break; + case tcp_socket_state_tag.TCP_SOCKET_STATE_UNBOUND: + case tcp_socket_state_tag.TCP_SOCKET_STATE_BOUND: + break; + default: + throw new NotImplementedException("TCP:" + socket->state.tag); + } + break; + } + case descriptor_table_entry_tag.DESCRIPTOR_TABLE_ENTRY_UDP_SOCKET: + { + udp_socket_t* socket = &(entry->entry.udp_socket); + switch (socket->state.tag) + { + case udp_socket_state_tag.UDP_SOCKET_STATE_UNBOUND: + case udp_socket_state_tag.UDP_SOCKET_STATE_BOUND_NOSTREAMS: + pollHandle.HandleEventsInline(Sys.HandleEvents.Read | Sys.HandleEvents.Write); + break; + case udp_socket_state_tag.UDP_SOCKET_STATE_BOUND_STREAMING: + case udp_socket_state_tag.UDP_SOCKET_STATE_CONNECTED: + { + udp_socket_streams_t* streams; + if (socket->state.tag == udp_socket_state_tag.UDP_SOCKET_STATE_BOUND_STREAMING) + { + streams = &(socket->state.state.bound_streaming.streams); + } + else + { + streams = &(socket->state.state.connected.streams); + } + pollableHandles.Add(streams->incoming_pollable.handle); + pollableHandles.Add(streams->outgoing_pollable.handle); + break; + } + + default: + throw new NotImplementedException("UDP" + socket->state.tag); + } + break; + } + default: + throw new NotImplementedException("TYPE" + entry->tag); + } + return pollableHandles; + } + + private static void HandlePollEvent(object? state) + { + PollableHandle pollHandle = (PollableHandle)state!; + try + { + using (ExecutionContext.SuppressFlow()) + { + pollHandle.HandleEventsInline(Sys.HandleEvents.Write | Sys.HandleEvents.Read); + } + } + catch (Exception e) + { + Environment.FailFast("Exception thrown from PollableHandle event handler: " + e.ToString(), e); + } + } + + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/PollableHandle.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/PollableHandle.cs new file mode 100644 index 00000000000000..be1be4d30a7e4e --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/PollableHandle.cs @@ -0,0 +1,775 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace System.Threading +{ + /// + /// Enables implementing asynchronous and synchronous I/O operations triggered by OS readiness notifications (epoll/kqueue). + /// + public sealed partial class PollableHandle : IDisposable + { + private enum QueueResult + { + Pending = 0, + Completed = 1, + Aborted = 2 + } + + // In debug builds, this struct guards against: + // (1) Unexpected lock reentrancy, which should never happen + // (2) Deadlock, by setting a reasonably large timeout + private readonly struct LockToken : IDisposable + { + private readonly object _lockObject; + + public LockToken(object lockObject) + { + Debug.Assert(lockObject != null); + + _lockObject = lockObject; + + Debug.Assert(!Monitor.IsEntered(_lockObject)); + +#if DEBUG + bool success = Monitor.TryEnter(_lockObject, 10000); + Debug.Assert(success, "Timed out waiting for queue lock"); +#else + Monitor.Enter(_lockObject); +#endif + } + + public void Dispose() + { + Debug.Assert(Monitor.IsEntered(_lockObject)); + Monitor.Exit(_lockObject); + } + } + + private struct OperationQueue + { + // Quick overview: + // + // When attempting to perform an IO operation, the caller first checks IsReady, + // and if true, attempts to perform the operation itself. + // If this returns EWOULDBLOCK, or if the queue was not ready, then the operation + // is enqueued by calling StartAsyncOperation and the state becomes Waiting. + // When an epoll notification is received, we check if the state is Waiting, + // and if so, change the state to Processing and enqueue a workitem to the threadpool + // to try to perform the enqueued operations. + // If an operation is successfully performed, we remove it from the queue, + // enqueue another threadpool workitem to process the next item in the queue (if any), + // and call the user's completion callback. + // If we successfully process all enqueued operations, then the state becomes Ready; + // otherwise, the state becomes Waiting and we wait for another epoll notification. + + private enum QueueState : int + { + Ready = 0, // Indicates that data MAY be available on the handle. + // Queue must be empty. + Waiting = 1, // Indicates that data is definitely not available on the handle. + // Queue must not be empty. + Processing = 2, // Indicates that a thread pool item has been scheduled (and may + // be executing) to process the IO operations in the queue. + // Queue must not be empty. + Stopped = 3, // Indicates that the queue has been stopped because the + // instance has been disposed. + // Queue must be empty. + } + + // These fields define the queue state. + + private QueueState _state; + private bool _nextMayBeInline; + private int _sequenceNumber; + private PollTriggeredOperation? _tail; + + private object _queueLock; + + internal LockToken Lock() + => new LockToken(_queueLock); + + public bool NextMayBeInline + => _nextMayBeInline; + + public bool IsStopped => _state == QueueState.Stopped; + + public void Init() + { + Debug.Assert(_queueLock == null); + _queueLock = new object(); + + _state = QueueState.Ready; + _sequenceNumber = 0; + } + + // IsReady returns whether an operation can be executed immediately. + // observedSequenceNumber must be passed to StartAsyncOperation. + public bool IsReady(out int observedSequenceNumber) + { + Debug.Assert(sizeof(QueueState) == sizeof(int)); + QueueState state = (QueueState)Volatile.Read(ref Unsafe.As(ref _state)); + observedSequenceNumber = Volatile.Read(ref _sequenceNumber); + + bool isReady = state == QueueState.Ready; + if (!isReady) + { + observedSequenceNumber--; + } + + return isReady; + } + + public PollOperationAsyncResult StartOperation(PollableHandle pollHandle, PollTriggeredOperation node, bool isReadOperation, int observedSequenceNumber, CancellationToken cancellationToken) + { + if (!pollHandle.IsRegistered && !pollHandle.Register(node)) + { + return PollOperationAsyncResult.Completed; + } + + while (true) + { + bool doAbort = false; + using (Lock()) + { + switch (_state) + { + case QueueState.Ready: + if (observedSequenceNumber != _sequenceNumber) + { + Debug.Assert(observedSequenceNumber - _sequenceNumber < 10000, "Very large sequence number increase???"); + observedSequenceNumber = _sequenceNumber; + break; + } + + _state = QueueState.Waiting; + goto case QueueState.Waiting; + + case QueueState.Waiting: + case QueueState.Processing: + node.Init(pollHandle, isReadOperation); + + if (_tail == null) + { + Debug.Assert(!_nextMayBeInline); + _nextMayBeInline = node.CanInline; + node.Next = node; + } + else + { + node.Next = _tail.Next; + _tail.Next = node; + } + + _tail = node; + + if (cancellationToken.CanBeCanceled) + { + node.CancellationRegistration = cancellationToken.UnsafeRegister( + static s => + { + var n = (PollTriggeredOperation)s!; + n.CancellationRegistration.Dispose(); + n.TryCancel(abort: false); + }, node); + } + + return PollOperationAsyncResult.Pending; + + case QueueState.Stopped: + Debug.Assert(_tail == null); + doAbort = true; + break; + + default: + Environment.FailFast("unexpected queue state"); + break; + } + } + + if (doAbort) + { + return PollOperationAsyncResult.Aborted; + } + + // Retry the operation. + // The node is not yet enqueued, so we can call TryCompleteOperation directly. + if (node.TryCompleteOperation(pollHandle.Handle)) + { + return PollOperationAsyncResult.Completed; + } + } + } + + public PollTriggeredOperation? ProcessInlineOrGet(bool inlineOnly = false) + { + PollTriggeredOperation node; + using (Lock()) + { + switch (_state) + { + case QueueState.Ready: + Debug.Assert(_tail == null, "State == Ready but queue is not empty!"); + _sequenceNumber++; + return null; + + case QueueState.Waiting: + Debug.Assert(_tail != null, "State == Waiting but queue is empty!"); + node = _tail.Next!; + if (inlineOnly && !node.CanInline) + { + return node; + } + + _state = QueueState.Processing; + break; + + case QueueState.Processing: + Debug.Assert(_tail != null, "State == Processing but queue is empty!"); + _sequenceNumber++; + return null; + + case QueueState.Stopped: + Debug.Assert(_tail == null); + return null; + + default: + Environment.FailFast("unexpected queue state"); + return null; + } + } + + if (node.CanInline) + { + // Inline operation. Process fully: TryExecute, remove from queue, + // Complete (if not canceled), and dispatch next. + node.DispatchProcess(); + return null; + } + else + { + // Async operation. The caller will figure out how to process the IO. + Debug.Assert(!inlineOnly); + return node; + } + } + + public QueueResult TryCompleteQueued(PollTriggeredOperation node) + { + int observedSequenceNumber; + using (Lock()) + { + if (_state == QueueState.Stopped) + { + Debug.Assert(_tail == null); + return QueueResult.Aborted; + } + else + { + Debug.Assert(_state == QueueState.Processing, $"_state={_state} while processing queue!"); + Debug.Assert(_tail != null, "Unexpected empty queue while processing I/O"); + Debug.Assert(node == _tail.Next, "Operation is not at head of queue???"); + observedSequenceNumber = _sequenceNumber; + } + } + + QueueResult result; + while (true) + { + PollOperationAsyncResult execResult = node.TryExecute(); + if (execResult != PollOperationAsyncResult.Pending) + { + result = execResult == PollOperationAsyncResult.Completed + ? QueueResult.Completed + : QueueResult.Aborted; + break; + } + + using (Lock()) + { + if (_state == QueueState.Stopped) + { + Debug.Assert(_tail == null); + return QueueResult.Aborted; + } + else + { + Debug.Assert(_state == QueueState.Processing, $"_state={_state} while processing queue!"); + + if (observedSequenceNumber != _sequenceNumber) + { + Debug.Assert(observedSequenceNumber - _sequenceNumber < 10000, "Very large sequence number increase???"); + observedSequenceNumber = _sequenceNumber; + } + else + { + _state = QueueState.Waiting; + return QueueResult.Pending; + } + } + } + } + + // Remove the node from the queue and see if there's more to process. + PollTriggeredOperation? nextNode = null; + using (Lock()) + { + if (_state == QueueState.Stopped) + { + Debug.Assert(_tail == null); + } + else + { + Debug.Assert(_state == QueueState.Processing, $"_state={_state} while processing queue!"); + Debug.Assert(_tail!.Next == node, "Queue modified while processing queue"); + + if (node == _tail) + { + _tail = null; + _nextMayBeInline = false; + _state = QueueState.Ready; + _sequenceNumber++; + } + else + { + nextNode = _tail.Next = node.Next; + _nextMayBeInline = nextNode!.CanInline; + } + } + } + + nextNode?.DispatchProcess(); + + Debug.Assert(result != QueueResult.Pending); + return result; + } + + // Removes a sync node from the queue on timeout and dispatches the next operation. + public void CancelSyncAndContinue(PollTriggeredOperation node) + { + PollTriggeredOperation? nextNode = null; + using (Lock()) + { + if (_state == QueueState.Stopped) + { + Debug.Assert(_tail == null); + return; + } + + Debug.Assert(_tail != null, "Unexpected empty queue in CancelSyncAndContinue"); + + if (_tail.Next == node) + { + // We're the head of the queue. + if (node == _tail) + { + // No more operations. + _tail = null; + _nextMayBeInline = false; + } + else + { + // Pop current operation and advance to next. + _tail.Next = node.Next; + _nextMayBeInline = node.Next!.CanInline; + } + + if (_state == QueueState.Processing) + { + // The queue has already handed off execution responsibility to us. + // We need to dispatch to the next op. + if (_tail == null) + { + _state = QueueState.Ready; + _sequenceNumber++; + } + else + { + nextNode = _tail.Next; + } + } + else if (_state == QueueState.Waiting) + { + if (_tail == null) + { + _state = QueueState.Ready; + _sequenceNumber++; + } + } + } + else + { + // We're not the head of the queue. + // Just find this op and remove it. + PollTriggeredOperation current = _tail.Next!; + while (current.Next != node) + { + current = current.Next!; + } + + current.Next = node.Next; + if (_tail == node) + { + _tail = current; + } + } + } + + nextNode?.DispatchProcess(); + } + + public bool StopAndAbort(out bool alreadyStopped) + { + bool aborted = false; + + using (Lock()) + { + alreadyStopped = _state == QueueState.Stopped; + if (alreadyStopped) + { + return false; + } + + _state = QueueState.Stopped; + + if (_tail != null) + { + PollTriggeredOperation node = _tail; + do + { + node.CancellationRegistration.Dispose(); + aborted |= node.Abort(); + node = node.Next!; + } while (node != _tail); + } + + _tail = null; + _nextMayBeInline = false; + } + + return aborted; + } + } + + private OperationQueue _readQueue; + private OperationQueue _writeQueue; + internal SafeHandle Handle { get; private set; } + internal int ContextIndex = -1; + + internal bool IsDisposed => _writeQueue.IsStopped; + + private PollableHandle(SafeHandle handle) + { + Handle = handle; + _readQueue.Init(); + _writeQueue.Init(); + } + + /// + /// Creates a for the specified handle. + /// + /// The OS handle to bind. + /// A reference to a field that will be atomically set to the new . + public static PollableHandle Create(SafeHandle handle, ref PollableHandle? field) + { + PollableHandle ph = new PollableHandle(handle); + PollableHandle? existing = Interlocked.CompareExchange(ref field, ph, null); + if (existing != null) + { + ph.Dispose(); + return existing; + } + return ph; + } + + /// + /// Gets or sets whether completion callbacks are processed inline or dispatched to the threadpool. + /// + public bool InlineCompletions { get; set; } + + /// + /// Returns a sequence number to pass to or . + /// If the method returns , the caller must first try executing the operation. + /// If the operation is pending, then the caller calls the ReadAsync/ReadSync method to execute the operation. + /// + /// A sequence number to pass to or . + /// if the handle is ready for reading; otherwise, . + /// Returns when the instance is disposed. + public bool IsReadReady(out int observedSequenceNumber) + => _readQueue.IsReady(out observedSequenceNumber); + + /// + /// Returns a sequence number to pass to or . + /// If the method returns , the caller must first try executing the operation. + /// If the operation is pending, then the caller calls the WriteAsync/WriteSync method to execute the operation. + /// + /// A sequence number to pass to or . + /// if the handle is ready for writing; otherwise, . + /// Returns when the instance is disposed. + public bool IsWriteReady(out int observedSequenceNumber) + => _writeQueue.IsReady(out observedSequenceNumber); + + /// + /// Executes the read operation asynchronously. If the handle is not ready, returns . + /// is then called when the operation completes/aborts/is cancelled. + /// + /// The operation to execute. + /// The sequence number obtained from . + /// A token to cancel the pending operation. + /// + /// The caller must first try to execute the operation after returns before calling this method. + /// The is used when the operation executes asynchronously. + /// If the token is cancelled, the operation completes with . + /// Returns when the instance is disposed. + /// The may be reused for another operation when the method returns . + /// + /// The result of the operation. + public PollOperationAsyncResult ReadAsync(PollTriggeredOperation operation, int observedSequenceNumber, CancellationToken cancellationToken) + => _readQueue.StartOperation(this, operation, isReadOperation: true, observedSequenceNumber, cancellationToken); + + /// + /// Executes the write operation asynchronously. If the handle is not ready, returns . + /// is then called when the operation completes/aborts/is cancelled. + /// + /// The operation to execute. + /// The sequence number obtained from . + /// A token to cancel the pending operation. + /// + /// The caller must first try to execute the operation after returns before calling this method. + /// The is used when the operation executes asynchronously. + /// If the token is cancelled, the operation completes with . + /// Returns when the instance is disposed. + /// The may be reused for another operation when the method returns . + /// + /// The result of the operation. + public PollOperationAsyncResult WriteAsync(PollTriggeredOperation operation, int observedSequenceNumber, CancellationToken cancellationToken) + => _writeQueue.StartOperation(this, operation, isReadOperation: false, observedSequenceNumber, cancellationToken); + + /// + /// Executes the read operation synchronously. Returns when the operation completes/aborts/times out. + /// + /// The operation to execute. + /// The sequence number obtained from . + /// Timeout in milliseconds. Use -1 for infinite timeout. + /// + /// The caller must first try to execute the operation after returns before calling this method. + /// Returns when the instance is disposed. + /// The may be reused for another operation when the method returns or . + /// + /// The result of the operation. + public PollOperationSyncResult ReadSync(PollTriggeredOperation operation, int observedSequenceNumber, int timeout) + => ExecuteSync(ref _readQueue, operation, isReadOperation: true, observedSequenceNumber, timeout); + + /// + /// Executes the write operation synchronously. Returns when the operation completes/aborts/times out. + /// + /// The operation to execute. + /// The sequence number obtained from . + /// Timeout in milliseconds. Use -1 for infinite timeout. + /// + /// The caller must first try to execute the operation after returns before calling this method. + /// Returns when the instance is disposed. + /// The may be reused for another operation when the method returns or . + /// + /// The result of the operation. + public PollOperationSyncResult WriteSync(PollTriggeredOperation operation, int observedSequenceNumber, int timeout) + => ExecuteSync(ref _writeQueue, operation, isReadOperation: false, observedSequenceNumber, timeout); + + private PollOperationSyncResult ExecuteSync(ref OperationQueue queue, PollTriggeredOperation operation, bool isReadOperation, int observedSequenceNumber, int timeout) + { + Debug.Assert(timeout == -1 || timeout > 0, $"Unexpected timeout: {timeout}"); + + PollOperationSyncResult result; + + using var syncEvent = new ManualResetEventSlim(false, 0); + operation.SyncEvent = syncEvent; + + PollOperationAsyncResult startResult = queue.StartOperation(this, operation, isReadOperation, observedSequenceNumber, default); + if (startResult != PollOperationAsyncResult.Pending) + { + // Completed synchronously or aborted. + result = (PollOperationSyncResult)startResult; + } + else + { + // Node is queued. Wait for readiness signals from the poll thread. + while (true) + { + long waitStart = Stopwatch.GetTimestamp(); + + if (!syncEvent.Wait(timeout)) + { + // Timeout expired. Remove node from queue. + queue.CancelSyncAndContinue(operation); + result = PollOperationSyncResult.TimedOut; + break; + } + + syncEvent.Reset(); + + QueueResult queueResult = queue.TryCompleteQueued(operation); + if (queueResult == QueueResult.Completed) + { + result = PollOperationSyncResult.Completed; + break; + } + + if (queueResult == QueueResult.Aborted) + { + result = PollOperationSyncResult.Aborted; + break; + } + + // EAGAIN (Pending). Adjust timeout and retry. + if (timeout > 0) + { + timeout -= (int)Stopwatch.GetElapsedTime(waitStart).TotalMilliseconds; + if (timeout <= 0) + { + queue.CancelSyncAndContinue(operation); + result = PollOperationSyncResult.TimedOut; + break; + } + } + } + } + + operation.SyncEvent = null; + + // The operation is in a poolable state, but we don't call ResetForReuse: + // the Aborted result documents that the operation must not be pooled. + // Most users don't have a use for pooling on Abort either. + if (result != PollOperationSyncResult.Aborted) + { + operation.ResetForReuse(); + } + + return result; + } + + internal void ProcessRead(PollTriggeredOperation node) + => Process(ref _readQueue, node); + internal void ProcessWrite(PollTriggeredOperation node) + => Process(ref _writeQueue, node); + + private static void Process(ref OperationQueue queue, PollTriggeredOperation node) + { + QueueResult result = queue.TryCompleteQueued(node); + + if (result != QueueResult.Pending) + { + node.CancellationRegistration.Dispose(); + + if (result == QueueResult.Completed) + { + node.ResetForReuse(); + node.OnCompleted(PollOperationOnCompletedResult.Completed); + } + } + } + + /// + /// Aborts all pending operations and Disposes the PollableHandle. + /// + /// if any operations were aborted; otherwise, . + public bool AbortAndDispose() + { + bool aborted = false; + + aborted |= _writeQueue.StopAndAbort(out bool alreadyStopped); + Debug.Assert(IsDisposed); // Due to stopping the write queue. + + // Only unregister once. + if (alreadyStopped) + { + return false; + } + + aborted |= _readQueue.StopAndAbort(out _); + + Unregister(); + + return aborted; + } + + // Called on the epoll thread, speculatively tries to process inline events and errors, + // and returns any remaining events that remain to be processed. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal Interop.Sys.HandleEvents ProcessInlineSpeculatively(Interop.Sys.HandleEvents events) + { + if ((events & Interop.Sys.HandleEvents.Error) != 0) + { + events ^= Interop.Sys.HandleEvents.Error; + events |= Interop.Sys.HandleEvents.Read | Interop.Sys.HandleEvents.Write; + } + + if ((events & Interop.Sys.HandleEvents.Read) != 0 && + _readQueue.NextMayBeInline && + _readQueue.ProcessInlineOrGet(inlineOnly: true) == null) + { + events ^= Interop.Sys.HandleEvents.Read; + } + + if ((events & Interop.Sys.HandleEvents.Write) != 0 && + _writeQueue.NextMayBeInline && + _writeQueue.ProcessInlineOrGet(inlineOnly: true) == null) + { + events ^= Interop.Sys.HandleEvents.Write; + } + + return events; + } + + internal void HandleEventsInline(Interop.Sys.HandleEvents events) + { + if ((events & Interop.Sys.HandleEvents.Error) != 0) + { + events ^= Interop.Sys.HandleEvents.Error; + events |= Interop.Sys.HandleEvents.Read | Interop.Sys.HandleEvents.Write; + } + + if ((events & Interop.Sys.HandleEvents.Read) != 0) + { + PollTriggeredOperation? receiveNode = _readQueue.ProcessInlineOrGet(); + if (receiveNode != null) ProcessRead(receiveNode); + } + + if ((events & Interop.Sys.HandleEvents.Write) != 0) + { + PollTriggeredOperation? sendNode = _writeQueue.ProcessInlineOrGet(); + if (sendNode != null) ProcessWrite(sendNode); + } + } + + internal void HandleEventsOnThreadPool(Interop.Sys.HandleEvents events) + { + Debug.Assert((events & Interop.Sys.HandleEvents.Error) == 0); + + PollTriggeredOperation? receiveNode = + (events & Interop.Sys.HandleEvents.Read) != 0 ? _readQueue.ProcessInlineOrGet() : null; + PollTriggeredOperation? sendNode = + (events & Interop.Sys.HandleEvents.Write) != 0 ? _writeQueue.ProcessInlineOrGet() : null; + + if (sendNode == null) + { + if (receiveNode != null) ProcessRead(receiveNode); + } + else + { + receiveNode?.ProcessOnThreadPool(); + ProcessWrite(sendNode); + } + } + + /// + /// Calls . + /// + public void Dispose() + { + AbortAndDispose(); + } + } +} diff --git a/src/libraries/System.Runtime/tests/System.IO.FileSystem.Tests/FileStream/FileStreamConformanceTests.cs b/src/libraries/System.Runtime/tests/System.IO.FileSystem.Tests/FileStream/FileStreamConformanceTests.cs index f02846332251d0..716ae2f0ae7673 100644 --- a/src/libraries/System.Runtime/tests/System.IO.FileSystem.Tests/FileStream/FileStreamConformanceTests.cs +++ b/src/libraries/System.Runtime/tests/System.IO.FileSystem.Tests/FileStream/FileStreamConformanceTests.cs @@ -241,7 +241,7 @@ protected override Task CreateConnectedStreamsAsync() protected override Type UnsupportedConcurrentExceptionType => null; protected override bool UsableAfterCanceledReads => false; - protected override bool FullyCancelableOperations => OperatingSystem.IsWindows(); + protected override bool FullyCancelableOperations => true; protected override bool BlocksOnZeroByteReads => OperatingSystem.IsWindows(); protected override bool SupportsConcurrentBidirectionalUse => false; } @@ -308,7 +308,7 @@ protected override async Task CreateConnectedStreamsAsync() protected override Type UnsupportedConcurrentExceptionType => null; protected override bool UsableAfterCanceledReads => false; - protected override bool FullyCancelableOperations => OperatingSystem.IsWindows(); + protected override bool FullyCancelableOperations => true; protected override bool BlocksOnZeroByteReads => OperatingSystem.IsWindows(); protected override bool SupportsConcurrentBidirectionalUse => false; } diff --git a/src/libraries/System.Runtime/tests/System.IO.FileSystem.Tests/RandomAccess/NonSeekable_AsyncHandles.cs b/src/libraries/System.Runtime/tests/System.IO.FileSystem.Tests/RandomAccess/NonSeekable_AsyncHandles.cs index 611760e361b17d..23abcbfe75d356 100644 --- a/src/libraries/System.Runtime/tests/System.IO.FileSystem.Tests/RandomAccess/NonSeekable_AsyncHandles.cs +++ b/src/libraries/System.Runtime/tests/System.IO.FileSystem.Tests/RandomAccess/NonSeekable_AsyncHandles.cs @@ -13,7 +13,7 @@ public class RandomAccess_NonSeekable_AsyncHandles : RandomAccess_NonSeekable { protected override bool AsyncHandles => true; - [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsWindows))] + [Theory] [InlineData(FileAccess.Read)] [InlineData(FileAccess.Write)] public async Task CancellationIsSupported(FileAccess access) diff --git a/src/libraries/System.Threading.ThreadPool/ref/System.Threading.ThreadPool.cs b/src/libraries/System.Threading.ThreadPool/ref/System.Threading.ThreadPool.cs index 47a420019ca6ef..43e25f197a5d43 100644 --- a/src/libraries/System.Threading.ThreadPool/ref/System.Threading.ThreadPool.cs +++ b/src/libraries/System.Threading.ThreadPool/ref/System.Threading.ThreadPool.cs @@ -78,6 +78,46 @@ public static partial class ThreadPool #endif public static System.Threading.RegisteredWaitHandle UnsafeRegisterWaitForSingleObject(System.Threading.WaitHandle waitObject, System.Threading.WaitOrTimerCallback callBack, object? state, uint millisecondsTimeOutInterval, bool executeOnlyOnce) { throw null; } } + public sealed partial class PollableHandle : System.IDisposable + { + internal PollableHandle() { } + public bool InlineCompletions { get { throw null; } set { } } + public static System.Threading.PollableHandle Create(System.Runtime.InteropServices.SafeHandle handle, ref System.Threading.PollableHandle? field) { throw null; } + public bool IsReadReady(out int observedSequenceNumber) { throw null; } + public bool IsWriteReady(out int observedSequenceNumber) { throw null; } + public System.Threading.PollOperationAsyncResult ReadAsync(System.Threading.PollTriggeredOperation operation, int observedSequenceNumber, System.Threading.CancellationToken cancellationToken) { throw null; } + public System.Threading.PollOperationAsyncResult WriteAsync(System.Threading.PollTriggeredOperation operation, int observedSequenceNumber, System.Threading.CancellationToken cancellationToken) { throw null; } + public System.Threading.PollOperationSyncResult ReadSync(System.Threading.PollTriggeredOperation operation, int observedSequenceNumber, int timeout) { throw null; } + public System.Threading.PollOperationSyncResult WriteSync(System.Threading.PollTriggeredOperation operation, int observedSequenceNumber, int timeout) { throw null; } + public bool AbortAndDispose() { throw null; } + public void Dispose() { } + } + public enum PollOperationAsyncResult + { + Pending = 0, + Completed = 1, + Aborted = 2, + } + public enum PollOperationOnCompletedResult + { + Completed = 1, + Aborted = 2, + Canceled = 3, + } + public enum PollOperationSyncResult + { + Completed = 1, + Aborted = 2, + TimedOut = 4, + } + public abstract partial class PollTriggeredOperation : System.Threading.IThreadPoolWorkItem + { + protected PollTriggeredOperation() { } + protected internal abstract bool TryCompleteOperation(System.Runtime.InteropServices.SafeHandle handle); + protected internal abstract void OnCompleted(System.Threading.PollOperationOnCompletedResult result); + protected virtual void ExecuteThreadPoolWorkItem() { } + void System.Threading.IThreadPoolWorkItem.Execute() { } + } public delegate void WaitCallback(object? state); public delegate void WaitOrTimerCallback(object? state, bool timedOut); } diff --git a/src/native/libs/System.Native/entrypoints.c b/src/native/libs/System.Native/entrypoints.c index 34131dee0312e5..8d9e60c1966fdb 100644 --- a/src/native/libs/System.Native/entrypoints.c +++ b/src/native/libs/System.Native/entrypoints.c @@ -106,13 +106,11 @@ static const Entry s_sysNative[] = DllImportEntry(SystemNative_PosixFAdvise) DllImportEntry(SystemNative_FAllocate) DllImportEntry(SystemNative_Read) - DllImportEntry(SystemNative_ReadFromNonblocking) DllImportEntry(SystemNative_ReadLink) DllImportEntry(SystemNative_Rename) DllImportEntry(SystemNative_RmDir) DllImportEntry(SystemNative_Sync) DllImportEntry(SystemNative_Write) - DllImportEntry(SystemNative_WriteToNonblocking) DllImportEntry(SystemNative_CopyFile) DllImportEntry(SystemNative_INotifyInit) DllImportEntry(SystemNative_INotifyAddWatch) @@ -189,12 +187,12 @@ static const Entry s_sysNative[] = DllImportEntry(SystemNative_GetSocketType) DllImportEntry(SystemNative_GetAtOutOfBandMark) DllImportEntry(SystemNative_GetBytesAvailable) - DllImportEntry(SystemNative_CreateSocketEventPort) - DllImportEntry(SystemNative_CloseSocketEventPort) - DllImportEntry(SystemNative_CreateSocketEventBuffer) - DllImportEntry(SystemNative_FreeSocketEventBuffer) - DllImportEntry(SystemNative_TryChangeSocketEventRegistration) - DllImportEntry(SystemNative_WaitForSocketEvents) + DllImportEntry(SystemNative_CreateHandleEventPort) + DllImportEntry(SystemNative_CloseHandleEventPort) + DllImportEntry(SystemNative_CreateHandleEventBuffer) + DllImportEntry(SystemNative_FreeHandleEventBuffer) + DllImportEntry(SystemNative_TryChangeHandleEventRegistration) + DllImportEntry(SystemNative_WaitForHandleEvents) DllImportEntry(SystemNative_GetWasiSocketDescriptor) DllImportEntry(SystemNative_PlatformSupportsDualModeIPv4PacketInfo) DllImportEntry(SystemNative_GetDomainSocketSizes) diff --git a/src/native/libs/System.Native/pal_io.c b/src/native/libs/System.Native/pal_io.c index 707d95646ea6b1..cf8ce44fd04903 100644 --- a/src/native/libs/System.Native/pal_io.c +++ b/src/native/libs/System.Native/pal_io.c @@ -43,6 +43,11 @@ #if HAVE_INOTIFY #include #endif +#if HAVE_EPOLL +#include +#elif HAVE_KQUEUE +#include +#endif #if HAVE_STATFS_VFS // Linux #include #elif HAVE_STATFS_MOUNT // BSD @@ -1249,66 +1254,7 @@ int32_t SystemNative_Read(intptr_t fd, void* buffer, int32_t bufferSize) return Common_Read(fd, buffer, bufferSize); } -int32_t SystemNative_ReadFromNonblocking(intptr_t fd, void* buffer, int32_t bufferSize) -{ - while (1) - { - int32_t result = Common_Read(fd, buffer, bufferSize); - if (result != -1 || (errno != EAGAIN && errno != EWOULDBLOCK)) - { - return result; - } - - // The fd is non-blocking and no data is available yet. - // Block (on a thread pool thread) until data arrives or the pipe/socket is closed. - PollEvent pollEvent = { .FileDescriptor = (int32_t)fd, .Events = PAL_POLLIN, .TriggeredEvents = 0 }; - uint32_t triggered = 0; - int32_t pollResult = Common_Poll(&pollEvent, 1, -1, &triggered); - if (pollResult != Error_SUCCESS) - { - errno = ConvertErrorPalToPlatform(pollResult); - return -1; - } - if ((pollEvent.TriggeredEvents & (PAL_POLLHUP | PAL_POLLERR)) != 0 && - (pollEvent.TriggeredEvents & PAL_POLLIN) == 0) - { - // The pipe/socket was closed with no data available (EOF). - return 0; - } - } -} - -int32_t SystemNative_WriteToNonblocking(intptr_t fd, const void* buffer, int32_t bufferSize) -{ - while (1) - { - int32_t result = Common_Write(fd, buffer, bufferSize); - if (result != -1 || (errno != EAGAIN && errno != EWOULDBLOCK)) - { - return result; - } - - // The fd is non-blocking and the write buffer is full. - // Block (on a thread pool thread) until space is available or the pipe/socket is closed. - PollEvent pollEvent = { .FileDescriptor = (int32_t)fd, .Events = PAL_POLLOUT, .TriggeredEvents = 0 }; - uint32_t triggered = 0; - int32_t pollResult = Common_Poll(&pollEvent, 1, -1, &triggered); - if (pollResult != Error_SUCCESS) - { - errno = ConvertErrorPalToPlatform(pollResult); - return -1; - } - - if ((pollEvent.TriggeredEvents & (PAL_POLLHUP | PAL_POLLERR)) != 0 && - (pollEvent.TriggeredEvents & PAL_POLLOUT) == 0) - { - // The pipe/socket was closed. - errno = EPIPE; - return -1; - } - } -} int32_t SystemNative_ReadLink(const char* path, char* buffer, int32_t bufferSize) { @@ -2019,35 +1965,11 @@ int64_t SystemNative_ReadV(intptr_t fd, IOVector* vectors, int32_t vectorCount) int fileDescriptor = ToFileDescriptor(fd); int allowedVectorCount = GetAllowedVectorCount(vectors, vectorCount); - while (1) - { - int64_t count; - while ((count = readv(fileDescriptor, (struct iovec*)vectors, allowedVectorCount)) < 0 && errno == EINTR); - - if (count != -1 || (errno != EAGAIN && errno != EWOULDBLOCK)) - { - assert(count >= -1); - return count; - } - - // The fd is non-blocking and no data is available yet. - // Block (on a thread pool thread) until data arrives or the pipe/socket is closed. - PollEvent pollEvent = { .FileDescriptor = fileDescriptor, .Events = PAL_POLLIN, .TriggeredEvents = 0 }; - uint32_t triggered = 0; - int32_t pollResult = Common_Poll(&pollEvent, 1, -1, &triggered); - if (pollResult != Error_SUCCESS) - { - errno = ConvertErrorPalToPlatform(pollResult); - return -1; - } + int64_t count; + while ((count = readv(fileDescriptor, (struct iovec*)vectors, allowedVectorCount)) < 0 && errno == EINTR); - if ((pollEvent.TriggeredEvents & (PAL_POLLHUP | PAL_POLLERR)) != 0 && - (pollEvent.TriggeredEvents & PAL_POLLIN) == 0) - { - // The pipe/socket was closed with no data available (EOF). - return 0; - } - } + assert(count >= -1); + return count; } int64_t SystemNative_PReadV(intptr_t fd, IOVector* vectors, int32_t vectorCount, int64_t fileOffset) @@ -2099,36 +2021,11 @@ int64_t SystemNative_WriteV(intptr_t fd, IOVector* vectors, int32_t vectorCount) int fileDescriptor = ToFileDescriptor(fd); int allowedVectorCount = GetAllowedVectorCount(vectors, vectorCount); - while (1) - { - int64_t count; - while ((count = writev(fileDescriptor, (struct iovec*)vectors, allowedVectorCount)) < 0 && errno == EINTR); - - if (count != -1 || (errno != EAGAIN && errno != EWOULDBLOCK)) - { - assert(count >= -1); - return count; - } + int64_t count; + while ((count = writev(fileDescriptor, (struct iovec*)vectors, allowedVectorCount)) < 0 && errno == EINTR); - // The fd is non-blocking and the write buffer is full. - // Block (on a thread pool thread) until space is available or the pipe/socket is closed. - PollEvent pollEvent = { .FileDescriptor = fileDescriptor, .Events = PAL_POLLOUT, .TriggeredEvents = 0 }; - uint32_t triggered = 0; - int32_t pollResult = Common_Poll(&pollEvent, 1, -1, &triggered); - if (pollResult != Error_SUCCESS) - { - errno = ConvertErrorPalToPlatform(pollResult); - return -1; - } - - if ((pollEvent.TriggeredEvents & (PAL_POLLHUP | PAL_POLLERR)) != 0 && - (pollEvent.TriggeredEvents & PAL_POLLOUT) == 0) - { - // The pipe/socket was closed. - errno = EPIPE; - return -1; - } - } + assert(count >= -1); + return count; } int64_t SystemNative_PWriteV(intptr_t fd, IOVector* vectors, int32_t vectorCount, int64_t fileOffset) @@ -2171,3 +2068,443 @@ int64_t SystemNative_PWriteV(intptr_t fd, IOVector* vectors, int32_t vectorCount assert(count >= -1); return count; } + +#if HAVE_KQUEUE +#if KEVENT_HAS_VOID_UDATA +static void* GetKeventUdata(uintptr_t udata) +{ + return (void*)udata; +} +static uintptr_t GetHandleEventData(void* udata) +{ + return (uintptr_t)udata; +} +#else +static intptr_t GetKeventUdata(uintptr_t udata) +{ + return (intptr_t)udata; +} +static uintptr_t GetHandleEventData(intptr_t udata) +{ + return (uintptr_t)udata; +} +#endif +#if KEVENT_REQUIRES_INT_PARAMS +static int GetKeventNchanges(int nchanges) +{ + return nchanges; +} +static int16_t GetKeventFilter(int16_t filter) +{ + return filter; +} +static uint16_t GetKeventFlags(uint16_t flags) +{ + return flags; +} +#else +static size_t GetKeventNchanges(int nchanges) +{ + return (size_t)nchanges; +} +static int16_t GetKeventFilter(uint32_t filter) +{ + return (int16_t)filter; +} +static uint16_t GetKeventFlags(uint32_t flags) +{ + return (uint16_t)flags; +} +#endif +#endif + +#if HAVE_EPOLL + +static const size_t HandleEventBufferElementSize = sizeof(struct epoll_event) > sizeof(HandleEvent) ? sizeof(struct epoll_event) : sizeof(HandleEvent); + +static int GetHandleEvents(uint32_t events) +{ + int asyncEvents = (((events & EPOLLIN) != 0) ? HandleEvents_READ : 0) | (((events & EPOLLOUT) != 0) ? HandleEvents_WRITE : 0) | + (((events & EPOLLRDHUP) != 0) ? HandleEvents_READCLOSE : 0) | + (((events & EPOLLHUP) != 0) ? HandleEvents_CLOSE : 0) | (((events & EPOLLERR) != 0) ? HandleEvents_ERROR : 0); + + return asyncEvents; +} + +static uint32_t GetEPollEvents(HandleEvents events) +{ + return (((events & HandleEvents_READ) != 0) ? EPOLLIN : 0) | (((events & HandleEvents_WRITE) != 0) ? EPOLLOUT : 0) | + (((events & HandleEvents_READCLOSE) != 0) ? EPOLLRDHUP : 0) | (((events & HandleEvents_CLOSE) != 0) ? EPOLLHUP : 0) | + (((events & HandleEvents_ERROR) != 0) ? EPOLLERR : 0); +} + +static int32_t CreateHandleEventPortInner(int32_t* port) +{ + assert(port != NULL); + + int epollFd = epoll_create1(EPOLL_CLOEXEC); + if (epollFd == -1) + { + *port = -1; + return SystemNative_ConvertErrorPlatformToPal(errno); + } + + *port = epollFd; + return Error_SUCCESS; +} + +static int32_t CloseHandleEventPortInner(int32_t port) +{ + int err = close(port); + return err == 0 || (err < 0 && errno == EINTR) ? Error_SUCCESS : SystemNative_ConvertErrorPlatformToPal(errno); +} + +static int32_t TryChangeHandleEventRegistrationInner( + int32_t port, int32_t socket, HandleEvents currentEvents, HandleEvents newEvents, uintptr_t data) +{ + assert(currentEvents != newEvents); + + int op = EPOLL_CTL_MOD; + if (currentEvents == HandleEvents_NONE) + { + op = EPOLL_CTL_ADD; + } + else if (newEvents == HandleEvents_NONE) + { + op = EPOLL_CTL_DEL; + } + + struct epoll_event evt; + memset(&evt, 0, sizeof(struct epoll_event)); + evt.events = GetEPollEvents(newEvents) | (unsigned int)EPOLLET; + evt.data.ptr = (void*)data; + int err = epoll_ctl(port, op, socket, &evt); + return err == 0 ? Error_SUCCESS : SystemNative_ConvertErrorPlatformToPal(errno); +} + +static void ConvertEventEPollToSocketAsync(HandleEvent* sae, struct epoll_event* epoll) +{ + assert(sae != NULL); + assert(epoll != NULL); + + // epoll does not play well with disconnected connection-oriented sockets, frequently + // reporting spurious EPOLLHUP events. Fortunately, EPOLLHUP may be handled as an + // EPOLLIN | EPOLLOUT event: the usual processing for these events will recognize and + // handle the HUP condition. + uint32_t events = epoll->events; + if ((events & EPOLLHUP) != 0) + { + events = (events & ((uint32_t)~EPOLLHUP)) | EPOLLIN | EPOLLOUT; + } + + memset(sae, 0, sizeof(HandleEvent)); + sae->Data = (uintptr_t)epoll->data.ptr; + sae->Events = GetHandleEvents(events); +} + +static int32_t WaitForHandleEventsInner(int32_t port, HandleEvent* buffer, int32_t* count) +{ + assert(buffer != NULL); + assert(count != NULL); + assert(*count >= 0); + + struct epoll_event* events = (struct epoll_event*)buffer; + int numEvents; + while ((numEvents = epoll_wait(port, events, *count, -1)) < 0 && errno == EINTR); + if (numEvents == -1) + { + *count = 0; + return SystemNative_ConvertErrorPlatformToPal(errno); + } + + // We should never see 0 events. Given an infinite timeout, epoll_wait will never return + // 0 events even if there are no file descriptors registered with the epoll fd. In + // that case, the wait will block until a file descriptor is added and an event occurs + // on the added file descriptor. + assert(numEvents != 0); + assert(numEvents <= *count); + + if (sizeof(struct epoll_event) < sizeof(HandleEvent)) + { + // Copy backwards to avoid overwriting earlier data. + for (int i = numEvents - 1; i >= 0; i--) + { + // This copy is made deliberately to avoid overwriting data. + struct epoll_event evt = events[i]; + ConvertEventEPollToSocketAsync(&buffer[i], &evt); + } + } + else + { + // Copy forwards for better cache behavior + for (int i = 0; i < numEvents; i++) + { + // This copy is made deliberately to avoid overwriting data. + struct epoll_event evt = events[i]; + ConvertEventEPollToSocketAsync(&buffer[i], &evt); + } + } + + *count = numEvents; + return Error_SUCCESS; +} + +#elif HAVE_KQUEUE + +c_static_assert(sizeof(HandleEvent) <= sizeof(struct kevent)); +static const size_t HandleEventBufferElementSize = sizeof(struct kevent); + +static HandleEvents GetHandleEvents(int16_t filter, uint16_t flags) +{ + int32_t events; + switch (filter) + { + case EVFILT_READ: + events = HandleEvents_READ; + if ((flags & EV_EOF) != 0) + { + events |= HandleEvents_READCLOSE; + } + break; + + case EVFILT_WRITE: + events = HandleEvents_WRITE; + + // kqueue does not play well with disconnected connection-oriented sockets, frequently + // reporting spurious EOF events. Fortunately, EOF may be handled as an EVFILT_READ | + // EVFILT_WRITE event: the usual processing for these events will recognize and + // handle the EOF condition. + if ((flags & EV_EOF) != 0) + { + events |= HandleEvents_READ; + } + break; + + default: + assert_msg(0, "unexpected kqueue filter type", (int)filter); + return HandleEvents_NONE; + } + + if ((flags & EV_ERROR) != 0) + { + events |= HandleEvents_ERROR; + } + + return (HandleEvents)events; +} + +static int32_t CreateHandleEventPortInner(int32_t* port) +{ + assert(port != NULL); + + int kqueueFd = kqueue(); + if (kqueueFd == -1) + { + *port = -1; + return SystemNative_ConvertErrorPlatformToPal(errno); + } + + *port = kqueueFd; + return Error_SUCCESS; +} + +static int32_t CloseHandleEventPortInner(int32_t port) +{ + int err = close(port); + return err == 0 || (err < 0 && errno == EINTR) ? Error_SUCCESS : SystemNative_ConvertErrorPlatformToPal(errno); +} + +static int32_t TryChangeHandleEventRegistrationInner( + int32_t port, int32_t socket, HandleEvents currentEvents, HandleEvents newEvents, uintptr_t data) +{ +#ifdef EV_RECEIPT + const uint16_t AddFlags = EV_ADD | EV_CLEAR | EV_RECEIPT; + const uint16_t RemoveFlags = EV_DELETE | EV_RECEIPT; +#else + const uint16_t AddFlags = EV_ADD | EV_CLEAR; + const uint16_t RemoveFlags = EV_DELETE; +#endif + + assert(currentEvents != newEvents); + + int32_t changes = currentEvents ^ newEvents; + int8_t readChanged = (changes & HandleEvents_READ) != 0; + int8_t writeChanged = (changes & HandleEvents_WRITE) != 0; + + struct kevent events[2]; + int err; + + int i = 0; + if (readChanged) + { + EV_SET(&events[i++], + (uint64_t)socket, + EVFILT_READ, + (newEvents & HandleEvents_READ) == 0 ? RemoveFlags : AddFlags, + 0, + 0, + GetKeventUdata(data)); +#if defined(__FreeBSD__) + // Issue: #30698 + // FreeBSD seems to have some issue when setting read/write events together. + // As a workaround use separate kevent() calls. + if (writeChanged) + { + while ((err = kevent(port, events, GetKeventNchanges(i), NULL, 0, NULL)) < 0 && errno == EINTR); + if (err != 0) + { + return SystemNative_ConvertErrorPlatformToPal(errno); + } + i = 0; + } +#endif + } + + if (writeChanged) + { + EV_SET(&events[i++], + (uint64_t)socket, + EVFILT_WRITE, + (newEvents & HandleEvents_WRITE) == 0 ? RemoveFlags : AddFlags, + 0, + 0, + GetKeventUdata(data)); + } + + while ((err = kevent(port, events, GetKeventNchanges(i), NULL, 0, NULL)) < 0 && errno == EINTR); + return err == 0 ? Error_SUCCESS : SystemNative_ConvertErrorPlatformToPal(errno); +} + +static int32_t WaitForHandleEventsInner(int32_t port, HandleEvent* buffer, int32_t* count) +{ + assert(buffer != NULL); + assert(count != NULL); + assert(*count >= 0); + + struct kevent* events = (struct kevent*)buffer; + int numEvents; + while ((numEvents = kevent(port, NULL, 0, events, GetKeventNchanges(*count), NULL)) < 0 && errno == EINTR); + if (numEvents == -1) + { + *count = -1; + return SystemNative_ConvertErrorPlatformToPal(errno); + } + + // We should never see 0 events. Given an infinite timeout, kevent will never return + // 0 events even if there are no file descriptors registered with the kqueue fd. In + // that case, the wait will block until a file descriptor is added and an event occurs + // on the added file descriptor. + assert(numEvents != 0); + assert(numEvents <= *count); + + for (int i = 0; i < numEvents; i++) + { + // This copy is made deliberately to avoid overwriting data. + struct kevent evt = events[i]; + memset(&buffer[i], 0, sizeof(HandleEvent)); + buffer[i].Data = GetHandleEventData(evt.udata); + buffer[i].Events = GetHandleEvents(GetKeventFilter(evt.filter), GetKeventFlags(evt.flags)); + } + + *count = numEvents; + return Error_SUCCESS; +} + +#else // !HAVE_KQUEUE !HAVE_EPOLL + +static const size_t HandleEventBufferElementSize = 0; + +static int32_t CloseHandleEventPortInner(int32_t port) +{ + return Error_ENOSYS; +} +static int32_t CreateHandleEventPortInner(int32_t* port) +{ + return Error_ENOSYS; +} +static int32_t TryChangeHandleEventRegistrationInner( + int32_t port, int32_t socket, HandleEvents currentEvents, HandleEvents newEvents, +uintptr_t data) +{ + return Error_ENOSYS; +} +static int32_t WaitForHandleEventsInner(int32_t port, HandleEvent* buffer, int32_t* count) +{ + return Error_ENOSYS; +} +#endif // !HAVE_KQUEUE !HAVE_EPOLL + +int32_t SystemNative_CreateHandleEventPort(intptr_t* port) +{ + if (port == NULL) + { + return Error_EFAULT; + } + + int fd; + int32_t error = CreateHandleEventPortInner(&fd); + *port = fd; + return error; +} + +int32_t SystemNative_CloseHandleEventPort(intptr_t port) +{ + return CloseHandleEventPortInner(ToFileDescriptor(port)); +} + +int32_t SystemNative_CreateHandleEventBuffer(int32_t count, HandleEvent** buffer) +{ + if (buffer == NULL || count < 0) + { + return Error_EFAULT; + } + + size_t bufferSize; + if (!multiply_s(HandleEventBufferElementSize, (size_t)count, &bufferSize) || + (*buffer = (HandleEvent*)malloc(bufferSize)) == NULL) + { + return Error_ENOMEM; + } + + return Error_SUCCESS; +} + +int32_t SystemNative_FreeHandleEventBuffer(HandleEvent* buffer) +{ + free(buffer); + return Error_SUCCESS; +} + +int32_t +SystemNative_TryChangeHandleEventRegistration(intptr_t port, intptr_t socket, int32_t currentEvents, int32_t newEvents, uintptr_t data) +{ + int portFd = ToFileDescriptor(port); + int socketFd = ToFileDescriptor(socket); + + const int32_t SupportedEvents = HandleEvents_READ | HandleEvents_WRITE | HandleEvents_READCLOSE | HandleEvents_CLOSE | HandleEvents_ERROR; + + if ((currentEvents & ~SupportedEvents) != 0 || (newEvents & ~SupportedEvents) != 0) + { + return Error_EINVAL; + } + + if (currentEvents == newEvents) + { + return Error_SUCCESS; + } + + return TryChangeHandleEventRegistrationInner( + portFd, socketFd, (HandleEvents)currentEvents, (HandleEvents)newEvents, data); +} + +int32_t SystemNative_WaitForHandleEvents(intptr_t port, HandleEvent* buffer, int32_t* count) +{ + if (buffer == NULL || count == NULL || *count < 0) + { + return Error_EFAULT; + } + + int fd = ToFileDescriptor(port); + + return WaitForHandleEventsInner(fd, buffer, count); +} diff --git a/src/native/libs/System.Native/pal_io.h b/src/native/libs/System.Native/pal_io.h index dbfc304b0b552c..1d908442b18828 100644 --- a/src/native/libs/System.Native/pal_io.h +++ b/src/native/libs/System.Native/pal_io.h @@ -713,14 +713,6 @@ PALEXPORT int32_t SystemNative_FAllocate(intptr_t fd, int64_t offset, int64_t le */ PALEXPORT int32_t SystemNative_Read(intptr_t fd, void* buffer, int32_t bufferSize); -/** - * Reads the number of bytes specified into the provided buffer from the specified, opened non-blocking file descriptor. - * If no data is currently available, polls the file descriptor until data arrives or the pipe/socket is closed. - * - * Returns the number of bytes read on success; 0 on EOF; otherwise, -1 is returned and errno is set. - */ -PALEXPORT int32_t SystemNative_ReadFromNonblocking(intptr_t fd, void* buffer, int32_t bufferSize); - /** * Takes a path to a symbolic link and attempts to place the link target path into the buffer. If the buffer is too * small, the path will be truncated. No matter what, the buffer will not be null terminated. @@ -756,14 +748,6 @@ PALEXPORT void SystemNative_Sync(void); */ PALEXPORT int32_t SystemNative_Write(intptr_t fd, const void* buffer, int32_t bufferSize); -/** - * Writes the specified buffer to the provided open non-blocking file descriptor. - * If the write buffer is currently full, polls the file descriptor until space is available or the pipe/socket is closed. - * - * Returns the number of bytes written on success; otherwise, returns -1 and sets errno. - */ -PALEXPORT int32_t SystemNative_WriteToNonblocking(intptr_t fd, const void* buffer, int32_t bufferSize); - /** * Copies all data from the source file descriptor to the destination file descriptor. * @@ -907,3 +891,38 @@ PALEXPORT int64_t SystemNative_ReadV(intptr_t fd, IOVector* vectors, int32_t vec * Returns the number of bytes written on success; otherwise, -1 is returned and errno is set. */ PALEXPORT int64_t SystemNative_WriteV(intptr_t fd, IOVector* vectors, int32_t vectorCount); + +/* + * Handle async events. + */ +typedef enum +{ + HandleEvents_NONE = 0x00, + HandleEvents_READ = 0x01, + HandleEvents_WRITE = 0x02, + HandleEvents_READCLOSE = 0x04, + HandleEvents_CLOSE = 0x08, + HandleEvents_ERROR = 0x10, + // Force the enum to use int32_t instead of uint32_t + HandleEvents__IGNORE_SIGNED = -1, +} HandleEvents; + +typedef struct +{ + uintptr_t Data; // User data for this event + int32_t Events; // Event flags + uint32_t Padding; // Pad out to 8-byte alignment +} HandleEvent; + +PALEXPORT int32_t SystemNative_CreateHandleEventPort(intptr_t* port); + +PALEXPORT int32_t SystemNative_CloseHandleEventPort(intptr_t port); + +PALEXPORT int32_t SystemNative_CreateHandleEventBuffer(int32_t count, HandleEvent** buffer); + +PALEXPORT int32_t SystemNative_FreeHandleEventBuffer(HandleEvent* buffer); + +PALEXPORT int32_t SystemNative_TryChangeHandleEventRegistration( + intptr_t port, intptr_t socket, int32_t currentEvents, int32_t newEvents, uintptr_t data); + +PALEXPORT int32_t SystemNative_WaitForHandleEvents(intptr_t port, HandleEvent* buffer, int32_t* count); diff --git a/src/native/libs/System.Native/pal_networking.c b/src/native/libs/System.Native/pal_networking.c index 7f13890ca61fe3..c560d73515cc77 100644 --- a/src/native/libs/System.Native/pal_networking.c +++ b/src/native/libs/System.Native/pal_networking.c @@ -17,12 +17,7 @@ #include #include #include -#if HAVE_EPOLL -#include -#elif HAVE_KQUEUE -#include -#include -#elif HAVE_SYS_POLL_H +#if HAVE_SYS_POLL_H #include #include #endif @@ -79,55 +74,6 @@ extern int getdomainname(char *name, int namelen); #endif -#if HAVE_KQUEUE -#if KEVENT_HAS_VOID_UDATA -static void* GetKeventUdata(uintptr_t udata) -{ - return (void*)udata; -} -static uintptr_t GetSocketEventData(void* udata) -{ - return (uintptr_t)udata; -} -#else -static intptr_t GetKeventUdata(uintptr_t udata) -{ - return (intptr_t)udata; -} -static uintptr_t GetSocketEventData(intptr_t udata) -{ - return (uintptr_t)udata; -} -#endif -#if KEVENT_REQUIRES_INT_PARAMS -static int GetKeventNchanges(int nchanges) -{ - return nchanges; -} -static int16_t GetKeventFilter(int16_t filter) -{ - return filter; -} -static uint16_t GetKeventFlags(uint16_t flags) -{ - return flags; -} -#else -static size_t GetKeventNchanges(int nchanges) -{ - return (size_t)nchanges; -} -static int16_t GetKeventFilter(uint32_t filter) -{ - return (int16_t)filter; -} -static uint16_t GetKeventFlags(uint32_t flags) -{ - return (uint16_t)flags; -} -#endif -#endif - #if !HAVE_IN_PKTINFO // On platforms, such as FreeBSD, where in_pktinfo // is not available, fallback to custom definition @@ -3128,322 +3074,6 @@ int32_t SystemNative_Select(int* readFds, int readFdsCount, int* writeFds, int w #endif } -#if HAVE_EPOLL - -static const size_t SocketEventBufferElementSize = sizeof(struct epoll_event) > sizeof(SocketEvent) ? sizeof(struct epoll_event) : sizeof(SocketEvent); - -static int GetSocketEvents(uint32_t events) -{ - int asyncEvents = (((events & EPOLLIN) != 0) ? SocketEvents_SA_READ : 0) | (((events & EPOLLOUT) != 0) ? SocketEvents_SA_WRITE : 0) | - (((events & EPOLLRDHUP) != 0) ? SocketEvents_SA_READCLOSE : 0) | - (((events & EPOLLHUP) != 0) ? SocketEvents_SA_CLOSE : 0) | (((events & EPOLLERR) != 0) ? SocketEvents_SA_ERROR : 0); - - return asyncEvents; -} - -static uint32_t GetEPollEvents(SocketEvents events) -{ - return (((events & SocketEvents_SA_READ) != 0) ? EPOLLIN : 0) | (((events & SocketEvents_SA_WRITE) != 0) ? EPOLLOUT : 0) | - (((events & SocketEvents_SA_READCLOSE) != 0) ? EPOLLRDHUP : 0) | (((events & SocketEvents_SA_CLOSE) != 0) ? EPOLLHUP : 0) | - (((events & SocketEvents_SA_ERROR) != 0) ? EPOLLERR : 0); -} - -static int32_t CreateSocketEventPortInner(int32_t* port) -{ - assert(port != NULL); - - int epollFd = epoll_create1(EPOLL_CLOEXEC); - if (epollFd == -1) - { - *port = -1; - return SystemNative_ConvertErrorPlatformToPal(errno); - } - - *port = epollFd; - return Error_SUCCESS; -} - -static int32_t CloseSocketEventPortInner(int32_t port) -{ - int err = close(port); - return err == 0 || (err < 0 && errno == EINTR) ? Error_SUCCESS : SystemNative_ConvertErrorPlatformToPal(errno); -} - -static int32_t TryChangeSocketEventRegistrationInner( - int32_t port, int32_t socket, SocketEvents currentEvents, SocketEvents newEvents, uintptr_t data) -{ - assert(currentEvents != newEvents); - - int op = EPOLL_CTL_MOD; - if (currentEvents == SocketEvents_SA_NONE) - { - op = EPOLL_CTL_ADD; - } - else if (newEvents == SocketEvents_SA_NONE) - { - op = EPOLL_CTL_DEL; - } - - struct epoll_event evt; - memset(&evt, 0, sizeof(struct epoll_event)); - evt.events = GetEPollEvents(newEvents) | (unsigned int)EPOLLET; - evt.data.ptr = (void*)data; - int err = epoll_ctl(port, op, socket, &evt); - return err == 0 ? Error_SUCCESS : SystemNative_ConvertErrorPlatformToPal(errno); -} - -static void ConvertEventEPollToSocketAsync(SocketEvent* sae, struct epoll_event* epoll) -{ - assert(sae != NULL); - assert(epoll != NULL); - - // epoll does not play well with disconnected connection-oriented sockets, frequently - // reporting spurious EPOLLHUP events. Fortunately, EPOLLHUP may be handled as an - // EPOLLIN | EPOLLOUT event: the usual processing for these events will recognize and - // handle the HUP condition. - uint32_t events = epoll->events; - if ((events & EPOLLHUP) != 0) - { - events = (events & ((uint32_t)~EPOLLHUP)) | EPOLLIN | EPOLLOUT; - } - - memset(sae, 0, sizeof(SocketEvent)); - sae->Data = (uintptr_t)epoll->data.ptr; - sae->Events = GetSocketEvents(events); -} - -static int32_t WaitForSocketEventsInner(int32_t port, SocketEvent* buffer, int32_t* count) -{ - assert(buffer != NULL); - assert(count != NULL); - assert(*count >= 0); - - struct epoll_event* events = (struct epoll_event*)buffer; - int numEvents; - while ((numEvents = epoll_wait(port, events, *count, -1)) < 0 && errno == EINTR); - if (numEvents == -1) - { - *count = 0; - return SystemNative_ConvertErrorPlatformToPal(errno); - } - - // We should never see 0 events. Given an infinite timeout, epoll_wait will never return - // 0 events even if there are no file descriptors registered with the epoll fd. In - // that case, the wait will block until a file descriptor is added and an event occurs - // on the added file descriptor. - assert(numEvents != 0); - assert(numEvents <= *count); - - if (sizeof(struct epoll_event) < sizeof(SocketEvent)) - { - // Copy backwards to avoid overwriting earlier data. - for (int i = numEvents - 1; i >= 0; i--) - { - // This copy is made deliberately to avoid overwriting data. - struct epoll_event evt = events[i]; - ConvertEventEPollToSocketAsync(&buffer[i], &evt); - } - } - else - { - // Copy forwards for better cache behavior - for (int i = 0; i < numEvents; i++) - { - // This copy is made deliberately to avoid overwriting data. - struct epoll_event evt = events[i]; - ConvertEventEPollToSocketAsync(&buffer[i], &evt); - } - } - - *count = numEvents; - return Error_SUCCESS; -} - -#elif HAVE_KQUEUE - -c_static_assert(sizeof(SocketEvent) <= sizeof(struct kevent)); -static const size_t SocketEventBufferElementSize = sizeof(struct kevent); - -static SocketEvents GetSocketEvents(int16_t filter, uint16_t flags) -{ - int32_t events; - switch (filter) - { - case EVFILT_READ: - events = SocketEvents_SA_READ; - if ((flags & EV_EOF) != 0) - { - events |= SocketEvents_SA_READCLOSE; - } - break; - - case EVFILT_WRITE: - events = SocketEvents_SA_WRITE; - - // kqueue does not play well with disconnected connection-oriented sockets, frequently - // reporting spurious EOF events. Fortunately, EOF may be handled as an EVFILT_READ | - // EVFILT_WRITE event: the usual processing for these events will recognize and - // handle the EOF condition. - if ((flags & EV_EOF) != 0) - { - events |= SocketEvents_SA_READ; - } - break; - - default: - assert_msg(0, "unexpected kqueue filter type", (int)filter); - return SocketEvents_SA_NONE; - } - - if ((flags & EV_ERROR) != 0) - { - events |= SocketEvents_SA_ERROR; - } - - return (SocketEvents)events; -} - -static int32_t CreateSocketEventPortInner(int32_t* port) -{ - assert(port != NULL); - - int kqueueFd = kqueue(); - if (kqueueFd == -1) - { - *port = -1; - return SystemNative_ConvertErrorPlatformToPal(errno); - } - - *port = kqueueFd; - return Error_SUCCESS; -} - -static int32_t CloseSocketEventPortInner(int32_t port) -{ - int err = close(port); - return err == 0 || (err < 0 && errno == EINTR) ? Error_SUCCESS : SystemNative_ConvertErrorPlatformToPal(errno); -} - -static int32_t TryChangeSocketEventRegistrationInner( - int32_t port, int32_t socket, SocketEvents currentEvents, SocketEvents newEvents, uintptr_t data) -{ -#ifdef EV_RECEIPT - const uint16_t AddFlags = EV_ADD | EV_CLEAR | EV_RECEIPT; - const uint16_t RemoveFlags = EV_DELETE | EV_RECEIPT; -#else - const uint16_t AddFlags = EV_ADD | EV_CLEAR; - const uint16_t RemoveFlags = EV_DELETE; -#endif - - assert(currentEvents != newEvents); - - int32_t changes = currentEvents ^ newEvents; - int8_t readChanged = (changes & SocketEvents_SA_READ) != 0; - int8_t writeChanged = (changes & SocketEvents_SA_WRITE) != 0; - - struct kevent events[2]; - int err; - - int i = 0; - if (readChanged) - { - EV_SET(&events[i++], - (uint64_t)socket, - EVFILT_READ, - (newEvents & SocketEvents_SA_READ) == 0 ? RemoveFlags : AddFlags, - 0, - 0, - GetKeventUdata(data)); -#if defined(__FreeBSD__) - // Issue: #30698 - // FreeBSD seems to have some issue when setting read/write events together. - // As a workaround use separate kevent() calls. - if (writeChanged) - { - while ((err = kevent(port, events, GetKeventNchanges(i), NULL, 0, NULL)) < 0 && errno == EINTR); - if (err != 0) - { - return SystemNative_ConvertErrorPlatformToPal(errno); - } - i = 0; - } -#endif - } - - if (writeChanged) - { - EV_SET(&events[i++], - (uint64_t)socket, - EVFILT_WRITE, - (newEvents & SocketEvents_SA_WRITE) == 0 ? RemoveFlags : AddFlags, - 0, - 0, - GetKeventUdata(data)); - } - - while ((err = kevent(port, events, GetKeventNchanges(i), NULL, 0, NULL)) < 0 && errno == EINTR); - return err == 0 ? Error_SUCCESS : SystemNative_ConvertErrorPlatformToPal(errno); -} - -static int32_t WaitForSocketEventsInner(int32_t port, SocketEvent* buffer, int32_t* count) -{ - assert(buffer != NULL); - assert(count != NULL); - assert(*count >= 0); - - struct kevent* events = (struct kevent*)buffer; - int numEvents; - while ((numEvents = kevent(port, NULL, 0, events, GetKeventNchanges(*count), NULL)) < 0 && errno == EINTR); - if (numEvents == -1) - { - *count = -1; - return SystemNative_ConvertErrorPlatformToPal(errno); - } - - // We should never see 0 events. Given an infinite timeout, kevent will never return - // 0 events even if there are no file descriptors registered with the kqueue fd. In - // that case, the wait will block until a file descriptor is added and an event occurs - // on the added file descriptor. - assert(numEvents != 0); - assert(numEvents <= *count); - - for (int i = 0; i < numEvents; i++) - { - // This copy is made deliberately to avoid overwriting data. - struct kevent evt = events[i]; - memset(&buffer[i], 0, sizeof(SocketEvent)); - buffer[i].Data = GetSocketEventData(evt.udata); - buffer[i].Events = GetSocketEvents(GetKeventFilter(evt.filter), GetKeventFlags(evt.flags)); - } - - *count = numEvents; - return Error_SUCCESS; -} - -#else // !HAVE_KQUEUE !HAVE_EPOLL - -static const size_t SocketEventBufferElementSize = 0; - -static int32_t CloseSocketEventPortInner(int32_t port) -{ - return Error_ENOSYS; -} -static int32_t CreateSocketEventPortInner(int32_t* port) -{ - return Error_ENOSYS; -} -static int32_t TryChangeSocketEventRegistrationInner( - int32_t port, int32_t socket, SocketEvents currentEvents, SocketEvents newEvents, -uintptr_t data) -{ - return Error_ENOSYS; -} -static int32_t WaitForSocketEventsInner(int32_t port, SocketEvent* buffer, int32_t* count) -{ - return Error_ENOSYS; -} -#endif // !HAVE_KQUEUE !HAVE_EPOLL - #if defined(TARGET_WASI) // from https://github.com/WebAssembly/wasi-libc/blob/230d4be6c54bec93181050f9e25c87150506bdd0/libc-bottom-half/headers/private/wasi/descriptor_table.h bool descriptor_table_get_ref(int fd, void **entry); @@ -3474,81 +3104,6 @@ int32_t SystemNative_GetWasiSocketDescriptor(intptr_t socket, void** entry) } #endif // TARGET_WASI -int32_t SystemNative_CreateSocketEventPort(intptr_t* port) -{ - if (port == NULL) - { - return Error_EFAULT; - } - - int fd; - int32_t error = CreateSocketEventPortInner(&fd); - *port = fd; - return error; -} - -int32_t SystemNative_CloseSocketEventPort(intptr_t port) -{ - return CloseSocketEventPortInner(ToFileDescriptor(port)); -} - -int32_t SystemNative_CreateSocketEventBuffer(int32_t count, SocketEvent** buffer) -{ - if (buffer == NULL || count < 0) - { - return Error_EFAULT; - } - - size_t bufferSize; - if (!multiply_s(SocketEventBufferElementSize, (size_t)count, &bufferSize) || - (*buffer = (SocketEvent*)malloc(bufferSize)) == NULL) - { - return Error_ENOMEM; - } - - return Error_SUCCESS; -} - -int32_t SystemNative_FreeSocketEventBuffer(SocketEvent* buffer) -{ - free(buffer); - return Error_SUCCESS; -} - -int32_t -SystemNative_TryChangeSocketEventRegistration(intptr_t port, intptr_t socket, int32_t currentEvents, int32_t newEvents, uintptr_t data) -{ - int portFd = ToFileDescriptor(port); - int socketFd = ToFileDescriptor(socket); - - const int32_t SupportedEvents = SocketEvents_SA_READ | SocketEvents_SA_WRITE | SocketEvents_SA_READCLOSE | SocketEvents_SA_CLOSE | SocketEvents_SA_ERROR; - - if ((currentEvents & ~SupportedEvents) != 0 || (newEvents & ~SupportedEvents) != 0) - { - return Error_EINVAL; - } - - if (currentEvents == newEvents) - { - return Error_SUCCESS; - } - - return TryChangeSocketEventRegistrationInner( - portFd, socketFd, (SocketEvents)currentEvents, (SocketEvents)newEvents, data); -} - -int32_t SystemNative_WaitForSocketEvents(intptr_t port, SocketEvent* buffer, int32_t* count) -{ - if (buffer == NULL || count == NULL || *count < 0) - { - return Error_EFAULT; - } - - int fd = ToFileDescriptor(port); - - return WaitForSocketEventsInner(fd, buffer, count); -} - int32_t SystemNative_PlatformSupportsDualModeIPv4PacketInfo(void) { #if HAVE_SUPPORT_FOR_DUAL_MODE_IPV4_PACKET_INFO diff --git a/src/native/libs/System.Native/pal_networking.h b/src/native/libs/System.Native/pal_networking.h index 2a42329fc02fe1..f4014affbe5cf2 100644 --- a/src/native/libs/System.Native/pal_networking.h +++ b/src/native/libs/System.Native/pal_networking.h @@ -213,20 +213,6 @@ typedef enum SocketFlags_MSG_ERRQUEUE = 0x2000, // used privately by Ping } SocketFlags; -/* - * Socket async events. - */ -typedef enum -{ - SocketEvents_SA_NONE = 0x00, - SocketEvents_SA_READ = 0x01, - SocketEvents_SA_WRITE = 0x02, - SocketEvents_SA_READCLOSE = 0x04, - SocketEvents_SA_CLOSE = 0x08, - SocketEvents_SA_ERROR = 0x10, - // Force the enum to use int32_t instead of uint32_t - SocketEvents__IGNORE_SIGNED = -1, -} SocketEvents; /** * IP address sizes. @@ -292,13 +278,6 @@ typedef struct int32_t Flags; } MessageHeader; -typedef struct -{ - uintptr_t Data; // User data for this event - int32_t Events; // Event flags - uint32_t Padding; // Pad out to 8-byte alignment -} SocketEvent; - PALEXPORT int32_t SystemNative_GetHostEntryForName(const uint8_t* address, int32_t addressFamily, HostEntry* entry); PALEXPORT void SystemNative_FreeHostEntry(HostEntry* entry); @@ -407,19 +386,6 @@ PALEXPORT int32_t SystemNative_GetBytesAvailable(intptr_t socket, int32_t* avail PALEXPORT int32_t SystemNative_GetWasiSocketDescriptor(intptr_t socket, void** entry); -PALEXPORT int32_t SystemNative_CreateSocketEventPort(intptr_t* port); - -PALEXPORT int32_t SystemNative_CloseSocketEventPort(intptr_t port); - -PALEXPORT int32_t SystemNative_CreateSocketEventBuffer(int32_t count, SocketEvent** buffer); - -PALEXPORT int32_t SystemNative_FreeSocketEventBuffer(SocketEvent* buffer); - -PALEXPORT int32_t SystemNative_TryChangeSocketEventRegistration( - intptr_t port, intptr_t socket, int32_t currentEvents, int32_t newEvents, uintptr_t data); - -PALEXPORT int32_t SystemNative_WaitForSocketEvents(intptr_t port, SocketEvent* buffer, int32_t* count); - PALEXPORT int32_t SystemNative_PlatformSupportsDualModeIPv4PacketInfo(void); PALEXPORT void SystemNative_GetDomainSocketSizes(int32_t* pathOffset, int32_t* pathSize, int32_t* addressSize); diff --git a/src/native/libs/System.Native/pal_networking_browser.c b/src/native/libs/System.Native/pal_networking_browser.c index 3f1de935f0e5ed..5eb4662a5efb40 100644 --- a/src/native/libs/System.Native/pal_networking_browser.c +++ b/src/native/libs/System.Native/pal_networking_browser.c @@ -423,50 +423,6 @@ int32_t SystemNative_GetWasiSocketDescriptor(intptr_t socket, void** entry) return Error_ENOTSUP; } -int32_t SystemNative_CreateSocketEventPort(intptr_t* port) -{ - (void)port; - return Error_ENOTSUP; -} - -int32_t SystemNative_CloseSocketEventPort(intptr_t port) -{ - (void)port; - return Error_ENOTSUP; -} - -int32_t SystemNative_CreateSocketEventBuffer(int32_t count, SocketEvent** buffer) -{ - (void)count; - (void)buffer; - return Error_ENOTSUP; -} - -int32_t SystemNative_FreeSocketEventBuffer(SocketEvent* buffer) -{ - (void)buffer; - return Error_ENOTSUP; -} - -int32_t SystemNative_TryChangeSocketEventRegistration( - intptr_t port, intptr_t socket, int32_t currentEvents, int32_t newEvents, uintptr_t data) -{ - (void)port; - (void)socket; - (void)currentEvents; - (void)newEvents; - (void)data; - return Error_ENOTSUP; -} - -int32_t SystemNative_WaitForSocketEvents(intptr_t port, SocketEvent* buffer, int32_t* count) -{ - (void)port; - (void)buffer; - (void)count; - return Error_ENOTSUP; -} - int32_t SystemNative_PlatformSupportsDualModeIPv4PacketInfo(void) { return 0; // false