From a6f9c6733b6a473770b9a89221bcaf4e7bf231ae Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 17 Apr 2026 08:00:27 +0000 Subject: [PATCH 01/25] Add Process.ReadAllLines synchronous API with platform-specific implementations and tests Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/bb3ce72c-ce34-45fe-b196-16d776008a31 Co-authored-by: adamsitnik <6011991+adamsitnik@users.noreply.github.com> --- .../ref/System.Diagnostics.Process.cs | 1 + .../Diagnostics/Process.Multiplexing.Unix.cs | 180 +++++++++++++ .../Process.Multiplexing.Windows.cs | 208 +++++++++++++++ .../Diagnostics/Process.Multiplexing.cs | 141 +++++++++++ .../tests/ProcessStreamingTests.cs | 237 +++++++++++++----- 5 files changed, 709 insertions(+), 58 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/ref/System.Diagnostics.Process.cs b/src/libraries/System.Diagnostics.Process/ref/System.Diagnostics.Process.cs index 4fa2522868feae..942b117769de28 100644 --- a/src/libraries/System.Diagnostics.Process/ref/System.Diagnostics.Process.cs +++ b/src/libraries/System.Diagnostics.Process/ref/System.Diagnostics.Process.cs @@ -159,6 +159,7 @@ public static void LeaveDebugMode() { } protected void OnExited() { } public (byte[] StandardOutput, byte[] StandardError) ReadAllBytes(System.TimeSpan? timeout = default(System.TimeSpan?)) { throw null; } public System.Threading.Tasks.Task<(byte[] StandardOutput, byte[] StandardError)> ReadAllBytesAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public System.Collections.Generic.IEnumerable ReadAllLines(System.TimeSpan? timeout = default(System.TimeSpan?)) { throw null; } public System.Collections.Generic.IAsyncEnumerable ReadAllLinesAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public (string StandardOutput, string StandardError) ReadAllText(System.TimeSpan? timeout = default(System.TimeSpan?)) { throw null; } public System.Threading.Tasks.Task<(string StandardOutput, string StandardError)> ReadAllTextAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs index 27393f1d50a8f2..206260a78a35f3 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs @@ -1,10 +1,13 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Buffers; +using System.Collections.Generic; using System.ComponentModel; using System.IO; using System.IO.Pipes; using System.Runtime.InteropServices; +using System.Text; using Microsoft.Win32.SafeHandles; namespace System.Diagnostics @@ -13,6 +16,183 @@ public partial class Process { private static SafePipeHandle GetSafeHandleFromStreamReader(StreamReader reader) => ((AnonymousPipeClientStream)reader.BaseStream).SafePipeHandle; + /// + /// Reads from both standard output and standard error pipes as lines of text using Unix + /// poll-based multiplexing with non-blocking reads. + /// The caller provides initially-rented buffers; this method takes ownership and returns them + /// to the pool when enumeration completes. + /// + private IEnumerable ReadPipesToLines( + int timeoutMs, + Encoding outputEncoding, + Encoding errorEncoding, + byte[] outputBuffer, + byte[] errorBuffer) + { + SafePipeHandle outputHandle = GetSafeHandleFromStreamReader(_standardOutput!); + SafePipeHandle errorHandle = GetSafeHandleFromStreamReader(_standardError!); + + bool outputRefAdded = false, errorRefAdded = false; + + try + { + outputHandle.DangerousAddRef(ref outputRefAdded); + errorHandle.DangerousAddRef(ref errorRefAdded); + + int outputFd = outputHandle.DangerousGetHandle().ToInt32(); + int errorFd = errorHandle.DangerousGetHandle().ToInt32(); + + if (Interop.Sys.Fcntl.DangerousSetIsNonBlocking(outputFd, 1) != 0 + || Interop.Sys.Fcntl.DangerousSetIsNonBlocking(errorFd, 1) != 0) + { + throw new Win32Exception(); + } + + // Cannot use stackalloc in an iterator method; use a regular array. + Interop.PollEvent[] pollFds = new Interop.PollEvent[2]; + + long deadline = timeoutMs >= 0 ? Environment.TickCount64 + timeoutMs : long.MaxValue; + + int outputStartIndex = 0, outputEndIndex = 0; + int errorStartIndex = 0, errorEndIndex = 0; + bool outputDone = false, errorDone = false; + + List lines = new(); + + while (!outputDone || !errorDone) + { + int numFds = 0; + int outputIndex = -1; + int errorIndex = -1; + + if (!errorDone) + { + errorIndex = numFds; + pollFds[numFds].FileDescriptor = errorFd; + pollFds[numFds].Events = Interop.PollEvents.POLLIN; + pollFds[numFds].TriggeredEvents = Interop.PollEvents.POLLNONE; + numFds++; + } + + if (!outputDone) + { + outputIndex = numFds; + pollFds[numFds].FileDescriptor = outputFd; + pollFds[numFds].Events = Interop.PollEvents.POLLIN; + pollFds[numFds].TriggeredEvents = Interop.PollEvents.POLLNONE; + numFds++; + } + + if (!TryGetRemainingTimeout(deadline, timeoutMs, out int pollTimeout)) + { + throw new TimeoutException(); + } + + Interop.Error pollError = PollPipes(pollFds, numFds, pollTimeout, out uint triggered); + if (pollError != Interop.Error.SUCCESS) + { + if (pollError == Interop.Error.EINTR) + { + continue; + } + + throw new Win32Exception(Interop.Sys.ConvertErrorPalToPlatform(pollError)); + } + + if (triggered == 0) + { + throw new TimeoutException(); + } + + // Process error pipe first (lower index) when both have data available. + for (int i = 0; i < numFds; i++) + { + if (pollFds[i].TriggeredEvents == Interop.PollEvents.POLLNONE) + { + continue; + } + + bool isError = i == errorIndex; + SafePipeHandle currentHandle = isError ? errorHandle : outputHandle; + Encoding currentEncoding = isError ? errorEncoding : outputEncoding; + + // Use explicit branching to avoid ref locals across yield points. + if (isError) + { + int bytesRead = ReadNonBlocking(currentHandle, errorBuffer, errorEndIndex); + if (bytesRead > 0) + { + errorEndIndex += bytesRead; + ParseLinesFromBuffer(errorBuffer, ref errorStartIndex, errorEndIndex, currentEncoding, true, lines); + CompactOrGrowLineBuffer(ref errorBuffer, ref errorStartIndex, ref errorEndIndex); + } + else if (bytesRead == 0) + { + EmitRemainingAsLine(errorBuffer, ref errorStartIndex, ref errorEndIndex, currentEncoding, true, lines); + errorDone = true; + } + // bytesRead < 0 means EAGAIN — nothing available yet, let poll retry. + } + else + { + int bytesRead = ReadNonBlocking(currentHandle, outputBuffer, outputEndIndex); + if (bytesRead > 0) + { + outputEndIndex += bytesRead; + ParseLinesFromBuffer(outputBuffer, ref outputStartIndex, outputEndIndex, currentEncoding, false, lines); + CompactOrGrowLineBuffer(ref outputBuffer, ref outputStartIndex, ref outputEndIndex); + } + else if (bytesRead == 0) + { + EmitRemainingAsLine(outputBuffer, ref outputStartIndex, ref outputEndIndex, currentEncoding, false, lines); + outputDone = true; + } + // bytesRead < 0 means EAGAIN — nothing available yet, let poll retry. + } + } + + // Yield parsed lines outside of any ref-local scope. + foreach (ProcessOutputLine line in lines) + { + yield return line; + } + + lines.Clear(); + } + } + finally + { + if (outputRefAdded) + { + outputHandle.DangerousRelease(); + } + + if (errorRefAdded) + { + errorHandle.DangerousRelease(); + } + + ArrayPool.Shared.Return(outputBuffer); + ArrayPool.Shared.Return(errorBuffer); + } + } + + /// + /// Calls poll(2) on the provided array of poll events. + /// + private static unsafe Interop.Error PollPipes(Interop.PollEvent[] pollFds, int numFds, int timeoutMs, out uint triggered) + { + uint localTriggered = 0; + Interop.Error result; + fixed (Interop.PollEvent* pPollFds = pollFds) + { + result = Interop.Sys.Poll(pPollFds, (uint)numFds, timeoutMs, &localTriggered); + } + + triggered = localTriggered; + return result; + } + /// /// Reads from both standard output and standard error pipes using Unix poll-based multiplexing /// with non-blocking reads. diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs index fb30cc07fb254c..ff56ef82cb3324 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs @@ -1,9 +1,12 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Buffers; +using System.Collections.Generic; using System.ComponentModel; using System.IO; using System.Runtime.InteropServices; +using System.Text; using System.Threading; using Microsoft.Win32.SafeHandles; @@ -13,6 +16,211 @@ public partial class Process { private static SafeFileHandle GetSafeHandleFromStreamReader(StreamReader reader) => ((FileStream)reader.BaseStream).SafeFileHandle; + /// + /// Reads from both standard output and standard error pipes as lines of text using Windows + /// overlapped IO with wait handles for single-threaded synchronous multiplexing. + /// The caller provides initially-rented buffers; this method takes ownership and returns them + /// to the pool when enumeration completes. + /// + private IEnumerable ReadPipesToLines( + int timeoutMs, + Encoding outputEncoding, + Encoding errorEncoding, + byte[] outputBuffer, + byte[] errorBuffer) + { + SafeFileHandle outputHandle = GetSafeHandleFromStreamReader(_standardOutput!); + SafeFileHandle errorHandle = GetSafeHandleFromStreamReader(_standardError!); + + bool outputRefAdded = false, errorRefAdded = false; + PinnedGCHandle outputPin = default, errorPin = default; + nint outputOverlappedNint = 0, errorOverlappedNint = 0; + EventWaitHandle? outputEvent = null, errorEvent = null; + + try + { + outputHandle.DangerousAddRef(ref outputRefAdded); + errorHandle.DangerousAddRef(ref errorRefAdded); + + outputPin = new PinnedGCHandle(outputBuffer); + errorPin = new PinnedGCHandle(errorBuffer); + + outputEvent = new EventWaitHandle(initialState: false, EventResetMode.ManualReset); + errorEvent = new EventWaitHandle(initialState: false, EventResetMode.ManualReset); + + unsafe + { + outputOverlappedNint = (nint)AllocateOverlapped(outputEvent); + errorOverlappedNint = (nint)AllocateOverlapped(errorEvent); + } + + // Error output gets index 0 so WaitAny services it first when both are signaled. + WaitHandle[] waitHandles = [errorEvent, outputEvent]; + + int outputStartIndex = 0, outputEndIndex = 0; + int errorStartIndex = 0, errorEndIndex = 0; + + bool outputDone, errorDone; + unsafe + { + outputDone = !QueueRead(outputHandle, outputPin.GetAddressOfArrayData(), + outputBuffer.Length, (NativeOverlapped*)outputOverlappedNint, outputEvent); + errorDone = !QueueRead(errorHandle, errorPin.GetAddressOfArrayData(), + errorBuffer.Length, (NativeOverlapped*)errorOverlappedNint, errorEvent); + } + + long deadline = timeoutMs >= 0 ? Environment.TickCount64 + timeoutMs : long.MaxValue; + List lines = new(); + + while (!outputDone || !errorDone) + { + int waitResult = TryGetRemainingTimeout(deadline, timeoutMs, out int remainingMilliseconds) + ? WaitHandle.WaitAny(waitHandles, remainingMilliseconds) + : WaitHandle.WaitTimeout; + + if (waitResult == WaitHandle.WaitTimeout) + { + unsafe + { + CancelPendingIOIfNeeded(outputHandle, outputDone, (NativeOverlapped*)outputOverlappedNint); + CancelPendingIOIfNeeded(errorHandle, errorDone, (NativeOverlapped*)errorOverlappedNint); + } + + throw new TimeoutException(); + } + + bool isError = waitResult == 0; + nint currentOverlappedNint = isError ? errorOverlappedNint : outputOverlappedNint; + SafeFileHandle currentHandle = isError ? errorHandle : outputHandle; + Encoding currentEncoding = isError ? errorEncoding : outputEncoding; + EventWaitHandle currentEvent = isError ? errorEvent! : outputEvent!; + + int bytesRead; + unsafe + { + bytesRead = GetOverlappedResultForPipe(currentHandle, (NativeOverlapped*)currentOverlappedNint); + } + + if (bytesRead > 0) + { + // Scoped block for ref locals that must not cross yield points. + { + ref int currentStartIndex = ref (isError ? ref errorStartIndex : ref outputStartIndex); + ref int currentEndIndex = ref (isError ? ref errorEndIndex : ref outputEndIndex); + ref byte[] currentBuffer = ref (isError ? ref errorBuffer : ref outputBuffer); + + currentEndIndex += bytesRead; + + ParseLinesFromBuffer(currentBuffer, ref currentStartIndex, currentEndIndex, currentEncoding, isError, lines); + CompactOrGrowLineBuffer(ref currentBuffer, ref currentStartIndex, ref currentEndIndex); + + // Update pin in case the buffer was replaced by growth. + if (isError) + { + errorPin.Target = errorBuffer; + } + else + { + outputPin.Target = outputBuffer; + } + + unsafe + { + ResetOverlapped(currentEvent, (NativeOverlapped*)currentOverlappedNint); + + byte* pinPointer = isError + ? errorPin.GetAddressOfArrayData() + : outputPin.GetAddressOfArrayData(); + + if (!QueueRead(currentHandle, pinPointer + currentEndIndex, + currentBuffer.Length - currentEndIndex, + (NativeOverlapped*)currentOverlappedNint, currentEvent)) + { + EmitRemainingAsLine(currentBuffer, ref currentStartIndex, ref currentEndIndex, + currentEncoding, isError, lines); + + if (isError) + { + errorDone = true; + } + else + { + outputDone = true; + } + + currentEvent.Reset(); + } + } + } + } + else + { + // EOF: pipe write end was closed. + { + ref int currentStartIndex = ref (isError ? ref errorStartIndex : ref outputStartIndex); + ref int currentEndIndex = ref (isError ? ref errorEndIndex : ref outputEndIndex); + ref byte[] currentBuffer = ref (isError ? ref errorBuffer : ref outputBuffer); + + EmitRemainingAsLine(currentBuffer, ref currentStartIndex, ref currentEndIndex, + currentEncoding, isError, lines); + } + + if (isError) + { + errorDone = true; + } + else + { + outputDone = true; + } + + currentEvent.Reset(); + } + + // Yield parsed lines outside of any unsafe or ref-local scope. + foreach (ProcessOutputLine line in lines) + { + yield return line; + } + + lines.Clear(); + } + } + finally + { + unsafe + { + if (outputOverlappedNint != 0) + { + NativeMemory.Free((void*)outputOverlappedNint); + } + + if (errorOverlappedNint != 0) + { + NativeMemory.Free((void*)errorOverlappedNint); + } + } + + outputEvent?.Dispose(); + errorEvent?.Dispose(); + outputPin.Dispose(); + errorPin.Dispose(); + + if (outputRefAdded) + { + outputHandle.DangerousRelease(); + } + + if (errorRefAdded) + { + errorHandle.DangerousRelease(); + } + + ArrayPool.Shared.Return(outputBuffer); + ArrayPool.Shared.Return(errorBuffer); + } + } + /// /// Reads from both standard output and standard error pipes using Windows overlapped IO /// with wait handles for single-threaded synchronous multiplexing. diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs index 8ff5443f4e2e16..75f97cda5c1007 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs @@ -113,6 +113,147 @@ public partial class Process } } + /// + /// Reads all standard output and standard error of the process as lines of text, + /// interleaving them as they become available. + /// + /// + /// The maximum amount of time to wait for the streams to be fully read. + /// When , waits indefinitely. + /// + /// + /// An enumerable of instances representing the lines + /// read from standard output and standard error. + /// + /// + /// Lines from standard output and standard error are yielded as they become available. + /// When data is available in both standard output and standard error, standard error + /// is processed first. + /// + /// + /// Standard output or standard error has not been redirected. + /// -or- + /// A redirected stream has already been used for synchronous or asynchronous reading. + /// + /// + /// The operation did not complete within the specified . + /// + /// + /// The process has been disposed. + /// + public IEnumerable ReadAllLines(TimeSpan? timeout = default) + { + ValidateReadAllState(); + + int timeoutMs = timeout.HasValue + ? ToTimeoutMilliseconds(timeout.Value) + : Timeout.Infinite; + + Encoding outputEncoding = _startInfo?.StandardOutputEncoding ?? GetStandardOutputEncoding(); + Encoding errorEncoding = _startInfo?.StandardErrorEncoding ?? GetStandardOutputEncoding(); + + byte[] outputBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); + byte[] errorBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); + + return ReadPipesToLines(timeoutMs, outputEncoding, errorEncoding, outputBuffer, errorBuffer); + } + + /// + /// Scans the buffer from to for complete lines + /// (delimited by \n), adds each as a to , + /// and advances past the consumed data. Handles both \r\n and \n + /// line endings. + /// + private static void ParseLinesFromBuffer( + byte[] buffer, + ref int startIndex, + int endIndex, + Encoding encoding, + bool standardError, + List lines) + { + while (startIndex < endIndex) + { + int remaining = endIndex - startIndex; + int lineEnd = buffer.AsSpan(startIndex, remaining).IndexOf((byte)'\n'); + if (lineEnd == -1) + { + break; + } + + int contentLength = lineEnd; + if (contentLength > 0 && buffer[startIndex + contentLength - 1] == (byte)'\r') + { + contentLength--; + } + + lines.Add(new ProcessOutputLine( + encoding.GetString(buffer, startIndex, contentLength), + standardError)); + + startIndex += lineEnd + 1; + } + } + + /// + /// Emits any remaining data in the buffer as a final line when an EOF is reached. + /// + private static void EmitRemainingAsLine( + byte[] buffer, + ref int startIndex, + ref int endIndex, + Encoding encoding, + bool standardError, + List lines) + { + if (startIndex < endIndex) + { + int length = endIndex - startIndex; + if (length > 0 && buffer[startIndex + length - 1] == (byte)'\r') + { + length--; + } + + if (length > 0) + { + lines.Add(new ProcessOutputLine( + encoding.GetString(buffer, startIndex, length), + standardError)); + } + + startIndex = 0; + endIndex = 0; + } + } + + /// + /// After line parsing, compacts remaining data to the front of the buffer if it has reached + /// the end, or rents a larger buffer if the entire buffer is filled with a single incomplete line. + /// + private static void CompactOrGrowLineBuffer(ref byte[] buffer, ref int startIndex, ref int endIndex) + { + if (endIndex < buffer.Length) + { + return; + } + + int remaining = endIndex - startIndex; + + if (remaining == buffer.Length) + { + // The buffer is too small to hold a single line — grow it. + RentLargerBuffer(ref buffer, remaining); + } + else + { + // Compact: move remaining data to the start of the buffer. + Buffer.BlockCopy(buffer, startIndex, buffer, 0, remaining); + } + + startIndex = 0; + endIndex = remaining; + } + /// /// Asynchronously reads all standard output and standard error of the process as text. /// diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs index 4b2967e1085611..b3caa874d949e6 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs @@ -13,8 +13,10 @@ public class ProcessStreamingTests : ProcessTestBase { private const string DontPrintAnything = "DO_NOT_PRINT"; - [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - public async Task ReadAllLinesAsync_ThrowsAfterDispose() + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData(true)] + [InlineData(false)] + public async Task ReadAllLines_ThrowsAfterDispose(bool useAsync) { Process process = CreateProcess(RemotelyInvokable.Dummy); process.Start(); @@ -22,54 +24,96 @@ public async Task ReadAllLinesAsync_ThrowsAfterDispose() process.Dispose(); - await Assert.ThrowsAsync(async () => + if (useAsync) { - await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + await Assert.ThrowsAsync(async () => { - } - }); + await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + { + } + }); + } + else + { + Assert.Throws(() => + { + foreach (ProcessOutputLine line in process.ReadAllLines()) + { + } + }); + } } - [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - public async Task ReadAllLinesAsync_ThrowsWhenNoStreamsRedirected() + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData(true)] + [InlineData(false)] + public async Task ReadAllLines_ThrowsWhenNoStreamsRedirected(bool useAsync) { Process process = CreateProcess(RemotelyInvokable.Dummy); process.Start(); - await Assert.ThrowsAsync(async () => + if (useAsync) { - await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + await Assert.ThrowsAsync(async () => { - } - }); + await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + { + } + }); + } + else + { + Assert.Throws(() => + { + foreach (ProcessOutputLine line in process.ReadAllLines()) + { + } + }); + } Assert.True(process.WaitForExit(WaitInMS)); } [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - [InlineData(true)] - [InlineData(false)] - public async Task ReadAllLinesAsync_ThrowsWhenOnlyOutputOrErrorIsRedirected(bool standardOutput) + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public async Task ReadAllLines_ThrowsWhenOnlyOutputOrErrorIsRedirected(bool standardOutput, bool useAsync) { Process process = CreateProcess(RemotelyInvokable.Dummy); process.StartInfo.RedirectStandardOutput = standardOutput; process.StartInfo.RedirectStandardError = !standardOutput; process.Start(); - await Assert.ThrowsAsync(async () => + if (useAsync) { - await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + await Assert.ThrowsAsync(async () => { - } - }); + await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + { + } + }); + } + else + { + Assert.Throws(() => + { + foreach (ProcessOutputLine line in process.ReadAllLines()) + { + } + }); + } Assert.True(process.WaitForExit(WaitInMS)); } [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - [InlineData(true)] - [InlineData(false)] - public async Task ReadAllLinesAsync_ThrowsWhenOutputOrErrorIsInSyncMode(bool standardOutput) + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public async Task ReadAllLines_ThrowsWhenOutputOrErrorIsInSyncMode(bool standardOutput, bool useAsync) { Process process = CreateProcess(RemotelyInvokable.Dummy); process.StartInfo.RedirectStandardOutput = true; @@ -79,20 +123,34 @@ public async Task ReadAllLinesAsync_ThrowsWhenOutputOrErrorIsInSyncMode(bool sta // Access the StreamReader property to set the stream to sync mode _ = standardOutput ? process.StandardOutput : process.StandardError; - await Assert.ThrowsAsync(async () => + if (useAsync) { - await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + await Assert.ThrowsAsync(async () => { - } - }); + await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + { + } + }); + } + else + { + Assert.Throws(() => + { + foreach (ProcessOutputLine line in process.ReadAllLines()) + { + } + }); + } Assert.True(process.WaitForExit(WaitInMS)); } [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - [InlineData(true)] - [InlineData(false)] - public async Task ReadAllLinesAsync_ThrowsWhenOutputOrErrorIsInAsyncMode(bool standardOutput) + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public async Task ReadAllLines_ThrowsWhenOutputOrErrorIsInAsyncMode(bool standardOutput, bool useAsync) { Process process = CreateProcess(RemotelyInvokable.StreamBody); process.StartInfo.RedirectStandardOutput = true; @@ -108,12 +166,24 @@ public async Task ReadAllLinesAsync_ThrowsWhenOutputOrErrorIsInAsyncMode(bool st process.BeginErrorReadLine(); } - await Assert.ThrowsAsync(async () => + if (useAsync) { - await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + await Assert.ThrowsAsync(async () => { - } - }); + await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + { + } + }); + } + else + { + Assert.Throws(() => + { + foreach (ProcessOutputLine line in process.ReadAllLines()) + { + } + }); + } if (standardOutput) { @@ -128,11 +198,15 @@ await Assert.ThrowsAsync(async () => } [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - [InlineData("hello", "world")] - [InlineData("just output", "")] - [InlineData("", "just error")] - [InlineData("", "")] - public async Task ReadAllLinesAsync_ReadsBothOutputAndError(string standardOutput, string standardError) + [InlineData("hello", "world", true)] + [InlineData("hello", "world", false)] + [InlineData("just output", "", true)] + [InlineData("just output", "", false)] + [InlineData("", "just error", true)] + [InlineData("", "just error", false)] + [InlineData("", "", true)] + [InlineData("", "", false)] + public async Task ReadAllLines_ReadsBothOutputAndError(string standardOutput, string standardError, bool useAsync) { using Process process = StartLinePrintingProcess( string.IsNullOrEmpty(standardOutput) ? DontPrintAnything : standardOutput, @@ -141,7 +215,7 @@ public async Task ReadAllLinesAsync_ReadsBothOutputAndError(string standardOutpu List capturedOutput = new(); List capturedError = new(); - await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + await foreach (ProcessOutputLine line in EnumerateLines(process, useAsync)) { if (line.StandardError) { @@ -174,8 +248,10 @@ public async Task ReadAllLinesAsync_ReadsBothOutputAndError(string standardOutpu Assert.True(process.WaitForExit(WaitInMS)); } - [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - public async Task ReadAllLinesAsync_ReadsInterleavedOutput() + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData(true)] + [InlineData(false)] + public async Task ReadAllLines_ReadsInterleavedOutput(bool useAsync) { const int iterations = 100; using Process process = CreateProcess(() => @@ -198,7 +274,7 @@ public async Task ReadAllLinesAsync_ReadsInterleavedOutput() List capturedOutput = new(); List capturedError = new(); - await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + await foreach (ProcessOutputLine line in EnumerateLines(process, useAsync)) { if (line.StandardError) { @@ -224,8 +300,10 @@ public async Task ReadAllLinesAsync_ReadsInterleavedOutput() Assert.True(process.WaitForExit(WaitInMS)); } - [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - public async Task ReadAllLinesAsync_ReadsLargeOutput() + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData(true)] + [InlineData(false)] + public async Task ReadAllLines_ReadsLargeOutput(bool useAsync) { const int lineCount = 1000; using Process process = CreateProcess(() => @@ -245,7 +323,7 @@ public async Task ReadAllLinesAsync_ReadsLargeOutput() List capturedOutput = new(); List capturedError = new(); - await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + await foreach (ProcessOutputLine line in EnumerateLines(process, useAsync)) { if (line.StandardError) { @@ -266,8 +344,10 @@ public async Task ReadAllLinesAsync_ReadsLargeOutput() Assert.True(process.WaitForExit(WaitInMS)); } - [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - public async Task ReadAllLinesAsync_ThrowsOperationCanceledOnCancellation() + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData(true)] + [InlineData(false)] + public async Task ReadAllLines_ThrowsOnCancellationOrTimeout(bool useAsync) { Process process = CreateProcess(RemotelyInvokable.ReadLine); process.StartInfo.RedirectStandardOutput = true; @@ -277,14 +357,26 @@ public async Task ReadAllLinesAsync_ThrowsOperationCanceledOnCancellation() try { - using CancellationTokenSource cts = new(TimeSpan.FromMilliseconds(100)); + if (useAsync) + { + using CancellationTokenSource cts = new(TimeSpan.FromMilliseconds(100)); - await Assert.ThrowsAnyAsync(async () => + await Assert.ThrowsAnyAsync(async () => + { + await foreach (ProcessOutputLine line in process.ReadAllLinesAsync(cts.Token)) + { + } + }); + } + else { - await foreach (ProcessOutputLine line in process.ReadAllLinesAsync(cts.Token)) + Assert.Throws(() => { - } - }); + foreach (ProcessOutputLine line in process.ReadAllLines(TimeSpan.FromMilliseconds(100))) + { + } + }); + } } finally { @@ -294,14 +386,16 @@ await Assert.ThrowsAnyAsync(async () => Assert.True(process.WaitForExit(WaitInMS)); } - [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - public async Task ReadAllLinesAsync_ProcessOutputLineProperties() + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData(true)] + [InlineData(false)] + public async Task ReadAllLines_ProcessOutputLineProperties(bool useAsync) { using Process process = StartLinePrintingProcess("stdout_line", "stderr_line"); List allLines = new(); - await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + await foreach (ProcessOutputLine line in EnumerateLines(process, useAsync)) { allLines.Add(line); } @@ -312,8 +406,10 @@ public async Task ReadAllLinesAsync_ProcessOutputLineProperties() Assert.True(process.WaitForExit(WaitInMS)); } - [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - public async Task ReadAllLinesAsync_StopsCleanlyWhenConsumerBreaksEarly() + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData(true)] + [InlineData(false)] + public async Task ReadAllLines_StopsCleanlyWhenConsumerBreaksEarly(bool useAsync) { using Process process = CreateProcess(() => { @@ -333,7 +429,7 @@ public async Task ReadAllLinesAsync_StopsCleanlyWhenConsumerBreaksEarly() ProcessOutputLine? firstLine = null; - await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + await foreach (ProcessOutputLine line in EnumerateLines(process, useAsync)) { firstLine = line; break; // stop after first line @@ -368,5 +464,30 @@ private Process StartLinePrintingProcess(string stdOutText, string stdErrText) return process; } + + /// + /// Helper that wraps both the sync and async line-reading APIs into an + /// so callers can always + /// use await foreach. + /// + private static async IAsyncEnumerable EnumerateLines(Process process, bool useAsync) + { + if (useAsync) + { + await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + { + yield return line; + } + } + else + { + foreach (ProcessOutputLine line in process.ReadAllLines()) + { + yield return line; + } + + await Task.CompletedTask; // Ensure the method is valid async + } + } } } From ab4eca94e43b5f49a5d550d3b4716a788231a22d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 17 Apr 2026 08:09:15 +0000 Subject: [PATCH 02/25] Move buffer renting into iterator for proper cleanup, remove unnecessary await in test helper Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/bb3ce72c-ce34-45fe-b196-16d776008a31 Co-authored-by: adamsitnik <6011991+adamsitnik@users.noreply.github.com> --- .../src/System/Diagnostics/Process.Multiplexing.Unix.cs | 9 ++++----- .../System/Diagnostics/Process.Multiplexing.Windows.cs | 9 ++++----- .../src/System/Diagnostics/Process.Multiplexing.cs | 5 +---- .../tests/ProcessStreamingTests.cs | 2 -- 4 files changed, 9 insertions(+), 16 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs index 206260a78a35f3..0886b926eee639 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs @@ -19,19 +19,18 @@ public partial class Process /// /// Reads from both standard output and standard error pipes as lines of text using Unix /// poll-based multiplexing with non-blocking reads. - /// The caller provides initially-rented buffers; this method takes ownership and returns them - /// to the pool when enumeration completes. + /// Buffers are rented from the pool and returned when enumeration completes. /// private IEnumerable ReadPipesToLines( int timeoutMs, Encoding outputEncoding, - Encoding errorEncoding, - byte[] outputBuffer, - byte[] errorBuffer) + Encoding errorEncoding) { SafePipeHandle outputHandle = GetSafeHandleFromStreamReader(_standardOutput!); SafePipeHandle errorHandle = GetSafeHandleFromStreamReader(_standardError!); + byte[] outputBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); + byte[] errorBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); bool outputRefAdded = false, errorRefAdded = false; try diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs index ff56ef82cb3324..25e458608fc7f8 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs @@ -19,19 +19,18 @@ public partial class Process /// /// Reads from both standard output and standard error pipes as lines of text using Windows /// overlapped IO with wait handles for single-threaded synchronous multiplexing. - /// The caller provides initially-rented buffers; this method takes ownership and returns them - /// to the pool when enumeration completes. + /// Buffers are rented from the pool and returned when enumeration completes. /// private IEnumerable ReadPipesToLines( int timeoutMs, Encoding outputEncoding, - Encoding errorEncoding, - byte[] outputBuffer, - byte[] errorBuffer) + Encoding errorEncoding) { SafeFileHandle outputHandle = GetSafeHandleFromStreamReader(_standardOutput!); SafeFileHandle errorHandle = GetSafeHandleFromStreamReader(_standardError!); + byte[] outputBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); + byte[] errorBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); bool outputRefAdded = false, errorRefAdded = false; PinnedGCHandle outputPin = default, errorPin = default; nint outputOverlappedNint = 0, errorOverlappedNint = 0; diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs index 75f97cda5c1007..0acd91471cc3b0 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs @@ -152,10 +152,7 @@ public IEnumerable ReadAllLines(TimeSpan? timeout = default) Encoding outputEncoding = _startInfo?.StandardOutputEncoding ?? GetStandardOutputEncoding(); Encoding errorEncoding = _startInfo?.StandardErrorEncoding ?? GetStandardOutputEncoding(); - byte[] outputBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); - byte[] errorBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); - - return ReadPipesToLines(timeoutMs, outputEncoding, errorEncoding, outputBuffer, errorBuffer); + return ReadPipesToLines(timeoutMs, outputEncoding, errorEncoding); } /// diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs index b3caa874d949e6..debf4816ef50cd 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs @@ -485,8 +485,6 @@ private static async IAsyncEnumerable EnumerateLines(Process { yield return line; } - - await Task.CompletedTask; // Ensure the method is valid async } } } From b0ed0c6e13c44b625276dfb6c5406a68753983de Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 19 Apr 2026 12:15:14 +0000 Subject: [PATCH 03/25] Fix encoding: use Decoder-based char-level line scanning for correct UTF-16/UTF-32 support; fix Windows use-after-free by canceling pending IO in finally Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/a9eadcb8-dd77-4933-9877-920b21513e25 Co-authored-by: adamsitnik <6011991+adamsitnik@users.noreply.github.com> --- .../Diagnostics/Process.Multiplexing.Unix.cs | 42 +++--- .../Process.Multiplexing.Windows.cs | 122 +++++++++--------- .../Diagnostics/Process.Multiplexing.cs | 82 +++++++++--- 3 files changed, 147 insertions(+), 99 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs index 0886b926eee639..1f6adb357988a0 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs @@ -29,8 +29,10 @@ private IEnumerable ReadPipesToLines( SafePipeHandle outputHandle = GetSafeHandleFromStreamReader(_standardOutput!); SafePipeHandle errorHandle = GetSafeHandleFromStreamReader(_standardError!); - byte[] outputBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); - byte[] errorBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); + byte[] outputByteBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); + byte[] errorByteBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); + char[] outputCharBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); + char[] errorCharBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); bool outputRefAdded = false, errorRefAdded = false; try @@ -50,10 +52,13 @@ private IEnumerable ReadPipesToLines( // Cannot use stackalloc in an iterator method; use a regular array. Interop.PollEvent[] pollFds = new Interop.PollEvent[2]; + Decoder outputDecoder = outputEncoding.GetDecoder(); + Decoder errorDecoder = errorEncoding.GetDecoder(); + long deadline = timeoutMs >= 0 ? Environment.TickCount64 + timeoutMs : long.MaxValue; - int outputStartIndex = 0, outputEndIndex = 0; - int errorStartIndex = 0, errorEndIndex = 0; + int outputCharStart = 0, outputCharEnd = 0; + int errorCharStart = 0, errorCharEnd = 0; bool outputDone = false, errorDone = false; List lines = new(); @@ -113,37 +118,38 @@ private IEnumerable ReadPipesToLines( bool isError = i == errorIndex; SafePipeHandle currentHandle = isError ? errorHandle : outputHandle; - Encoding currentEncoding = isError ? errorEncoding : outputEncoding; // Use explicit branching to avoid ref locals across yield points. if (isError) { - int bytesRead = ReadNonBlocking(currentHandle, errorBuffer, errorEndIndex); + int bytesRead = ReadNonBlocking(currentHandle, errorByteBuffer, 0); if (bytesRead > 0) { - errorEndIndex += bytesRead; - ParseLinesFromBuffer(errorBuffer, ref errorStartIndex, errorEndIndex, currentEncoding, true, lines); - CompactOrGrowLineBuffer(ref errorBuffer, ref errorStartIndex, ref errorEndIndex); + DecodeAndAppendChars(errorDecoder, errorByteBuffer, 0, bytesRead, flush: false, ref errorCharBuffer, ref errorCharEnd); + ParseLinesFromCharBuffer(errorCharBuffer, ref errorCharStart, errorCharEnd, true, lines); + CompactOrGrowCharBuffer(ref errorCharBuffer, ref errorCharStart, ref errorCharEnd); } else if (bytesRead == 0) { - EmitRemainingAsLine(errorBuffer, ref errorStartIndex, ref errorEndIndex, currentEncoding, true, lines); + DecodeAndAppendChars(errorDecoder, Array.Empty(), 0, 0, flush: true, ref errorCharBuffer, ref errorCharEnd); + EmitRemainingCharsAsLine(errorCharBuffer, ref errorCharStart, ref errorCharEnd, true, lines); errorDone = true; } // bytesRead < 0 means EAGAIN — nothing available yet, let poll retry. } else { - int bytesRead = ReadNonBlocking(currentHandle, outputBuffer, outputEndIndex); + int bytesRead = ReadNonBlocking(currentHandle, outputByteBuffer, 0); if (bytesRead > 0) { - outputEndIndex += bytesRead; - ParseLinesFromBuffer(outputBuffer, ref outputStartIndex, outputEndIndex, currentEncoding, false, lines); - CompactOrGrowLineBuffer(ref outputBuffer, ref outputStartIndex, ref outputEndIndex); + DecodeAndAppendChars(outputDecoder, outputByteBuffer, 0, bytesRead, flush: false, ref outputCharBuffer, ref outputCharEnd); + ParseLinesFromCharBuffer(outputCharBuffer, ref outputCharStart, outputCharEnd, false, lines); + CompactOrGrowCharBuffer(ref outputCharBuffer, ref outputCharStart, ref outputCharEnd); } else if (bytesRead == 0) { - EmitRemainingAsLine(outputBuffer, ref outputStartIndex, ref outputEndIndex, currentEncoding, false, lines); + DecodeAndAppendChars(outputDecoder, Array.Empty(), 0, 0, flush: true, ref outputCharBuffer, ref outputCharEnd); + EmitRemainingCharsAsLine(outputCharBuffer, ref outputCharStart, ref outputCharEnd, false, lines); outputDone = true; } // bytesRead < 0 means EAGAIN — nothing available yet, let poll retry. @@ -171,8 +177,10 @@ private IEnumerable ReadPipesToLines( errorHandle.DangerousRelease(); } - ArrayPool.Shared.Return(outputBuffer); - ArrayPool.Shared.Return(errorBuffer); + ArrayPool.Shared.Return(outputByteBuffer); + ArrayPool.Shared.Return(errorByteBuffer); + ArrayPool.Shared.Return(outputCharBuffer); + ArrayPool.Shared.Return(errorCharBuffer); } } diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs index 25e458608fc7f8..28c2bcddfbbe31 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs @@ -29,20 +29,23 @@ private IEnumerable ReadPipesToLines( SafeFileHandle outputHandle = GetSafeHandleFromStreamReader(_standardOutput!); SafeFileHandle errorHandle = GetSafeHandleFromStreamReader(_standardError!); - byte[] outputBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); - byte[] errorBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); + byte[] outputByteBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); + byte[] errorByteBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); + char[] outputCharBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); + char[] errorCharBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); bool outputRefAdded = false, errorRefAdded = false; PinnedGCHandle outputPin = default, errorPin = default; nint outputOverlappedNint = 0, errorOverlappedNint = 0; EventWaitHandle? outputEvent = null, errorEvent = null; + bool outputDone = true, errorDone = true; try { outputHandle.DangerousAddRef(ref outputRefAdded); errorHandle.DangerousAddRef(ref errorRefAdded); - outputPin = new PinnedGCHandle(outputBuffer); - errorPin = new PinnedGCHandle(errorBuffer); + outputPin = new PinnedGCHandle(outputByteBuffer); + errorPin = new PinnedGCHandle(errorByteBuffer); outputEvent = new EventWaitHandle(initialState: false, EventResetMode.ManualReset); errorEvent = new EventWaitHandle(initialState: false, EventResetMode.ManualReset); @@ -56,16 +59,18 @@ private IEnumerable ReadPipesToLines( // Error output gets index 0 so WaitAny services it first when both are signaled. WaitHandle[] waitHandles = [errorEvent, outputEvent]; - int outputStartIndex = 0, outputEndIndex = 0; - int errorStartIndex = 0, errorEndIndex = 0; + Decoder outputDecoder = outputEncoding.GetDecoder(); + Decoder errorDecoder = errorEncoding.GetDecoder(); + + int outputCharStart = 0, outputCharEnd = 0; + int errorCharStart = 0, errorCharEnd = 0; - bool outputDone, errorDone; unsafe { outputDone = !QueueRead(outputHandle, outputPin.GetAddressOfArrayData(), - outputBuffer.Length, (NativeOverlapped*)outputOverlappedNint, outputEvent); + outputByteBuffer.Length, (NativeOverlapped*)outputOverlappedNint, outputEvent); errorDone = !QueueRead(errorHandle, errorPin.GetAddressOfArrayData(), - errorBuffer.Length, (NativeOverlapped*)errorOverlappedNint, errorEvent); + errorByteBuffer.Length, (NativeOverlapped*)errorOverlappedNint, errorEvent); } long deadline = timeoutMs >= 0 ? Environment.TickCount64 + timeoutMs : long.MaxValue; @@ -91,7 +96,6 @@ private IEnumerable ReadPipesToLines( bool isError = waitResult == 0; nint currentOverlappedNint = isError ? errorOverlappedNint : outputOverlappedNint; SafeFileHandle currentHandle = isError ? errorHandle : outputHandle; - Encoding currentEncoding = isError ? errorEncoding : outputEncoding; EventWaitHandle currentEvent = isError ? errorEvent! : outputEvent!; int bytesRead; @@ -102,74 +106,64 @@ private IEnumerable ReadPipesToLines( if (bytesRead > 0) { - // Scoped block for ref locals that must not cross yield points. + // Decode bytes to chars and parse lines. + if (isError) + { + DecodeAndAppendChars(errorDecoder, errorByteBuffer, 0, bytesRead, flush: false, ref errorCharBuffer, ref errorCharEnd); + ParseLinesFromCharBuffer(errorCharBuffer, ref errorCharStart, errorCharEnd, true, lines); + CompactOrGrowCharBuffer(ref errorCharBuffer, ref errorCharStart, ref errorCharEnd); + } + else { - ref int currentStartIndex = ref (isError ? ref errorStartIndex : ref outputStartIndex); - ref int currentEndIndex = ref (isError ? ref errorEndIndex : ref outputEndIndex); - ref byte[] currentBuffer = ref (isError ? ref errorBuffer : ref outputBuffer); + DecodeAndAppendChars(outputDecoder, outputByteBuffer, 0, bytesRead, flush: false, ref outputCharBuffer, ref outputCharEnd); + ParseLinesFromCharBuffer(outputCharBuffer, ref outputCharStart, outputCharEnd, false, lines); + CompactOrGrowCharBuffer(ref outputCharBuffer, ref outputCharStart, ref outputCharEnd); + } - currentEndIndex += bytesRead; + unsafe + { + ResetOverlapped(currentEvent, (NativeOverlapped*)currentOverlappedNint); - ParseLinesFromBuffer(currentBuffer, ref currentStartIndex, currentEndIndex, currentEncoding, isError, lines); - CompactOrGrowLineBuffer(ref currentBuffer, ref currentStartIndex, ref currentEndIndex); + byte* pinPointer = isError + ? errorPin.GetAddressOfArrayData() + : outputPin.GetAddressOfArrayData(); + byte[] currentByteBuffer = isError ? errorByteBuffer : outputByteBuffer; - // Update pin in case the buffer was replaced by growth. - if (isError) - { - errorPin.Target = errorBuffer; - } - else + if (!QueueRead(currentHandle, pinPointer, + currentByteBuffer.Length, + (NativeOverlapped*)currentOverlappedNint, currentEvent)) { - outputPin.Target = outputBuffer; - } - - unsafe - { - ResetOverlapped(currentEvent, (NativeOverlapped*)currentOverlappedNint); - - byte* pinPointer = isError - ? errorPin.GetAddressOfArrayData() - : outputPin.GetAddressOfArrayData(); - - if (!QueueRead(currentHandle, pinPointer + currentEndIndex, - currentBuffer.Length - currentEndIndex, - (NativeOverlapped*)currentOverlappedNint, currentEvent)) + // EOF during QueueRead — flush decoder and emit remaining. + if (isError) + { + DecodeAndAppendChars(errorDecoder, Array.Empty(), 0, 0, flush: true, ref errorCharBuffer, ref errorCharEnd); + EmitRemainingCharsAsLine(errorCharBuffer, ref errorCharStart, ref errorCharEnd, true, lines); + errorDone = true; + } + else { - EmitRemainingAsLine(currentBuffer, ref currentStartIndex, ref currentEndIndex, - currentEncoding, isError, lines); - - if (isError) - { - errorDone = true; - } - else - { - outputDone = true; - } - - currentEvent.Reset(); + DecodeAndAppendChars(outputDecoder, Array.Empty(), 0, 0, flush: true, ref outputCharBuffer, ref outputCharEnd); + EmitRemainingCharsAsLine(outputCharBuffer, ref outputCharStart, ref outputCharEnd, false, lines); + outputDone = true; } + + currentEvent.Reset(); } } } else { - // EOF: pipe write end was closed. - { - ref int currentStartIndex = ref (isError ? ref errorStartIndex : ref outputStartIndex); - ref int currentEndIndex = ref (isError ? ref errorEndIndex : ref outputEndIndex); - ref byte[] currentBuffer = ref (isError ? ref errorBuffer : ref outputBuffer); - - EmitRemainingAsLine(currentBuffer, ref currentStartIndex, ref currentEndIndex, - currentEncoding, isError, lines); - } - + // EOF: flush decoder and emit remaining chars. if (isError) { + DecodeAndAppendChars(errorDecoder, Array.Empty(), 0, 0, flush: true, ref errorCharBuffer, ref errorCharEnd); + EmitRemainingCharsAsLine(errorCharBuffer, ref errorCharStart, ref errorCharEnd, true, lines); errorDone = true; } else { + DecodeAndAppendChars(outputDecoder, Array.Empty(), 0, 0, flush: true, ref outputCharBuffer, ref outputCharEnd); + EmitRemainingCharsAsLine(outputCharBuffer, ref outputCharStart, ref outputCharEnd, false, lines); outputDone = true; } @@ -191,11 +185,13 @@ private IEnumerable ReadPipesToLines( { if (outputOverlappedNint != 0) { + CancelPendingIOIfNeeded(outputHandle, outputDone, (NativeOverlapped*)outputOverlappedNint); NativeMemory.Free((void*)outputOverlappedNint); } if (errorOverlappedNint != 0) { + CancelPendingIOIfNeeded(errorHandle, errorDone, (NativeOverlapped*)errorOverlappedNint); NativeMemory.Free((void*)errorOverlappedNint); } } @@ -215,8 +211,10 @@ private IEnumerable ReadPipesToLines( errorHandle.DangerousRelease(); } - ArrayPool.Shared.Return(outputBuffer); - ArrayPool.Shared.Return(errorBuffer); + ArrayPool.Shared.Return(outputByteBuffer); + ArrayPool.Shared.Return(errorByteBuffer); + ArrayPool.Shared.Return(outputCharBuffer); + ArrayPool.Shared.Return(errorCharBuffer); } } diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs index 0acd91471cc3b0..b6b80d4ec80a17 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs @@ -156,36 +156,65 @@ public IEnumerable ReadAllLines(TimeSpan? timeout = default) } /// - /// Scans the buffer from to for complete lines - /// (delimited by \n), adds each as a to , - /// and advances past the consumed data. Handles both \r\n and \n - /// line endings. + /// Decodes bytes from the byte buffer using the and appends the + /// resulting characters to the char buffer, growing it if necessary. + /// To flush the decoder at EOF, pass an empty byte array with set to + /// . /// - private static void ParseLinesFromBuffer( - byte[] buffer, + private static void DecodeAndAppendChars( + Decoder decoder, + byte[] byteBuffer, + int byteIndex, + int byteCount, + bool flush, + ref char[] charBuffer, + ref int charEndIndex) + { + int charCount = decoder.GetCharCount(byteBuffer, byteIndex, byteCount, flush); + if (charCount == 0) + { + return; + } + + while (charEndIndex + charCount > charBuffer.Length) + { + RentLargerCharBuffer(ref charBuffer, charEndIndex); + } + + int decoded = decoder.GetChars(byteBuffer, byteIndex, byteCount, charBuffer, charEndIndex, flush); + charEndIndex += decoded; + } + + /// + /// Scans the char buffer from to for complete + /// lines (delimited by \n), adds each as a to + /// , and advances past the consumed data. + /// Handles both \r\n and \n line endings. + /// + private static void ParseLinesFromCharBuffer( + char[] buffer, ref int startIndex, int endIndex, - Encoding encoding, bool standardError, List lines) { while (startIndex < endIndex) { int remaining = endIndex - startIndex; - int lineEnd = buffer.AsSpan(startIndex, remaining).IndexOf((byte)'\n'); + int lineEnd = buffer.AsSpan(startIndex, remaining).IndexOf('\n'); if (lineEnd == -1) { break; } int contentLength = lineEnd; - if (contentLength > 0 && buffer[startIndex + contentLength - 1] == (byte)'\r') + if (contentLength > 0 && buffer[startIndex + contentLength - 1] == '\r') { contentLength--; } lines.Add(new ProcessOutputLine( - encoding.GetString(buffer, startIndex, contentLength), + new string(buffer, startIndex, contentLength), standardError)); startIndex += lineEnd + 1; @@ -193,20 +222,19 @@ private static void ParseLinesFromBuffer( } /// - /// Emits any remaining data in the buffer as a final line when an EOF is reached. + /// Emits any remaining characters in the buffer as a final line when an EOF is reached. /// - private static void EmitRemainingAsLine( - byte[] buffer, + private static void EmitRemainingCharsAsLine( + char[] buffer, ref int startIndex, ref int endIndex, - Encoding encoding, bool standardError, List lines) { if (startIndex < endIndex) { int length = endIndex - startIndex; - if (length > 0 && buffer[startIndex + length - 1] == (byte)'\r') + if (length > 0 && buffer[startIndex + length - 1] == '\r') { length--; } @@ -214,7 +242,7 @@ private static void EmitRemainingAsLine( if (length > 0) { lines.Add(new ProcessOutputLine( - encoding.GetString(buffer, startIndex, length), + new string(buffer, startIndex, length), standardError)); } @@ -224,10 +252,10 @@ private static void EmitRemainingAsLine( } /// - /// After line parsing, compacts remaining data to the front of the buffer if it has reached + /// After line parsing, compacts remaining data to the front of the char buffer if it has reached /// the end, or rents a larger buffer if the entire buffer is filled with a single incomplete line. /// - private static void CompactOrGrowLineBuffer(ref byte[] buffer, ref int startIndex, ref int endIndex) + private static void CompactOrGrowCharBuffer(ref char[] buffer, ref int startIndex, ref int endIndex) { if (endIndex < buffer.Length) { @@ -239,18 +267,32 @@ private static void CompactOrGrowLineBuffer(ref byte[] buffer, ref int startInde if (remaining == buffer.Length) { // The buffer is too small to hold a single line — grow it. - RentLargerBuffer(ref buffer, remaining); + RentLargerCharBuffer(ref buffer, remaining); } else { // Compact: move remaining data to the start of the buffer. - Buffer.BlockCopy(buffer, startIndex, buffer, 0, remaining); + Array.Copy(buffer, startIndex, buffer, 0, remaining); } startIndex = 0; endIndex = remaining; } + /// + /// Rents a larger char buffer from the array pool, copies existing data, and returns the old buffer. + /// + private static void RentLargerCharBuffer(ref char[] buffer, int charsUsed) + { + int newSize = (int)Math.Min((long)buffer.Length * 2, Array.MaxLength); + newSize = Math.Max(buffer.Length + 1, newSize); + char[] newBuffer = ArrayPool.Shared.Rent(newSize); + Array.Copy(buffer, newBuffer, charsUsed); + char[] oldBuffer = buffer; + buffer = newBuffer; + ArrayPool.Shared.Return(oldBuffer); + } + /// /// Asynchronously reads all standard output and standard error of the process as text. /// From 1665be62a304b167388b76d78ddaf2b3a2451fdc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 19 Apr 2026 12:17:47 +0000 Subject: [PATCH 04/25] Remove unused using System.Runtime.InteropServices from Process.Multiplexing.Unix.cs Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/a9eadcb8-dd77-4933-9877-920b21513e25 Co-authored-by: adamsitnik <6011991+adamsitnik@users.noreply.github.com> --- .../src/System/Diagnostics/Process.Multiplexing.Unix.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs index 1f6adb357988a0..cae438134628f1 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs @@ -6,7 +6,6 @@ using System.ComponentModel; using System.IO; using System.IO.Pipes; -using System.Runtime.InteropServices; using System.Text; using Microsoft.Win32.SafeHandles; From 711986f0316ab91be8686333f75353cd876fbb6a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 19 Apr 2026 12:47:12 +0000 Subject: [PATCH 05/25] Add BOM-stripping for sync ReadAllLines and encoding tests for UTF-8/UTF-16/UTF-32 Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/4a5f20d7-dcd7-415c-b5c8-e15806a7e5c0 Co-authored-by: adamsitnik <6011991+adamsitnik@users.noreply.github.com> --- .../Diagnostics/Process.Multiplexing.Unix.cs | 11 +++++ .../Process.Multiplexing.Windows.cs | 11 +++++ .../Diagnostics/Process.Multiplexing.cs | 13 ++++++ .../tests/ProcessStreamingTests.cs | 40 +++++++++++++++++++ .../tests/RemotelyInvokable.cs | 16 ++++++++ 5 files changed, 91 insertions(+) diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs index cae438134628f1..d4a12dbb8983af 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs @@ -59,6 +59,7 @@ private IEnumerable ReadPipesToLines( int outputCharStart = 0, outputCharEnd = 0; int errorCharStart = 0, errorCharEnd = 0; bool outputDone = false, errorDone = false; + bool outputBomChecked = false, errorBomChecked = false; List lines = new(); @@ -125,6 +126,11 @@ private IEnumerable ReadPipesToLines( if (bytesRead > 0) { DecodeAndAppendChars(errorDecoder, errorByteBuffer, 0, bytesRead, flush: false, ref errorCharBuffer, ref errorCharEnd); + if (!errorBomChecked && errorCharEnd > 0) + { + SkipBomIfPresent(errorCharBuffer, errorCharEnd, ref errorCharStart); + errorBomChecked = true; + } ParseLinesFromCharBuffer(errorCharBuffer, ref errorCharStart, errorCharEnd, true, lines); CompactOrGrowCharBuffer(ref errorCharBuffer, ref errorCharStart, ref errorCharEnd); } @@ -142,6 +148,11 @@ private IEnumerable ReadPipesToLines( if (bytesRead > 0) { DecodeAndAppendChars(outputDecoder, outputByteBuffer, 0, bytesRead, flush: false, ref outputCharBuffer, ref outputCharEnd); + if (!outputBomChecked && outputCharEnd > 0) + { + SkipBomIfPresent(outputCharBuffer, outputCharEnd, ref outputCharStart); + outputBomChecked = true; + } ParseLinesFromCharBuffer(outputCharBuffer, ref outputCharStart, outputCharEnd, false, lines); CompactOrGrowCharBuffer(ref outputCharBuffer, ref outputCharStart, ref outputCharEnd); } diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs index 28c2bcddfbbe31..be917ecc00676a 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs @@ -64,6 +64,7 @@ private IEnumerable ReadPipesToLines( int outputCharStart = 0, outputCharEnd = 0; int errorCharStart = 0, errorCharEnd = 0; + bool outputBomChecked = false, errorBomChecked = false; unsafe { @@ -110,12 +111,22 @@ private IEnumerable ReadPipesToLines( if (isError) { DecodeAndAppendChars(errorDecoder, errorByteBuffer, 0, bytesRead, flush: false, ref errorCharBuffer, ref errorCharEnd); + if (!errorBomChecked && errorCharEnd > 0) + { + SkipBomIfPresent(errorCharBuffer, errorCharEnd, ref errorCharStart); + errorBomChecked = true; + } ParseLinesFromCharBuffer(errorCharBuffer, ref errorCharStart, errorCharEnd, true, lines); CompactOrGrowCharBuffer(ref errorCharBuffer, ref errorCharStart, ref errorCharEnd); } else { DecodeAndAppendChars(outputDecoder, outputByteBuffer, 0, bytesRead, flush: false, ref outputCharBuffer, ref outputCharEnd); + if (!outputBomChecked && outputCharEnd > 0) + { + SkipBomIfPresent(outputCharBuffer, outputCharEnd, ref outputCharStart); + outputBomChecked = true; + } ParseLinesFromCharBuffer(outputCharBuffer, ref outputCharStart, outputCharEnd, false, lines); CompactOrGrowCharBuffer(ref outputCharBuffer, ref outputCharStart, ref outputCharEnd); } diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs index b6b80d4ec80a17..c151aa5a0c5844 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs @@ -185,6 +185,19 @@ private static void DecodeAndAppendChars( charEndIndex += decoded; } + /// + /// If the first character at is the Unicode BOM (U+FEFF), + /// advances past it. Called once per stream after the first + /// decode that produces characters, to match BOM-stripping behavior. + /// + private static void SkipBomIfPresent(char[] charBuffer, int endIndex, ref int startIndex) + { + if (startIndex < endIndex && charBuffer[startIndex] == '\uFEFF') + { + startIndex++; + } + } + /// /// Scans the char buffer from to for complete /// lines (delimited by \n), adds each as a to diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs index debf4816ef50cd..7c3127c12ff0ca 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Generic; +using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.DotNet.RemoteExecutor; @@ -441,6 +442,45 @@ public async Task ReadAllLines_StopsCleanlyWhenConsumerBreaksEarly(bool useAsync Assert.True(process.WaitForExit(WaitInMS)); } + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData("utf-8", true)] + [InlineData("utf-8", false)] + [InlineData("utf-16", true)] + [InlineData("utf-16", false)] + [InlineData("utf-32", true)] + [InlineData("utf-32", false)] + public async Task ReadAllLines_WorksWithNonDefaultEncodings(string encodingName, bool useAsync) + { + Encoding encoding = Encoding.GetEncoding(encodingName); + + using Process process = CreateProcessPortable(RemotelyInvokable.WriteLinesToBothStreamsWithEncoding, encodingName); + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardError = true; + process.StartInfo.StandardOutputEncoding = encoding; + process.StartInfo.StandardErrorEncoding = encoding; + process.Start(); + + List capturedOutput = new(); + List capturedError = new(); + + await foreach (ProcessOutputLine line in EnumerateLines(process, useAsync)) + { + if (line.StandardError) + { + capturedError.Add(line.Content); + } + else + { + capturedOutput.Add(line.Content); + } + } + + Assert.Equal(new[] { "stdout_line" }, capturedOutput); + Assert.Equal(new[] { "stderr_line" }, capturedError); + + Assert.True(process.WaitForExit(WaitInMS)); + } + private Process StartLinePrintingProcess(string stdOutText, string stdErrText) { Process process = CreateProcess((stdOut, stdErr) => diff --git a/src/libraries/System.Diagnostics.Process/tests/RemotelyInvokable.cs b/src/libraries/System.Diagnostics.Process/tests/RemotelyInvokable.cs index a61596b687e5cc..c002adae5938bb 100644 --- a/src/libraries/System.Diagnostics.Process/tests/RemotelyInvokable.cs +++ b/src/libraries/System.Diagnostics.Process/tests/RemotelyInvokable.cs @@ -135,6 +135,22 @@ public static int ConcatThreeArguments(string one, string two, string three) return SuccessExitCode; } + public static int WriteLinesToBothStreamsWithEncoding(string encodingName) + { + Encoding encoding = Encoding.GetEncoding(encodingName); + using (var outputWriter = new StreamWriter(Console.OpenStandardOutput(), encoding)) + { + outputWriter.WriteLine("stdout_line"); + } + + using (var errorWriter = new StreamWriter(Console.OpenStandardError(), encoding)) + { + errorWriter.WriteLine("stderr_line"); + } + + return SuccessExitCode; + } + public static int SelfTerminate() { Process.GetCurrentProcess().Kill(); From 0dcc7f85bec7a821c8d85baeec64a89cd24815e2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 19 Apr 2026 12:48:51 +0000 Subject: [PATCH 06/25] Add XML param docs to SkipBomIfPresent Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/4a5f20d7-dcd7-415c-b5c8-e15806a7e5c0 Co-authored-by: adamsitnik <6011991+adamsitnik@users.noreply.github.com> --- .../src/System/Diagnostics/Process.Multiplexing.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs index c151aa5a0c5844..19731bfb957422 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs @@ -190,6 +190,11 @@ private static void DecodeAndAppendChars( /// advances past it. Called once per stream after the first /// decode that produces characters, to match BOM-stripping behavior. /// + /// The decoded character buffer. + /// Exclusive upper bound of valid characters in . + /// + /// Current read position in ; advanced by one if a BOM is found. + /// private static void SkipBomIfPresent(char[] charBuffer, int endIndex, ref int startIndex) { if (startIndex < endIndex && charBuffer[startIndex] == '\uFEFF') From 7fdd0eae0255d2b6a40a9f35c16646b257c167d8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 19 Apr 2026 17:17:43 +0000 Subject: [PATCH 07/25] Optimize DecodeAndAppendChars to compact before growing when startIndex > 0 Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/74094051-584c-4273-ac2d-0adb99b1e933 Co-authored-by: adamsitnik <6011991+adamsitnik@users.noreply.github.com> --- .../System/Diagnostics/Process.Multiplexing.Unix.cs | 8 ++++---- .../Diagnostics/Process.Multiplexing.Windows.cs | 12 ++++++------ .../src/System/Diagnostics/Process.Multiplexing.cs | 11 +++++++++++ 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs index d4a12dbb8983af..508a51857c46c1 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs @@ -125,7 +125,7 @@ private IEnumerable ReadPipesToLines( int bytesRead = ReadNonBlocking(currentHandle, errorByteBuffer, 0); if (bytesRead > 0) { - DecodeAndAppendChars(errorDecoder, errorByteBuffer, 0, bytesRead, flush: false, ref errorCharBuffer, ref errorCharEnd); + DecodeAndAppendChars(errorDecoder, errorByteBuffer, 0, bytesRead, flush: false, ref errorCharBuffer, ref errorCharStart, ref errorCharEnd); if (!errorBomChecked && errorCharEnd > 0) { SkipBomIfPresent(errorCharBuffer, errorCharEnd, ref errorCharStart); @@ -136,7 +136,7 @@ private IEnumerable ReadPipesToLines( } else if (bytesRead == 0) { - DecodeAndAppendChars(errorDecoder, Array.Empty(), 0, 0, flush: true, ref errorCharBuffer, ref errorCharEnd); + DecodeAndAppendChars(errorDecoder, Array.Empty(), 0, 0, flush: true, ref errorCharBuffer, ref errorCharStart, ref errorCharEnd); EmitRemainingCharsAsLine(errorCharBuffer, ref errorCharStart, ref errorCharEnd, true, lines); errorDone = true; } @@ -147,7 +147,7 @@ private IEnumerable ReadPipesToLines( int bytesRead = ReadNonBlocking(currentHandle, outputByteBuffer, 0); if (bytesRead > 0) { - DecodeAndAppendChars(outputDecoder, outputByteBuffer, 0, bytesRead, flush: false, ref outputCharBuffer, ref outputCharEnd); + DecodeAndAppendChars(outputDecoder, outputByteBuffer, 0, bytesRead, flush: false, ref outputCharBuffer, ref outputCharStart, ref outputCharEnd); if (!outputBomChecked && outputCharEnd > 0) { SkipBomIfPresent(outputCharBuffer, outputCharEnd, ref outputCharStart); @@ -158,7 +158,7 @@ private IEnumerable ReadPipesToLines( } else if (bytesRead == 0) { - DecodeAndAppendChars(outputDecoder, Array.Empty(), 0, 0, flush: true, ref outputCharBuffer, ref outputCharEnd); + DecodeAndAppendChars(outputDecoder, Array.Empty(), 0, 0, flush: true, ref outputCharBuffer, ref outputCharStart, ref outputCharEnd); EmitRemainingCharsAsLine(outputCharBuffer, ref outputCharStart, ref outputCharEnd, false, lines); outputDone = true; } diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs index be917ecc00676a..22f3c4e8e1bb59 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs @@ -110,7 +110,7 @@ private IEnumerable ReadPipesToLines( // Decode bytes to chars and parse lines. if (isError) { - DecodeAndAppendChars(errorDecoder, errorByteBuffer, 0, bytesRead, flush: false, ref errorCharBuffer, ref errorCharEnd); + DecodeAndAppendChars(errorDecoder, errorByteBuffer, 0, bytesRead, flush: false, ref errorCharBuffer, ref errorCharStart, ref errorCharEnd); if (!errorBomChecked && errorCharEnd > 0) { SkipBomIfPresent(errorCharBuffer, errorCharEnd, ref errorCharStart); @@ -121,7 +121,7 @@ private IEnumerable ReadPipesToLines( } else { - DecodeAndAppendChars(outputDecoder, outputByteBuffer, 0, bytesRead, flush: false, ref outputCharBuffer, ref outputCharEnd); + DecodeAndAppendChars(outputDecoder, outputByteBuffer, 0, bytesRead, flush: false, ref outputCharBuffer, ref outputCharStart, ref outputCharEnd); if (!outputBomChecked && outputCharEnd > 0) { SkipBomIfPresent(outputCharBuffer, outputCharEnd, ref outputCharStart); @@ -147,13 +147,13 @@ private IEnumerable ReadPipesToLines( // EOF during QueueRead — flush decoder and emit remaining. if (isError) { - DecodeAndAppendChars(errorDecoder, Array.Empty(), 0, 0, flush: true, ref errorCharBuffer, ref errorCharEnd); + DecodeAndAppendChars(errorDecoder, Array.Empty(), 0, 0, flush: true, ref errorCharBuffer, ref errorCharStart, ref errorCharEnd); EmitRemainingCharsAsLine(errorCharBuffer, ref errorCharStart, ref errorCharEnd, true, lines); errorDone = true; } else { - DecodeAndAppendChars(outputDecoder, Array.Empty(), 0, 0, flush: true, ref outputCharBuffer, ref outputCharEnd); + DecodeAndAppendChars(outputDecoder, Array.Empty(), 0, 0, flush: true, ref outputCharBuffer, ref outputCharStart, ref outputCharEnd); EmitRemainingCharsAsLine(outputCharBuffer, ref outputCharStart, ref outputCharEnd, false, lines); outputDone = true; } @@ -167,13 +167,13 @@ private IEnumerable ReadPipesToLines( // EOF: flush decoder and emit remaining chars. if (isError) { - DecodeAndAppendChars(errorDecoder, Array.Empty(), 0, 0, flush: true, ref errorCharBuffer, ref errorCharEnd); + DecodeAndAppendChars(errorDecoder, Array.Empty(), 0, 0, flush: true, ref errorCharBuffer, ref errorCharStart, ref errorCharEnd); EmitRemainingCharsAsLine(errorCharBuffer, ref errorCharStart, ref errorCharEnd, true, lines); errorDone = true; } else { - DecodeAndAppendChars(outputDecoder, Array.Empty(), 0, 0, flush: true, ref outputCharBuffer, ref outputCharEnd); + DecodeAndAppendChars(outputDecoder, Array.Empty(), 0, 0, flush: true, ref outputCharBuffer, ref outputCharStart, ref outputCharEnd); EmitRemainingCharsAsLine(outputCharBuffer, ref outputCharStart, ref outputCharEnd, false, lines); outputDone = true; } diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs index 19731bfb957422..b87a5d8dabd03d 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs @@ -168,6 +168,7 @@ private static void DecodeAndAppendChars( int byteCount, bool flush, ref char[] charBuffer, + ref int charStartIndex, ref int charEndIndex) { int charCount = decoder.GetCharCount(byteBuffer, byteIndex, byteCount, flush); @@ -176,6 +177,16 @@ private static void DecodeAndAppendChars( return; } + // If there isn't enough room at the end but there's free space at the start + // (from already-consumed data), compact first to avoid unnecessary buffer growth. + if (charEndIndex + charCount > charBuffer.Length && charStartIndex > 0) + { + int remaining = charEndIndex - charStartIndex; + Array.Copy(charBuffer, charStartIndex, charBuffer, 0, remaining); + charStartIndex = 0; + charEndIndex = remaining; + } + while (charEndIndex + charCount > charBuffer.Length) { RentLargerCharBuffer(ref charBuffer, charEndIndex); From 421cb5df5af7589ff48e5e54a867284550115a40 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 19 Apr 2026 17:19:45 +0000 Subject: [PATCH 08/25] Only compact char buffer when it frees enough space to avoid wasteful copy before resize Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/74094051-584c-4273-ac2d-0adb99b1e933 Co-authored-by: adamsitnik <6011991+adamsitnik@users.noreply.github.com> --- .../src/System/Diagnostics/Process.Multiplexing.cs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs index b87a5d8dabd03d..8d84c85126bee5 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs @@ -177,14 +177,17 @@ private static void DecodeAndAppendChars( return; } - // If there isn't enough room at the end but there's free space at the start - // (from already-consumed data), compact first to avoid unnecessary buffer growth. + // If there isn't enough room at the end but compacting the consumed space at the start + // would free enough room, compact to avoid unnecessary buffer growth. if (charEndIndex + charCount > charBuffer.Length && charStartIndex > 0) { int remaining = charEndIndex - charStartIndex; - Array.Copy(charBuffer, charStartIndex, charBuffer, 0, remaining); - charStartIndex = 0; - charEndIndex = remaining; + if (remaining + charCount <= charBuffer.Length) + { + Array.Copy(charBuffer, charStartIndex, charBuffer, 0, remaining); + charStartIndex = 0; + charEndIndex = remaining; + } } while (charEndIndex + charCount > charBuffer.Length) From 6436c51b125a35bbf45e86ce9d0ba97c709170e9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 24 Apr 2026 07:07:29 +0000 Subject: [PATCH 09/25] Address review feedback: handle \r line endings, make RentLargerBuffer generic, extract Unix helpers, pass decoders, fix var usages, update tests Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/cc963aa8-4f4e-4f1b-a990-1867ed535441 Co-authored-by: adamsitnik <6011991+adamsitnik@users.noreply.github.com> --- .../Diagnostics/Process.Multiplexing.Unix.cs | 204 +++++++++--------- .../Process.Multiplexing.Windows.cs | 9 +- .../Diagnostics/Process.Multiplexing.cs | 89 ++++---- .../tests/ProcessStreamingTests.cs | 102 ++++----- .../tests/RemotelyInvokable.cs | 4 +- 5 files changed, 192 insertions(+), 216 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs index 508a51857c46c1..1d8cbeee5bb70a 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs @@ -22,8 +22,8 @@ public partial class Process /// private IEnumerable ReadPipesToLines( int timeoutMs, - Encoding outputEncoding, - Encoding errorEncoding) + Decoder outputDecoder, + Decoder errorDecoder) { SafePipeHandle outputHandle = GetSafeHandleFromStreamReader(_standardOutput!); SafePipeHandle errorHandle = GetSafeHandleFromStreamReader(_standardError!); @@ -51,9 +51,6 @@ private IEnumerable ReadPipesToLines( // Cannot use stackalloc in an iterator method; use a regular array. Interop.PollEvent[] pollFds = new Interop.PollEvent[2]; - Decoder outputDecoder = outputEncoding.GetDecoder(); - Decoder errorDecoder = errorEncoding.GetDecoder(); - long deadline = timeoutMs >= 0 ? Environment.TickCount64 + timeoutMs : long.MaxValue; int outputCharStart = 0, outputCharEnd = 0; @@ -65,27 +62,7 @@ private IEnumerable ReadPipesToLines( while (!outputDone || !errorDone) { - int numFds = 0; - int outputIndex = -1; - int errorIndex = -1; - - if (!errorDone) - { - errorIndex = numFds; - pollFds[numFds].FileDescriptor = errorFd; - pollFds[numFds].Events = Interop.PollEvents.POLLIN; - pollFds[numFds].TriggeredEvents = Interop.PollEvents.POLLNONE; - numFds++; - } - - if (!outputDone) - { - outputIndex = numFds; - pollFds[numFds].FileDescriptor = outputFd; - pollFds[numFds].Events = Interop.PollEvents.POLLIN; - pollFds[numFds].TriggeredEvents = Interop.PollEvents.POLLNONE; - numFds++; - } + int numFds = PreparePollFds(pollFds, errorFd, outputFd, errorDone, outputDone, out int errorIndex, out int outputIndex); if (!TryGetRemainingTimeout(deadline, timeoutMs, out int pollTimeout)) { @@ -122,47 +99,15 @@ private IEnumerable ReadPipesToLines( // Use explicit branching to avoid ref locals across yield points. if (isError) { - int bytesRead = ReadNonBlocking(currentHandle, errorByteBuffer, 0); - if (bytesRead > 0) - { - DecodeAndAppendChars(errorDecoder, errorByteBuffer, 0, bytesRead, flush: false, ref errorCharBuffer, ref errorCharStart, ref errorCharEnd); - if (!errorBomChecked && errorCharEnd > 0) - { - SkipBomIfPresent(errorCharBuffer, errorCharEnd, ref errorCharStart); - errorBomChecked = true; - } - ParseLinesFromCharBuffer(errorCharBuffer, ref errorCharStart, errorCharEnd, true, lines); - CompactOrGrowCharBuffer(ref errorCharBuffer, ref errorCharStart, ref errorCharEnd); - } - else if (bytesRead == 0) - { - DecodeAndAppendChars(errorDecoder, Array.Empty(), 0, 0, flush: true, ref errorCharBuffer, ref errorCharStart, ref errorCharEnd); - EmitRemainingCharsAsLine(errorCharBuffer, ref errorCharStart, ref errorCharEnd, true, lines); - errorDone = true; - } - // bytesRead < 0 means EAGAIN — nothing available yet, let poll retry. + HandlePipeLineRead(currentHandle, errorDecoder, errorByteBuffer, + ref errorCharBuffer, ref errorCharStart, ref errorCharEnd, + ref errorBomChecked, ref errorDone, standardError: true, lines); } else { - int bytesRead = ReadNonBlocking(currentHandle, outputByteBuffer, 0); - if (bytesRead > 0) - { - DecodeAndAppendChars(outputDecoder, outputByteBuffer, 0, bytesRead, flush: false, ref outputCharBuffer, ref outputCharStart, ref outputCharEnd); - if (!outputBomChecked && outputCharEnd > 0) - { - SkipBomIfPresent(outputCharBuffer, outputCharEnd, ref outputCharStart); - outputBomChecked = true; - } - ParseLinesFromCharBuffer(outputCharBuffer, ref outputCharStart, outputCharEnd, false, lines); - CompactOrGrowCharBuffer(ref outputCharBuffer, ref outputCharStart, ref outputCharEnd); - } - else if (bytesRead == 0) - { - DecodeAndAppendChars(outputDecoder, Array.Empty(), 0, 0, flush: true, ref outputCharBuffer, ref outputCharStart, ref outputCharEnd); - EmitRemainingCharsAsLine(outputCharBuffer, ref outputCharStart, ref outputCharEnd, false, lines); - outputDone = true; - } - // bytesRead < 0 means EAGAIN — nothing available yet, let poll retry. + HandlePipeLineRead(currentHandle, outputDecoder, outputByteBuffer, + ref outputCharBuffer, ref outputCharStart, ref outputCharEnd, + ref outputBomChecked, ref outputDone, standardError: false, lines); } } @@ -194,6 +139,80 @@ private IEnumerable ReadPipesToLines( } } + /// + /// Populates the poll fd array with the active pipe file descriptors. + /// Error is added first so it gets serviced first when both have data. + /// Returns the number of active file descriptors. + /// + private static int PreparePollFds( + Interop.PollEvent[] pollFds, + int errorFd, int outputFd, + bool errorDone, bool outputDone, + out int errorIndex, out int outputIndex) + { + int numFds = 0; + errorIndex = -1; + outputIndex = -1; + + if (!errorDone) + { + errorIndex = numFds; + pollFds[numFds].FileDescriptor = errorFd; + pollFds[numFds].Events = Interop.PollEvents.POLLIN; + pollFds[numFds].TriggeredEvents = Interop.PollEvents.POLLNONE; + numFds++; + } + + if (!outputDone) + { + outputIndex = numFds; + pollFds[numFds].FileDescriptor = outputFd; + pollFds[numFds].Events = Interop.PollEvents.POLLIN; + pollFds[numFds].TriggeredEvents = Interop.PollEvents.POLLNONE; + numFds++; + } + + return numFds; + } + + /// + /// Handles a poll notification for a single pipe: reads bytes, decodes to chars, + /// strips BOM on first decode, parses lines, compacts the char buffer, and sets + /// to on EOF. + /// + private static void HandlePipeLineRead( + SafePipeHandle handle, + Decoder decoder, + byte[] byteBuffer, + ref char[] charBuffer, + ref int charStart, + ref int charEnd, + ref bool bomChecked, + ref bool done, + bool standardError, + List lines) + { + int bytesRead = ReadNonBlocking(handle, byteBuffer, 0); + if (bytesRead > 0) + { + DecodeAndAppendChars(decoder, byteBuffer, 0, bytesRead, flush: false, ref charBuffer, ref charStart, ref charEnd); + if (!bomChecked && charEnd > 0) + { + SkipBomIfPresent(charBuffer, charEnd, ref charStart); + bomChecked = true; + } + ParseLinesFromCharBuffer(charBuffer, ref charStart, charEnd, standardError, lines); + CompactOrGrowCharBuffer(ref charBuffer, ref charStart, ref charEnd); + } + else if (bytesRead == 0) + { + DecodeAndAppendChars(decoder, Array.Empty(), 0, 0, flush: true, ref charBuffer, ref charStart, ref charEnd); + EmitRemainingCharsAsLine(charBuffer, ref charStart, ref charEnd, standardError, lines); + done = true; + } + // bytesRead < 0 means EAGAIN — nothing available yet, let poll retry. + } + /// /// Calls poll(2) on the provided array of poll events. /// @@ -231,7 +250,7 @@ private static void ReadPipes( throw new Win32Exception(); } - Span pollFds = stackalloc Interop.PollEvent[2]; + Interop.PollEvent[] pollFds = new Interop.PollEvent[2]; long deadline = timeoutMs >= 0 ? Environment.TickCount64 + timeoutMs @@ -240,58 +259,27 @@ private static void ReadPipes( bool outputDone = false, errorDone = false; while (!outputDone || !errorDone) { - int numFds = 0; - - int outputIndex = -1; - int errorIndex = -1; + int numFds = PreparePollFds(pollFds, errorFd, outputFd, errorDone, outputDone, out int errorIndex, out int outputIndex); - if (!outputDone) + if (!TryGetRemainingTimeout(deadline, timeoutMs, out int pollTimeout)) { - outputIndex = numFds; - pollFds[numFds].FileDescriptor = outputFd; - pollFds[numFds].Events = Interop.PollEvents.POLLIN; - pollFds[numFds].TriggeredEvents = Interop.PollEvents.POLLNONE; - numFds++; + throw new TimeoutException(); } - if (!errorDone) + Interop.Error pollError = PollPipes(pollFds, numFds, pollTimeout, out uint triggered); + if (pollError != Interop.Error.SUCCESS) { - errorIndex = numFds; - pollFds[numFds].FileDescriptor = errorFd; - pollFds[numFds].Events = Interop.PollEvents.POLLIN; - pollFds[numFds].TriggeredEvents = Interop.PollEvents.POLLNONE; - numFds++; - } + if (pollError == Interop.Error.EINTR) + { + continue; + } - int pollTimeout; - if (!TryGetRemainingTimeout(deadline, timeoutMs, out pollTimeout)) - { - throw new TimeoutException(); + throw new Win32Exception(Interop.Sys.ConvertErrorPalToPlatform(pollError)); } - unsafe + if (triggered == 0) { - uint triggered; - fixed (Interop.PollEvent* pPollFds = pollFds) - { - Interop.Error error = Interop.Sys.Poll(pPollFds, (uint)numFds, pollTimeout, &triggered); - if (error != Interop.Error.SUCCESS) - { - if (error == Interop.Error.EINTR) - { - // We don't re-issue the poll immediately because we need to check - // if we've already exceeded the overall timeout. - continue; - } - - throw new Win32Exception(Interop.Sys.ConvertErrorPalToPlatform(error)); - } - - if (triggered == 0) - { - throw new TimeoutException(); - } - } + throw new TimeoutException(); } for (int i = 0; i < numFds; i++) diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs index 22f3c4e8e1bb59..87c509735e8697 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs @@ -23,8 +23,8 @@ public partial class Process /// private IEnumerable ReadPipesToLines( int timeoutMs, - Encoding outputEncoding, - Encoding errorEncoding) + Decoder outputDecoder, + Decoder errorDecoder) { SafeFileHandle outputHandle = GetSafeHandleFromStreamReader(_standardOutput!); SafeFileHandle errorHandle = GetSafeHandleFromStreamReader(_standardError!); @@ -35,6 +35,8 @@ private IEnumerable ReadPipesToLines( char[] errorCharBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); bool outputRefAdded = false, errorRefAdded = false; PinnedGCHandle outputPin = default, errorPin = default; + // NativeOverlapped* can't be used as iterator state machine fields (pointers aren't + // allowed in managed types). Store as nint and cast back inside scoped unsafe blocks. nint outputOverlappedNint = 0, errorOverlappedNint = 0; EventWaitHandle? outputEvent = null, errorEvent = null; bool outputDone = true, errorDone = true; @@ -59,9 +61,6 @@ private IEnumerable ReadPipesToLines( // Error output gets index 0 so WaitAny services it first when both are signaled. WaitHandle[] waitHandles = [errorEvent, outputEvent]; - Decoder outputDecoder = outputEncoding.GetDecoder(); - Decoder errorDecoder = errorEncoding.GetDecoder(); - int outputCharStart = 0, outputCharEnd = 0; int errorCharStart = 0, errorCharEnd = 0; bool outputBomChecked = false, errorBomChecked = false; diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs index 8d84c85126bee5..769e86ec426171 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs @@ -152,7 +152,7 @@ public IEnumerable ReadAllLines(TimeSpan? timeout = default) Encoding outputEncoding = _startInfo?.StandardOutputEncoding ?? GetStandardOutputEncoding(); Encoding errorEncoding = _startInfo?.StandardErrorEncoding ?? GetStandardOutputEncoding(); - return ReadPipesToLines(timeoutMs, outputEncoding, errorEncoding); + return ReadPipesToLines(timeoutMs, outputEncoding.GetDecoder(), errorEncoding.GetDecoder()); } /// @@ -192,7 +192,7 @@ private static void DecodeAndAppendChars( while (charEndIndex + charCount > charBuffer.Length) { - RentLargerCharBuffer(ref charBuffer, charEndIndex); + RentLargerBuffer(ref charBuffer, charEndIndex); } int decoded = decoder.GetChars(byteBuffer, byteIndex, byteCount, charBuffer, charEndIndex, flush); @@ -219,9 +219,10 @@ private static void SkipBomIfPresent(char[] charBuffer, int endIndex, ref int st /// /// Scans the char buffer from to for complete - /// lines (delimited by \n), adds each as a to - /// , and advances past the consumed data. - /// Handles both \r\n and \n line endings. + /// lines (delimited by \r, \n, or \r\n), adds each as a + /// to , and advances + /// past the consumed data. + /// This matches behavior used by the async path. /// private static void ParseLinesFromCharBuffer( char[] buffer, @@ -233,28 +234,51 @@ private static void ParseLinesFromCharBuffer( while (startIndex < endIndex) { int remaining = endIndex - startIndex; - int lineEnd = buffer.AsSpan(startIndex, remaining).IndexOf('\n'); + int lineEnd = buffer.AsSpan(startIndex, remaining).IndexOfAny('\r', '\n'); if (lineEnd == -1) { break; } - int contentLength = lineEnd; - if (contentLength > 0 && buffer[startIndex + contentLength - 1] == '\r') + char terminator = buffer[startIndex + lineEnd]; + + // If we found '\r', we need to check for a following '\n' to treat \r\n as one terminator. + // If '\n' isn't available yet (end of current data), stop and wait for more data. + if (terminator == '\r') { - contentLength--; - } + if (startIndex + lineEnd + 1 >= endIndex) + { + // The '\r' is at the very end of available data — we can't tell yet + // whether it's a standalone '\r' or part of '\r\n'. Wait for more data. + break; + } - lines.Add(new ProcessOutputLine( - new string(buffer, startIndex, contentLength), - standardError)); + lines.Add(new ProcessOutputLine( + new string(buffer, startIndex, lineEnd), + standardError)); + + // Skip \r and also \n if it immediately follows. + startIndex += lineEnd + 1; + if (startIndex < endIndex && buffer[startIndex] == '\n') + { + startIndex++; + } + } + else + { + // terminator == '\n' + lines.Add(new ProcessOutputLine( + new string(buffer, startIndex, lineEnd), + standardError)); - startIndex += lineEnd + 1; + startIndex += lineEnd + 1; + } } } /// /// Emits any remaining characters in the buffer as a final line when an EOF is reached. + /// A trailing \r is stripped to match behavior. /// private static void EmitRemainingCharsAsLine( char[] buffer, @@ -271,12 +295,9 @@ private static void EmitRemainingCharsAsLine( length--; } - if (length > 0) - { - lines.Add(new ProcessOutputLine( - new string(buffer, startIndex, length), - standardError)); - } + lines.Add(new ProcessOutputLine( + new string(buffer, startIndex, length), + standardError)); startIndex = 0; endIndex = 0; @@ -299,7 +320,7 @@ private static void CompactOrGrowCharBuffer(ref char[] buffer, ref int startInde if (remaining == buffer.Length) { // The buffer is too small to hold a single line — grow it. - RentLargerCharBuffer(ref buffer, remaining); + RentLargerBuffer(ref buffer, remaining); } else { @@ -311,20 +332,6 @@ private static void CompactOrGrowCharBuffer(ref char[] buffer, ref int startInde endIndex = remaining; } - /// - /// Rents a larger char buffer from the array pool, copies existing data, and returns the old buffer. - /// - private static void RentLargerCharBuffer(ref char[] buffer, int charsUsed) - { - int newSize = (int)Math.Min((long)buffer.Length * 2, Array.MaxLength); - newSize = Math.Max(buffer.Length + 1, newSize); - char[] newBuffer = ArrayPool.Shared.Rent(newSize); - Array.Copy(buffer, newBuffer, charsUsed); - char[] oldBuffer = buffer; - buffer = newBuffer; - ArrayPool.Shared.Return(oldBuffer); - } - /// /// Asynchronously reads all standard output and standard error of the process as text. /// @@ -631,17 +638,17 @@ private void ReadPipesToBuffers( } /// - /// Rents a larger buffer from the array pool and copies the existing data to it. + /// Rents a larger buffer from the array pool, copies existing data, and returns the old buffer. /// - private static void RentLargerBuffer(ref byte[] buffer, int bytesRead) + private static void RentLargerBuffer(ref T[] buffer, int dataLength) { int newSize = (int)Math.Min((long)buffer.Length * 2, Array.MaxLength); newSize = Math.Max(buffer.Length + 1, newSize); - byte[] newBuffer = ArrayPool.Shared.Rent(newSize); - Buffer.BlockCopy(buffer, 0, newBuffer, 0, bytesRead); - byte[] oldBuffer = buffer; + T[] newBuffer = ArrayPool.Shared.Rent(newSize); + Array.Copy(buffer, newBuffer, dataLength); + T[] oldBuffer = buffer; buffer = newBuffer; - ArrayPool.Shared.Return(oldBuffer); + ArrayPool.Shared.Return(oldBuffer); } private static bool TryGetRemainingTimeout(long deadline, int originalTimeout, out int remainingTimeoutMs) diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs index 7c3127c12ff0ca..de0c95b15d8b5f 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs @@ -216,17 +216,7 @@ public async Task ReadAllLines_ReadsBothOutputAndError(string standardOutput, st List capturedOutput = new(); List capturedError = new(); - await foreach (ProcessOutputLine line in EnumerateLines(process, useAsync)) - { - if (line.StandardError) - { - capturedError.Add(line.Content); - } - else - { - capturedOutput.Add(line.Content); - } - } + await EnumerateLines(process, useAsync, capturedOutput, capturedError); if (string.IsNullOrEmpty(standardOutput)) { @@ -275,17 +265,7 @@ public async Task ReadAllLines_ReadsInterleavedOutput(bool useAsync) List capturedOutput = new(); List capturedError = new(); - await foreach (ProcessOutputLine line in EnumerateLines(process, useAsync)) - { - if (line.StandardError) - { - capturedError.Add(line.Content); - } - else - { - capturedOutput.Add(line.Content); - } - } + await EnumerateLines(process, useAsync, capturedOutput, capturedError); List expectedOutput = new(); List expectedError = new(); @@ -324,17 +304,7 @@ public async Task ReadAllLines_ReadsLargeOutput(bool useAsync) List capturedOutput = new(); List capturedError = new(); - await foreach (ProcessOutputLine line in EnumerateLines(process, useAsync)) - { - if (line.StandardError) - { - capturedError.Add(line.Content); - } - else - { - capturedOutput.Add(line.Content); - } - } + await EnumerateLines(process, useAsync, capturedOutput, capturedError); for (int i = 0; i < lineCount; i++) { @@ -394,15 +364,13 @@ public async Task ReadAllLines_ProcessOutputLineProperties(bool useAsync) { using Process process = StartLinePrintingProcess("stdout_line", "stderr_line"); - List allLines = new(); + List capturedOutput = new(); + List capturedError = new(); - await foreach (ProcessOutputLine line in EnumerateLines(process, useAsync)) - { - allLines.Add(line); - } + await EnumerateLines(process, useAsync, capturedOutput, capturedError); - Assert.Single(allLines, line => line.Content == "stdout_line" && !line.StandardError); - Assert.Single(allLines, line => line.Content == "stderr_line" && line.StandardError); + Assert.Single(capturedOutput, line => line == "stdout_line"); + Assert.Single(capturedError, line => line == "stderr_line"); Assert.True(process.WaitForExit(WaitInMS)); } @@ -430,10 +398,21 @@ public async Task ReadAllLines_StopsCleanlyWhenConsumerBreaksEarly(bool useAsync ProcessOutputLine? firstLine = null; - await foreach (ProcessOutputLine line in EnumerateLines(process, useAsync)) + if (useAsync) { - firstLine = line; - break; // stop after first line + await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + { + firstLine = line; + break; + } + } + else + { + foreach (ProcessOutputLine line in process.ReadAllLines()) + { + firstLine = line; + break; + } } Assert.NotNull(firstLine); @@ -463,17 +442,7 @@ public async Task ReadAllLines_WorksWithNonDefaultEncodings(string encodingName, List capturedOutput = new(); List capturedError = new(); - await foreach (ProcessOutputLine line in EnumerateLines(process, useAsync)) - { - if (line.StandardError) - { - capturedError.Add(line.Content); - } - else - { - capturedOutput.Add(line.Content); - } - } + await EnumerateLines(process, useAsync, capturedOutput, capturedError); Assert.Equal(new[] { "stdout_line" }, capturedOutput); Assert.Equal(new[] { "stderr_line" }, capturedError); @@ -506,24 +475,37 @@ private Process StartLinePrintingProcess(string stdOutText, string stdErrText) } /// - /// Helper that wraps both the sync and async line-reading APIs into an - /// so callers can always - /// use await foreach. + /// Helper that wraps both the sync and async line-reading APIs, populating + /// the provided output and error lists. /// - private static async IAsyncEnumerable EnumerateLines(Process process, bool useAsync) + private static async Task EnumerateLines(Process process, bool useAsync, List capturedOutput, List capturedError) { if (useAsync) { await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) { - yield return line; + if (line.StandardError) + { + capturedError.Add(line.Content); + } + else + { + capturedOutput.Add(line.Content); + } } } else { foreach (ProcessOutputLine line in process.ReadAllLines()) { - yield return line; + if (line.StandardError) + { + capturedError.Add(line.Content); + } + else + { + capturedOutput.Add(line.Content); + } } } } diff --git a/src/libraries/System.Diagnostics.Process/tests/RemotelyInvokable.cs b/src/libraries/System.Diagnostics.Process/tests/RemotelyInvokable.cs index c002adae5938bb..31b96568d44dea 100644 --- a/src/libraries/System.Diagnostics.Process/tests/RemotelyInvokable.cs +++ b/src/libraries/System.Diagnostics.Process/tests/RemotelyInvokable.cs @@ -138,12 +138,12 @@ public static int ConcatThreeArguments(string one, string two, string three) public static int WriteLinesToBothStreamsWithEncoding(string encodingName) { Encoding encoding = Encoding.GetEncoding(encodingName); - using (var outputWriter = new StreamWriter(Console.OpenStandardOutput(), encoding)) + using (StreamWriter outputWriter = new(Console.OpenStandardOutput(), encoding)) { outputWriter.WriteLine("stdout_line"); } - using (var errorWriter = new StreamWriter(Console.OpenStandardError(), encoding)) + using (StreamWriter errorWriter = new(Console.OpenStandardError(), encoding)) { errorWriter.WriteLine("stderr_line"); } From 856cd0e544715fc0fcec4cf21af62b20e25d822b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 24 Apr 2026 09:02:18 +0000 Subject: [PATCH 10/25] Extract PollForPipeActivity helper to deduplicate poll logic in Unix Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/ff45ade2-4c7f-4a60-b67a-0eae98240c4a Co-authored-by: adamsitnik <6011991+adamsitnik@users.noreply.github.com> --- .../Diagnostics/Process.Multiplexing.Unix.cs | 84 ++++++++++--------- 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs index 1d8cbeee5bb70a..91c7b7ec7c5ff9 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs @@ -62,27 +62,10 @@ private IEnumerable ReadPipesToLines( while (!outputDone || !errorDone) { - int numFds = PreparePollFds(pollFds, errorFd, outputFd, errorDone, outputDone, out int errorIndex, out int outputIndex); - - if (!TryGetRemainingTimeout(deadline, timeoutMs, out int pollTimeout)) - { - throw new TimeoutException(); - } - - Interop.Error pollError = PollPipes(pollFds, numFds, pollTimeout, out uint triggered); - if (pollError != Interop.Error.SUCCESS) - { - if (pollError == Interop.Error.EINTR) - { - continue; - } - - throw new Win32Exception(Interop.Sys.ConvertErrorPalToPlatform(pollError)); - } - - if (triggered == 0) + int numFds = PollForPipeActivity(pollFds, errorFd, outputFd, errorDone, outputDone, deadline, timeoutMs, out int errorIndex, out int outputIndex); + if (numFds == 0) { - throw new TimeoutException(); + continue; // EINTR } // Process error pipe first (lower index) when both have data available. @@ -175,6 +158,44 @@ private static int PreparePollFds( return numFds; } + /// + /// Prepares the poll fd array, checks the remaining timeout, calls poll(2), and handles + /// errors. Returns the number of polled fds, or 0 if poll was interrupted (EINTR) and + /// the caller should retry. + /// + private static int PollForPipeActivity( + Interop.PollEvent[] pollFds, + int errorFd, int outputFd, + bool errorDone, bool outputDone, + long deadline, int timeoutMs, + out int errorIndex, out int outputIndex) + { + int numFds = PreparePollFds(pollFds, errorFd, outputFd, errorDone, outputDone, out errorIndex, out outputIndex); + + if (!TryGetRemainingTimeout(deadline, timeoutMs, out int pollTimeout)) + { + throw new TimeoutException(); + } + + Interop.Error pollError = PollPipes(pollFds, numFds, pollTimeout, out uint triggered); + if (pollError != Interop.Error.SUCCESS) + { + if (pollError == Interop.Error.EINTR) + { + return 0; + } + + throw new Win32Exception(Interop.Sys.ConvertErrorPalToPlatform(pollError)); + } + + if (triggered == 0) + { + throw new TimeoutException(); + } + + return numFds; + } + /// /// Handles a poll notification for a single pipe: reads bytes, decodes to chars, /// strips BOM on first decode, parses lines, compacts the char buffer, and sets @@ -259,27 +280,10 @@ private static void ReadPipes( bool outputDone = false, errorDone = false; while (!outputDone || !errorDone) { - int numFds = PreparePollFds(pollFds, errorFd, outputFd, errorDone, outputDone, out int errorIndex, out int outputIndex); - - if (!TryGetRemainingTimeout(deadline, timeoutMs, out int pollTimeout)) - { - throw new TimeoutException(); - } - - Interop.Error pollError = PollPipes(pollFds, numFds, pollTimeout, out uint triggered); - if (pollError != Interop.Error.SUCCESS) - { - if (pollError == Interop.Error.EINTR) - { - continue; - } - - throw new Win32Exception(Interop.Sys.ConvertErrorPalToPlatform(pollError)); - } - - if (triggered == 0) + int numFds = PollForPipeActivity(pollFds, errorFd, outputFd, errorDone, outputDone, deadline, timeoutMs, out int errorIndex, out int outputIndex); + if (numFds == 0) { - throw new TimeoutException(); + continue; // EINTR } for (int i = 0; i < numFds; i++) From 33b5d6a86bba6127f764c8dd0f9ee01d8f0ec54b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 24 Apr 2026 10:57:39 +0000 Subject: [PATCH 11/25] Fix build: use ProcessUtils.ToTimeoutMilliseconds, remove redundant EINTR continue blocks Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/bbf726d9-5357-41b4-9573-44f75b0746aa Co-authored-by: adamsitnik <6011991+adamsitnik@users.noreply.github.com> --- .../src/System/Diagnostics/Process.Multiplexing.Unix.cs | 8 -------- .../src/System/Diagnostics/Process.Multiplexing.cs | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs index 91c7b7ec7c5ff9..aa47e610cb9bfd 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs @@ -63,10 +63,6 @@ private IEnumerable ReadPipesToLines( while (!outputDone || !errorDone) { int numFds = PollForPipeActivity(pollFds, errorFd, outputFd, errorDone, outputDone, deadline, timeoutMs, out int errorIndex, out int outputIndex); - if (numFds == 0) - { - continue; // EINTR - } // Process error pipe first (lower index) when both have data available. for (int i = 0; i < numFds; i++) @@ -281,10 +277,6 @@ private static void ReadPipes( while (!outputDone || !errorDone) { int numFds = PollForPipeActivity(pollFds, errorFd, outputFd, errorDone, outputDone, deadline, timeoutMs, out int errorIndex, out int outputIndex); - if (numFds == 0) - { - continue; // EINTR - } for (int i = 0; i < numFds; i++) { diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs index 94a69e1c15f0bb..70f3d4a909478c 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs @@ -146,7 +146,7 @@ public IEnumerable ReadAllLines(TimeSpan? timeout = default) ValidateReadAllState(); int timeoutMs = timeout.HasValue - ? ToTimeoutMilliseconds(timeout.Value) + ? ProcessUtils.ToTimeoutMilliseconds(timeout.Value) : Timeout.Infinite; Encoding outputEncoding = _startInfo?.StandardOutputEncoding ?? GetStandardOutputEncoding(); From 14a90716f2b9d60d0f35f38809cffadee50939e6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 24 Apr 2026 17:59:51 +0000 Subject: [PATCH 12/25] Address feedback: inline PollPipes, remove DangerousAddRef on Windows, add long-line test Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/5c4e4fde-3b17-46d9-a106-9df220d4532b Co-authored-by: adamsitnik <6011991+adamsitnik@users.noreply.github.com> --- .../Diagnostics/Process.Multiplexing.Unix.cs | 30 ++++++-------- .../Process.Multiplexing.Windows.cs | 14 ------- .../tests/ProcessStreamingTests.cs | 41 +++++++++++++++++++ 3 files changed, 54 insertions(+), 31 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs index aa47e610cb9bfd..2701a454fb4054 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs @@ -173,7 +173,19 @@ private static int PollForPipeActivity( throw new TimeoutException(); } - Interop.Error pollError = PollPipes(pollFds, numFds, pollTimeout, out uint triggered); + uint triggered; + Interop.Error pollError; + unsafe + { + uint localTriggered = 0; + fixed (Interop.PollEvent* pPollFds = pollFds) + { + pollError = Interop.Sys.Poll(pPollFds, (uint)numFds, pollTimeout, &localTriggered); + } + + triggered = localTriggered; + } + if (pollError != Interop.Error.SUCCESS) { if (pollError == Interop.Error.EINTR) @@ -230,22 +242,6 @@ private static void HandlePipeLineRead( // bytesRead < 0 means EAGAIN — nothing available yet, let poll retry. } - /// - /// Calls poll(2) on the provided array of poll events. - /// - private static unsafe Interop.Error PollPipes(Interop.PollEvent[] pollFds, int numFds, int timeoutMs, out uint triggered) - { - uint localTriggered = 0; - Interop.Error result; - fixed (Interop.PollEvent* pPollFds = pollFds) - { - result = Interop.Sys.Poll(pPollFds, (uint)numFds, timeoutMs, &localTriggered); - } - - triggered = localTriggered; - return result; - } - /// /// Reads from both standard output and standard error pipes using Unix poll-based multiplexing /// with non-blocking reads. diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs index 87c509735e8697..0daeb92cfdad51 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs @@ -33,7 +33,6 @@ private IEnumerable ReadPipesToLines( byte[] errorByteBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); char[] outputCharBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); char[] errorCharBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); - bool outputRefAdded = false, errorRefAdded = false; PinnedGCHandle outputPin = default, errorPin = default; // NativeOverlapped* can't be used as iterator state machine fields (pointers aren't // allowed in managed types). Store as nint and cast back inside scoped unsafe blocks. @@ -43,9 +42,6 @@ private IEnumerable ReadPipesToLines( try { - outputHandle.DangerousAddRef(ref outputRefAdded); - errorHandle.DangerousAddRef(ref errorRefAdded); - outputPin = new PinnedGCHandle(outputByteBuffer); errorPin = new PinnedGCHandle(errorByteBuffer); @@ -211,16 +207,6 @@ private IEnumerable ReadPipesToLines( outputPin.Dispose(); errorPin.Dispose(); - if (outputRefAdded) - { - outputHandle.DangerousRelease(); - } - - if (errorRefAdded) - { - errorHandle.DangerousRelease(); - } - ArrayPool.Shared.Return(outputByteBuffer); ArrayPool.Shared.Return(errorByteBuffer); ArrayPool.Shared.Return(outputCharBuffer); diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs index de0c95b15d8b5f..47a32cddce2e75 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs @@ -315,6 +315,47 @@ public async Task ReadAllLines_ReadsLargeOutput(bool useAsync) Assert.True(process.WaitForExit(WaitInMS)); } + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData(true)] + [InlineData(false)] + public async Task ReadAllLines_ReadsVeryLongLines(bool useAsync) + { + const int lineLength = 8192; + const int lineCount = 3; + using Process process = CreateProcess(() => + { + for (int i = 0; i < lineCount; i++) + { + Console.Out.WriteLine(new string((char)('A' + i), lineLength)); + Console.Out.Flush(); + Console.Error.WriteLine(new string((char)('a' + i), lineLength)); + Console.Error.Flush(); + } + + return RemoteExecutor.SuccessExitCode; + }); + + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardError = true; + process.Start(); + + List capturedOutput = new(); + List capturedError = new(); + + await EnumerateLines(process, useAsync, capturedOutput, capturedError); + + Assert.Equal(lineCount, capturedOutput.Count); + Assert.Equal(lineCount, capturedError.Count); + + for (int i = 0; i < lineCount; i++) + { + Assert.Equal(new string((char)('A' + i), lineLength), capturedOutput[i]); + Assert.Equal(new string((char)('a' + i), lineLength), capturedError[i]); + } + + Assert.True(process.WaitForExit(WaitInMS)); + } + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] [InlineData(true)] [InlineData(false)] From 9b51e98a5a016273db2c7aaae0bc641dcc5dd185 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 24 Apr 2026 18:20:27 +0000 Subject: [PATCH 13/25] Use single triggered variable in PollForPipeActivity Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/93d247fb-d921-46f1-9964-ef96f1217901 Co-authored-by: adamsitnik <6011991+adamsitnik@users.noreply.github.com> --- .../src/System/Diagnostics/Process.Multiplexing.Unix.cs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs index 2701a454fb4054..d67d0b9890c323 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs @@ -173,17 +173,14 @@ private static int PollForPipeActivity( throw new TimeoutException(); } - uint triggered; + uint triggered = 0; Interop.Error pollError; unsafe { - uint localTriggered = 0; fixed (Interop.PollEvent* pPollFds = pollFds) { - pollError = Interop.Sys.Poll(pPollFds, (uint)numFds, pollTimeout, &localTriggered); + pollError = Interop.Sys.Poll(pPollFds, (uint)numFds, pollTimeout, &triggered); } - - triggered = localTriggered; } if (pollError != Interop.Error.SUCCESS) From 658e832f42a7fbb71af0feaaa6dd1c7a75c16fd2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 24 Apr 2026 19:10:40 +0000 Subject: [PATCH 14/25] Always compact before growing in DecodeAndAppendChars; fix RentLargerBuffer XML comment Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/f46fdccb-fe5d-4456-abf6-2ca15b1f07bf Co-authored-by: adamsitnik <6011991+adamsitnik@users.noreply.github.com> --- .../System/Diagnostics/Process.Multiplexing.cs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs index 70f3d4a909478c..1908843af45528 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs @@ -177,17 +177,14 @@ private static void DecodeAndAppendChars( return; } - // If there isn't enough room at the end but compacting the consumed space at the start - // would free enough room, compact to avoid unnecessary buffer growth. + // If there isn't enough room at the end, compact the consumed space at the start first + // so that if growth is still needed, RentLargerBuffer copies only the unconsumed data. if (charEndIndex + charCount > charBuffer.Length && charStartIndex > 0) { int remaining = charEndIndex - charStartIndex; - if (remaining + charCount <= charBuffer.Length) - { - Array.Copy(charBuffer, charStartIndex, charBuffer, 0, remaining); - charStartIndex = 0; - charEndIndex = remaining; - } + Array.Copy(charBuffer, charStartIndex, charBuffer, 0, remaining); + charStartIndex = 0; + charEndIndex = remaining; } while (charEndIndex + charCount > charBuffer.Length) @@ -638,7 +635,7 @@ private void ReadPipesToBuffers( } /// - /// Rents a larger buffer from the array pool, copies existing data, and returns the old buffer. + /// Rents a larger buffer from the array pool, copies existing data, and returns the old buffer to the pool. /// private static void RentLargerBuffer(ref T[] buffer, int dataLength) { From 1f1f41a9230a6b3d08f653d7de9cd3d4e9632c1a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 24 Apr 2026 21:00:45 +0000 Subject: [PATCH 15/25] Restore stackalloc in ReadPipes; accept Span in PollForPipeActivity/PreparePollFds Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/bfdc186c-a30a-4294-99ec-6b264c27af2f Co-authored-by: adamsitnik <6011991+adamsitnik@users.noreply.github.com> --- .../src/System/Diagnostics/Process.Multiplexing.Unix.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs index d67d0b9890c323..cf3f5fd2a9456b 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs @@ -124,7 +124,7 @@ private IEnumerable ReadPipesToLines( /// Returns the number of active file descriptors. /// private static int PreparePollFds( - Interop.PollEvent[] pollFds, + Span pollFds, int errorFd, int outputFd, bool errorDone, bool outputDone, out int errorIndex, out int outputIndex) @@ -160,7 +160,7 @@ private static int PreparePollFds( /// the caller should retry. /// private static int PollForPipeActivity( - Interop.PollEvent[] pollFds, + Span pollFds, int errorFd, int outputFd, bool errorDone, bool outputDone, long deadline, int timeoutMs, @@ -260,7 +260,7 @@ private static void ReadPipes( throw new Win32Exception(); } - Interop.PollEvent[] pollFds = new Interop.PollEvent[2]; + Span pollFds = stackalloc Interop.PollEvent[2]; long deadline = timeoutMs >= 0 ? Environment.TickCount64 + timeoutMs From e4e05db2ba406a3051da1fc64aa99fdd81517153 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Sat, 25 Apr 2026 15:08:25 +0200 Subject: [PATCH 16/25] reduce code duplication: - CancelPendingIOIfNeeded is called anyway from the finally block - move common logic for handling EOF to a helper method - move common logic for handling bytes read to a helper method - test improvements --- .../Diagnostics/Process.Multiplexing.Unix.cs | 13 +---- .../Process.Multiplexing.Windows.cs | 55 +++---------------- .../Diagnostics/Process.Multiplexing.cs | 19 +++++++ .../tests/ProcessStreamingTests.cs | 13 +++-- 4 files changed, 37 insertions(+), 63 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs index cf3f5fd2a9456b..57e35c1c922753 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs @@ -221,20 +221,11 @@ private static void HandlePipeLineRead( int bytesRead = ReadNonBlocking(handle, byteBuffer, 0); if (bytesRead > 0) { - DecodeAndAppendChars(decoder, byteBuffer, 0, bytesRead, flush: false, ref charBuffer, ref charStart, ref charEnd); - if (!bomChecked && charEnd > 0) - { - SkipBomIfPresent(charBuffer, charEnd, ref charStart); - bomChecked = true; - } - ParseLinesFromCharBuffer(charBuffer, ref charStart, charEnd, standardError, lines); - CompactOrGrowCharBuffer(ref charBuffer, ref charStart, ref charEnd); + DecodeBytesAndParseLines(decoder, byteBuffer, bytesRead, ref charBuffer, ref charStart, ref charEnd, ref bomChecked, standardError, lines); } else if (bytesRead == 0) { - DecodeAndAppendChars(decoder, Array.Empty(), 0, 0, flush: true, ref charBuffer, ref charStart, ref charEnd); - EmitRemainingCharsAsLine(charBuffer, ref charStart, ref charEnd, standardError, lines); - done = true; + done = FlushDecoderAndEmitRemainingChars(decoder, ref charBuffer, ref charStart, ref charEnd, standardError, lines); } // bytesRead < 0 means EAGAIN — nothing available yet, let poll retry. } diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs index 0daeb92cfdad51..183bdec04a6e66 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs @@ -80,12 +80,6 @@ private IEnumerable ReadPipesToLines( if (waitResult == WaitHandle.WaitTimeout) { - unsafe - { - CancelPendingIOIfNeeded(outputHandle, outputDone, (NativeOverlapped*)outputOverlappedNint); - CancelPendingIOIfNeeded(errorHandle, errorDone, (NativeOverlapped*)errorOverlappedNint); - } - throw new TimeoutException(); } @@ -102,28 +96,15 @@ private IEnumerable ReadPipesToLines( if (bytesRead > 0) { - // Decode bytes to chars and parse lines. if (isError) { - DecodeAndAppendChars(errorDecoder, errorByteBuffer, 0, bytesRead, flush: false, ref errorCharBuffer, ref errorCharStart, ref errorCharEnd); - if (!errorBomChecked && errorCharEnd > 0) - { - SkipBomIfPresent(errorCharBuffer, errorCharEnd, ref errorCharStart); - errorBomChecked = true; - } - ParseLinesFromCharBuffer(errorCharBuffer, ref errorCharStart, errorCharEnd, true, lines); - CompactOrGrowCharBuffer(ref errorCharBuffer, ref errorCharStart, ref errorCharEnd); + DecodeBytesAndParseLines(errorDecoder, errorByteBuffer, bytesRead, ref errorCharBuffer, ref errorCharStart, + ref errorCharEnd, ref errorBomChecked, isError, lines); } else { - DecodeAndAppendChars(outputDecoder, outputByteBuffer, 0, bytesRead, flush: false, ref outputCharBuffer, ref outputCharStart, ref outputCharEnd); - if (!outputBomChecked && outputCharEnd > 0) - { - SkipBomIfPresent(outputCharBuffer, outputCharEnd, ref outputCharStart); - outputBomChecked = true; - } - ParseLinesFromCharBuffer(outputCharBuffer, ref outputCharStart, outputCharEnd, false, lines); - CompactOrGrowCharBuffer(ref outputCharBuffer, ref outputCharStart, ref outputCharEnd); + DecodeBytesAndParseLines(outputDecoder, outputByteBuffer, bytesRead, ref outputCharBuffer, ref outputCharStart, + ref outputCharEnd, ref outputBomChecked, isError, lines); } unsafe @@ -139,38 +120,20 @@ private IEnumerable ReadPipesToLines( currentByteBuffer.Length, (NativeOverlapped*)currentOverlappedNint, currentEvent)) { - // EOF during QueueRead — flush decoder and emit remaining. - if (isError) - { - DecodeAndAppendChars(errorDecoder, Array.Empty(), 0, 0, flush: true, ref errorCharBuffer, ref errorCharStart, ref errorCharEnd); - EmitRemainingCharsAsLine(errorCharBuffer, ref errorCharStart, ref errorCharEnd, true, lines); - errorDone = true; - } - else - { - DecodeAndAppendChars(outputDecoder, Array.Empty(), 0, 0, flush: true, ref outputCharBuffer, ref outputCharStart, ref outputCharEnd); - EmitRemainingCharsAsLine(outputCharBuffer, ref outputCharStart, ref outputCharEnd, false, lines); - outputDone = true; - } - - currentEvent.Reset(); + bytesRead = 0; // EOF during QueueRead } } } - else + + if (bytesRead == 0) // EOF { - // EOF: flush decoder and emit remaining chars. if (isError) { - DecodeAndAppendChars(errorDecoder, Array.Empty(), 0, 0, flush: true, ref errorCharBuffer, ref errorCharStart, ref errorCharEnd); - EmitRemainingCharsAsLine(errorCharBuffer, ref errorCharStart, ref errorCharEnd, true, lines); - errorDone = true; + errorDone = FlushDecoderAndEmitRemainingChars(errorDecoder, ref errorCharBuffer, ref errorCharStart, ref errorCharEnd, isError, lines); } else { - DecodeAndAppendChars(outputDecoder, Array.Empty(), 0, 0, flush: true, ref outputCharBuffer, ref outputCharStart, ref outputCharEnd); - EmitRemainingCharsAsLine(outputCharBuffer, ref outputCharStart, ref outputCharEnd, false, lines); - outputDone = true; + outputDone = FlushDecoderAndEmitRemainingChars(outputDecoder, ref outputCharBuffer, ref outputCharStart, ref outputCharEnd, isError, lines); } currentEvent.Reset(); diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs index 1908843af45528..d30a8e8b15f4aa 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs @@ -329,6 +329,25 @@ private static void CompactOrGrowCharBuffer(ref char[] buffer, ref int startInde endIndex = remaining; } + private static void DecodeBytesAndParseLines(Decoder decoder, byte[] byteBuffer, int bytesRead, ref char[] charBuffer, ref int charStart, ref int charEnd, ref bool bomChecked, bool standardError, List lines) + { + DecodeAndAppendChars(decoder, byteBuffer, 0, bytesRead, flush: false, ref charBuffer, ref charStart, ref charEnd); + if (!bomChecked && charEnd > 0) + { + SkipBomIfPresent(charBuffer, charEnd, ref charStart); + bomChecked = true; + } + ParseLinesFromCharBuffer(charBuffer, ref charStart, charEnd, standardError, lines); + CompactOrGrowCharBuffer(ref charBuffer, ref charStart, ref charEnd); + } + + private static bool FlushDecoderAndEmitRemainingChars(Decoder decoder, ref char[] charBuffer, ref int charStart, ref int charEnd, bool standardError, List lines) + { + DecodeAndAppendChars(decoder, Array.Empty(), 0, 0, flush: true, ref charBuffer, ref charStart, ref charEnd); + EmitRemainingCharsAsLine(charBuffer, ref charStart, ref charEnd, standardError, lines); + return true; + } + /// /// Asynchronously reads all standard output and standard error of the process as text. /// diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs index 47a32cddce2e75..d0b2c76c0985da 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs @@ -19,7 +19,7 @@ public class ProcessStreamingTests : ProcessTestBase [InlineData(false)] public async Task ReadAllLines_ThrowsAfterDispose(bool useAsync) { - Process process = CreateProcess(RemotelyInvokable.Dummy); + using Process process = CreateProcess(static () => RemoteExecutor.SuccessExitCode); process.Start(); Assert.True(process.WaitForExit(WaitInMS)); @@ -50,7 +50,7 @@ await Assert.ThrowsAsync(async () => [InlineData(false)] public async Task ReadAllLines_ThrowsWhenNoStreamsRedirected(bool useAsync) { - Process process = CreateProcess(RemotelyInvokable.Dummy); + using Process process = CreateProcess(static () => RemoteExecutor.SuccessExitCode); process.Start(); if (useAsync) @@ -82,7 +82,7 @@ await Assert.ThrowsAsync(async () => [InlineData(false, false)] public async Task ReadAllLines_ThrowsWhenOnlyOutputOrErrorIsRedirected(bool standardOutput, bool useAsync) { - Process process = CreateProcess(RemotelyInvokable.Dummy); + using Process process = CreateProcess(static () => RemoteExecutor.SuccessExitCode); process.StartInfo.RedirectStandardOutput = standardOutput; process.StartInfo.RedirectStandardError = !standardOutput; process.Start(); @@ -116,7 +116,7 @@ await Assert.ThrowsAsync(async () => [InlineData(false, false)] public async Task ReadAllLines_ThrowsWhenOutputOrErrorIsInSyncMode(bool standardOutput, bool useAsync) { - Process process = CreateProcess(RemotelyInvokable.Dummy); + using Process process = CreateProcess(static () => RemoteExecutor.SuccessExitCode); process.StartInfo.RedirectStandardOutput = true; process.StartInfo.RedirectStandardError = true; process.Start(); @@ -153,7 +153,7 @@ await Assert.ThrowsAsync(async () => [InlineData(false, false)] public async Task ReadAllLines_ThrowsWhenOutputOrErrorIsInAsyncMode(bool standardOutput, bool useAsync) { - Process process = CreateProcess(RemotelyInvokable.StreamBody); + using Process process = CreateProcess(static () => RemoteExecutor.SuccessExitCode); process.StartInfo.RedirectStandardOutput = true; process.StartInfo.RedirectStandardError = true; process.Start(); @@ -306,6 +306,7 @@ public async Task ReadAllLines_ReadsLargeOutput(bool useAsync) await EnumerateLines(process, useAsync, capturedOutput, capturedError); + Assert.Equal(lineCount, capturedOutput.Count); for (int i = 0; i < lineCount; i++) { Assert.Equal($"line{i}", capturedOutput[i]); @@ -361,7 +362,7 @@ public async Task ReadAllLines_ReadsVeryLongLines(bool useAsync) [InlineData(false)] public async Task ReadAllLines_ThrowsOnCancellationOrTimeout(bool useAsync) { - Process process = CreateProcess(RemotelyInvokable.ReadLine); + using Process process = CreateProcess(RemotelyInvokable.ReadLine); process.StartInfo.RedirectStandardOutput = true; process.StartInfo.RedirectStandardError = true; process.StartInfo.RedirectStandardInput = true; From 7c196ccf84cdc1886af2fbca1c522690ba29aa13 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 27 Apr 2026 20:11:05 +0000 Subject: [PATCH 17/25] Fix DecodeAndAppendChars early return, use single byte buffer on Unix, add multi-byte encoding tests Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/c08a9de7-b817-42f5-9272-0d54c0a69ad3 Co-authored-by: adamsitnik <6011991+adamsitnik@users.noreply.github.com> --- .../Diagnostics/Process.Multiplexing.Unix.cs | 10 +++---- .../Diagnostics/Process.Multiplexing.cs | 2 +- .../tests/ProcessStreamingTests.cs | 29 +++++++++++++++++++ .../tests/RemotelyInvokable.cs | 18 ++++++++++++ 4 files changed, 52 insertions(+), 7 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs index 57e35c1c922753..0cac781e109c90 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs @@ -28,8 +28,7 @@ private IEnumerable ReadPipesToLines( SafePipeHandle outputHandle = GetSafeHandleFromStreamReader(_standardOutput!); SafePipeHandle errorHandle = GetSafeHandleFromStreamReader(_standardError!); - byte[] outputByteBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); - byte[] errorByteBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); + byte[] byteBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); char[] outputCharBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); char[] errorCharBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); bool outputRefAdded = false, errorRefAdded = false; @@ -78,13 +77,13 @@ private IEnumerable ReadPipesToLines( // Use explicit branching to avoid ref locals across yield points. if (isError) { - HandlePipeLineRead(currentHandle, errorDecoder, errorByteBuffer, + HandlePipeLineRead(currentHandle, errorDecoder, byteBuffer, ref errorCharBuffer, ref errorCharStart, ref errorCharEnd, ref errorBomChecked, ref errorDone, standardError: true, lines); } else { - HandlePipeLineRead(currentHandle, outputDecoder, outputByteBuffer, + HandlePipeLineRead(currentHandle, outputDecoder, byteBuffer, ref outputCharBuffer, ref outputCharStart, ref outputCharEnd, ref outputBomChecked, ref outputDone, standardError: false, lines); } @@ -111,8 +110,7 @@ private IEnumerable ReadPipesToLines( errorHandle.DangerousRelease(); } - ArrayPool.Shared.Return(outputByteBuffer); - ArrayPool.Shared.Return(errorByteBuffer); + ArrayPool.Shared.Return(byteBuffer); ArrayPool.Shared.Return(outputCharBuffer); ArrayPool.Shared.Return(errorCharBuffer); } diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs index d30a8e8b15f4aa..7564a2acf60952 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs @@ -172,7 +172,7 @@ private static void DecodeAndAppendChars( ref int charEndIndex) { int charCount = decoder.GetCharCount(byteBuffer, byteIndex, byteCount, flush); - if (charCount == 0) + if (charCount == 0 && byteCount == 0) { return; } diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs index d0b2c76c0985da..da60d39ca0c29d 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs @@ -492,6 +492,35 @@ public async Task ReadAllLines_WorksWithNonDefaultEncodings(string encodingName, Assert.True(process.WaitForExit(WaitInMS)); } + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData("utf-8", true)] + [InlineData("utf-8", false)] + [InlineData("utf-16", true)] + [InlineData("utf-16", false)] + [InlineData("utf-32", true)] + [InlineData("utf-32", false)] + public async Task ReadAllLines_WorksWithMultiByteCharacters(string encodingName, bool useAsync) + { + Encoding encoding = Encoding.GetEncoding(encodingName); + + using Process process = CreateProcessPortable(RemotelyInvokable.WriteMultiByteLinesToBothStreamsWithEncoding, encodingName); + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardError = true; + process.StartInfo.StandardOutputEncoding = encoding; + process.StartInfo.StandardErrorEncoding = encoding; + process.Start(); + + List capturedOutput = new(); + List capturedError = new(); + + await EnumerateLines(process, useAsync, capturedOutput, capturedError); + + Assert.Equal(new[] { "hello_\u4e16\u754c_stdout" }, capturedOutput); + Assert.Equal(new[] { "hello_\u4e16\u754c_stderr" }, capturedError); + + Assert.True(process.WaitForExit(WaitInMS)); + } + private Process StartLinePrintingProcess(string stdOutText, string stdErrText) { Process process = CreateProcess((stdOut, stdErr) => diff --git a/src/libraries/System.Diagnostics.Process/tests/RemotelyInvokable.cs b/src/libraries/System.Diagnostics.Process/tests/RemotelyInvokable.cs index 31b96568d44dea..8e1afb4c92f534 100644 --- a/src/libraries/System.Diagnostics.Process/tests/RemotelyInvokable.cs +++ b/src/libraries/System.Diagnostics.Process/tests/RemotelyInvokable.cs @@ -151,6 +151,24 @@ public static int WriteLinesToBothStreamsWithEncoding(string encodingName) return SuccessExitCode; } + public static int WriteMultiByteLinesToBothStreamsWithEncoding(string encodingName) + { + Encoding encoding = Encoding.GetEncoding(encodingName); + // Use characters that require multiple bytes in UTF-8 and exercise decoder state across reads: + // CJK characters (U+4E16 U+754C = "世界") require 3 bytes each in UTF-8, 2 bytes in UTF-16, 4 bytes in UTF-32. + using (StreamWriter outputWriter = new(Console.OpenStandardOutput(), encoding)) + { + outputWriter.WriteLine("hello_世界_stdout"); + } + + using (StreamWriter errorWriter = new(Console.OpenStandardError(), encoding)) + { + errorWriter.WriteLine("hello_世界_stderr"); + } + + return SuccessExitCode; + } + public static int SelfTerminate() { Process.GetCurrentProcess().Kill(); From f583fcafb8b6d4acaaa15159771ecf973878daa8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 28 Apr 2026 15:24:49 +0000 Subject: [PATCH 18/25] Address review feedback: byte-level preamble/encoding detection, remove CompactOrGrowCharBuffer, inline test helpers, remove early return Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/1b8575a8-c115-4b27-ae4b-213035585f83 Co-authored-by: adamsitnik <6011991+adamsitnik@users.noreply.github.com> --- .../Diagnostics/Process.Multiplexing.Unix.cs | 23 ++-- .../Process.Multiplexing.Windows.cs | 16 +-- .../Diagnostics/Process.Multiplexing.cs | 112 ++++++++++-------- .../tests/ProcessStreamingTests.cs | 34 +++++- .../tests/RemotelyInvokable.cs | 34 ------ 5 files changed, 117 insertions(+), 102 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs index 0cac781e109c90..6f78461dd2e2a8 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs @@ -22,8 +22,8 @@ public partial class Process /// private IEnumerable ReadPipesToLines( int timeoutMs, - Decoder outputDecoder, - Decoder errorDecoder) + Encoding outputEncoding, + Encoding errorEncoding) { SafePipeHandle outputHandle = GetSafeHandleFromStreamReader(_standardOutput!); SafePipeHandle errorHandle = GetSafeHandleFromStreamReader(_standardError!); @@ -52,10 +52,12 @@ private IEnumerable ReadPipesToLines( long deadline = timeoutMs >= 0 ? Environment.TickCount64 + timeoutMs : long.MaxValue; + Decoder outputDecoder = outputEncoding.GetDecoder(); + Decoder errorDecoder = errorEncoding.GetDecoder(); int outputCharStart = 0, outputCharEnd = 0; int errorCharStart = 0, errorCharEnd = 0; bool outputDone = false, errorDone = false; - bool outputBomChecked = false, errorBomChecked = false; + bool outputPreambleChecked = false, errorPreambleChecked = false; List lines = new(); @@ -77,15 +79,15 @@ private IEnumerable ReadPipesToLines( // Use explicit branching to avoid ref locals across yield points. if (isError) { - HandlePipeLineRead(currentHandle, errorDecoder, byteBuffer, + HandlePipeLineRead(currentHandle, ref errorDecoder, ref errorEncoding, byteBuffer, ref errorCharBuffer, ref errorCharStart, ref errorCharEnd, - ref errorBomChecked, ref errorDone, standardError: true, lines); + ref errorPreambleChecked, ref errorDone, standardError: true, lines); } else { - HandlePipeLineRead(currentHandle, outputDecoder, byteBuffer, + HandlePipeLineRead(currentHandle, ref outputDecoder, ref outputEncoding, byteBuffer, ref outputCharBuffer, ref outputCharStart, ref outputCharEnd, - ref outputBomChecked, ref outputDone, standardError: false, lines); + ref outputPreambleChecked, ref outputDone, standardError: false, lines); } } @@ -206,12 +208,13 @@ private static int PollForPipeActivity( /// private static void HandlePipeLineRead( SafePipeHandle handle, - Decoder decoder, + ref Decoder decoder, + ref Encoding encoding, byte[] byteBuffer, ref char[] charBuffer, ref int charStart, ref int charEnd, - ref bool bomChecked, + ref bool preambleChecked, ref bool done, bool standardError, List lines) @@ -219,7 +222,7 @@ private static void HandlePipeLineRead( int bytesRead = ReadNonBlocking(handle, byteBuffer, 0); if (bytesRead > 0) { - DecodeBytesAndParseLines(decoder, byteBuffer, bytesRead, ref charBuffer, ref charStart, ref charEnd, ref bomChecked, standardError, lines); + DecodeBytesAndParseLines(ref decoder, ref encoding, byteBuffer, bytesRead, ref charBuffer, ref charStart, ref charEnd, ref preambleChecked, standardError, lines); } else if (bytesRead == 0) { diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs index 183bdec04a6e66..0715bafbd3f738 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs @@ -23,8 +23,8 @@ public partial class Process /// private IEnumerable ReadPipesToLines( int timeoutMs, - Decoder outputDecoder, - Decoder errorDecoder) + Encoding outputEncoding, + Encoding errorEncoding) { SafeFileHandle outputHandle = GetSafeHandleFromStreamReader(_standardOutput!); SafeFileHandle errorHandle = GetSafeHandleFromStreamReader(_standardError!); @@ -57,9 +57,11 @@ private IEnumerable ReadPipesToLines( // Error output gets index 0 so WaitAny services it first when both are signaled. WaitHandle[] waitHandles = [errorEvent, outputEvent]; + Decoder outputDecoder = outputEncoding.GetDecoder(); + Decoder errorDecoder = errorEncoding.GetDecoder(); int outputCharStart = 0, outputCharEnd = 0; int errorCharStart = 0, errorCharEnd = 0; - bool outputBomChecked = false, errorBomChecked = false; + bool outputPreambleChecked = false, errorPreambleChecked = false; unsafe { @@ -98,13 +100,13 @@ private IEnumerable ReadPipesToLines( { if (isError) { - DecodeBytesAndParseLines(errorDecoder, errorByteBuffer, bytesRead, ref errorCharBuffer, ref errorCharStart, - ref errorCharEnd, ref errorBomChecked, isError, lines); + DecodeBytesAndParseLines(ref errorDecoder, ref errorEncoding, errorByteBuffer, bytesRead, ref errorCharBuffer, ref errorCharStart, + ref errorCharEnd, ref errorPreambleChecked, isError, lines); } else { - DecodeBytesAndParseLines(outputDecoder, outputByteBuffer, bytesRead, ref outputCharBuffer, ref outputCharStart, - ref outputCharEnd, ref outputBomChecked, isError, lines); + DecodeBytesAndParseLines(ref outputDecoder, ref outputEncoding, outputByteBuffer, bytesRead, ref outputCharBuffer, ref outputCharStart, + ref outputCharEnd, ref outputPreambleChecked, isError, lines); } unsafe diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs index 7564a2acf60952..8d46ff81b506e6 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Buffers; +using System.Buffers.Binary; using System.Collections.Generic; using System.IO; using System.Runtime.CompilerServices; @@ -152,7 +153,7 @@ public IEnumerable ReadAllLines(TimeSpan? timeout = default) Encoding outputEncoding = _startInfo?.StandardOutputEncoding ?? GetStandardOutputEncoding(); Encoding errorEncoding = _startInfo?.StandardErrorEncoding ?? GetStandardOutputEncoding(); - return ReadPipesToLines(timeoutMs, outputEncoding.GetDecoder(), errorEncoding.GetDecoder()); + return ReadPipesToLines(timeoutMs, outputEncoding, errorEncoding); } /// @@ -172,10 +173,6 @@ private static void DecodeAndAppendChars( ref int charEndIndex) { int charCount = decoder.GetCharCount(byteBuffer, byteIndex, byteCount, flush); - if (charCount == 0 && byteCount == 0) - { - return; - } // If there isn't enough room at the end, compact the consumed space at the start first // so that if growth is still needed, RentLargerBuffer copies only the unconsumed data. @@ -197,21 +194,65 @@ private static void DecodeAndAppendChars( } /// - /// If the first character at is the Unicode BOM (U+FEFF), - /// advances past it. Called once per stream after the first - /// decode that produces characters, to match BOM-stripping behavior. + /// Checks for the encoding's preamble or a BOM from a different encoding at the start of + /// the byte buffer, mimicking behavior. + /// If the encoding's own preamble is found, returns the number of bytes to skip. + /// If a different encoding's BOM is detected, updates and + /// and returns the BOM length to skip. /// - /// The decoded character buffer. - /// Exclusive upper bound of valid characters in . - /// - /// Current read position in ; advanced by one if a BOM is found. - /// - private static void SkipBomIfPresent(char[] charBuffer, int endIndex, ref int startIndex) + private static int SkipPreambleOrDetectEncoding(byte[] byteBuffer, int byteCount, ref Encoding encoding, ref Decoder decoder) { - if (startIndex < endIndex && charBuffer[startIndex] == '\uFEFF') + // Check for the encoding's own preamble first (like StreamReader.IsPreamble). + ReadOnlySpan preamble = encoding.Preamble; + if (preamble.Length > 0 && byteCount >= preamble.Length + && byteBuffer.AsSpan(0, preamble.Length).SequenceEqual(preamble)) { - startIndex++; + return preamble.Length; } + + // No preamble match — check for BOM from other encodings (like StreamReader.DetectEncoding). + if (byteCount >= 2) + { + ushort firstTwoBytes = BinaryPrimitives.ReadUInt16LittleEndian(byteBuffer); + + if (firstTwoBytes == 0xFFFE) + { + // Big Endian Unicode + encoding = Encoding.BigEndianUnicode; + decoder = encoding.GetDecoder(); + return 2; + } + + if (firstTwoBytes == 0xFEFF) + { + if (byteCount >= 4 && byteBuffer[2] == 0 && byteBuffer[3] == 0) + { + encoding = Encoding.UTF32; + decoder = encoding.GetDecoder(); + return 4; + } + + encoding = Encoding.Unicode; + decoder = encoding.GetDecoder(); + return 2; + } + + if (byteCount >= 3 && firstTwoBytes == 0xBBEF && byteBuffer[2] == 0xBF) + { + encoding = Encoding.UTF8; + decoder = encoding.GetDecoder(); + return 3; + } + + if (byteCount >= 4 && firstTwoBytes == 0 && byteBuffer[2] == 0xFE && byteBuffer[3] == 0xFF) + { + encoding = new UTF32Encoding(bigEndian: true, byteOrderMark: true); + decoder = encoding.GetDecoder(); + return 4; + } + } + + return 0; } /// @@ -301,44 +342,17 @@ private static void EmitRemainingCharsAsLine( } } - /// - /// After line parsing, compacts remaining data to the front of the char buffer if it has reached - /// the end, or rents a larger buffer if the entire buffer is filled with a single incomplete line. - /// - private static void CompactOrGrowCharBuffer(ref char[] buffer, ref int startIndex, ref int endIndex) + private static void DecodeBytesAndParseLines(ref Decoder decoder, ref Encoding encoding, byte[] byteBuffer, int bytesRead, ref char[] charBuffer, ref int charStart, ref int charEnd, ref bool preambleChecked, bool standardError, List lines) { - if (endIndex < buffer.Length) + int byteOffset = 0; + if (!preambleChecked) { - return; + preambleChecked = true; + byteOffset = SkipPreambleOrDetectEncoding(byteBuffer, bytesRead, ref encoding, ref decoder); } - int remaining = endIndex - startIndex; - - if (remaining == buffer.Length) - { - // The buffer is too small to hold a single line — grow it. - RentLargerBuffer(ref buffer, remaining); - } - else - { - // Compact: move remaining data to the start of the buffer. - Array.Copy(buffer, startIndex, buffer, 0, remaining); - } - - startIndex = 0; - endIndex = remaining; - } - - private static void DecodeBytesAndParseLines(Decoder decoder, byte[] byteBuffer, int bytesRead, ref char[] charBuffer, ref int charStart, ref int charEnd, ref bool bomChecked, bool standardError, List lines) - { - DecodeAndAppendChars(decoder, byteBuffer, 0, bytesRead, flush: false, ref charBuffer, ref charStart, ref charEnd); - if (!bomChecked && charEnd > 0) - { - SkipBomIfPresent(charBuffer, charEnd, ref charStart); - bomChecked = true; - } + DecodeAndAppendChars(decoder, byteBuffer, byteOffset, bytesRead - byteOffset, flush: false, ref charBuffer, ref charStart, ref charEnd); ParseLinesFromCharBuffer(charBuffer, ref charStart, charEnd, standardError, lines); - CompactOrGrowCharBuffer(ref charBuffer, ref charStart, ref charEnd); } private static bool FlushDecoderAndEmitRemainingChars(Decoder decoder, ref char[] charBuffer, ref int charStart, ref int charEnd, bool standardError, List lines) diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs index da60d39ca0c29d..977ad6b65f1261 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs @@ -474,7 +474,21 @@ public async Task ReadAllLines_WorksWithNonDefaultEncodings(string encodingName, { Encoding encoding = Encoding.GetEncoding(encodingName); - using Process process = CreateProcessPortable(RemotelyInvokable.WriteLinesToBothStreamsWithEncoding, encodingName); + using Process process = CreateProcess(static (string encodingArg) => + { + Encoding enc = Encoding.GetEncoding(encodingArg); + using (StreamWriter outputWriter = new(Console.OpenStandardOutput(), enc)) + { + outputWriter.WriteLine("stdout_line"); + } + + using (StreamWriter errorWriter = new(Console.OpenStandardError(), enc)) + { + errorWriter.WriteLine("stderr_line"); + } + + return RemoteExecutor.SuccessExitCode; + }, encodingName); process.StartInfo.RedirectStandardOutput = true; process.StartInfo.RedirectStandardError = true; process.StartInfo.StandardOutputEncoding = encoding; @@ -503,7 +517,23 @@ public async Task ReadAllLines_WorksWithMultiByteCharacters(string encodingName, { Encoding encoding = Encoding.GetEncoding(encodingName); - using Process process = CreateProcessPortable(RemotelyInvokable.WriteMultiByteLinesToBothStreamsWithEncoding, encodingName); + using Process process = CreateProcess(static (string encodingArg) => + { + Encoding enc = Encoding.GetEncoding(encodingArg); + // Use characters that require multiple bytes in UTF-8 and exercise decoder state across reads: + // CJK characters (U+4E16 U+754C = "世界") require 3 bytes each in UTF-8, 2 bytes in UTF-16, 4 bytes in UTF-32. + using (StreamWriter outputWriter = new(Console.OpenStandardOutput(), enc)) + { + outputWriter.WriteLine("hello_\u4e16\u754c_stdout"); + } + + using (StreamWriter errorWriter = new(Console.OpenStandardError(), enc)) + { + errorWriter.WriteLine("hello_\u4e16\u754c_stderr"); + } + + return RemoteExecutor.SuccessExitCode; + }, encodingName); process.StartInfo.RedirectStandardOutput = true; process.StartInfo.RedirectStandardError = true; process.StartInfo.StandardOutputEncoding = encoding; diff --git a/src/libraries/System.Diagnostics.Process/tests/RemotelyInvokable.cs b/src/libraries/System.Diagnostics.Process/tests/RemotelyInvokable.cs index 8e1afb4c92f534..a61596b687e5cc 100644 --- a/src/libraries/System.Diagnostics.Process/tests/RemotelyInvokable.cs +++ b/src/libraries/System.Diagnostics.Process/tests/RemotelyInvokable.cs @@ -135,40 +135,6 @@ public static int ConcatThreeArguments(string one, string two, string three) return SuccessExitCode; } - public static int WriteLinesToBothStreamsWithEncoding(string encodingName) - { - Encoding encoding = Encoding.GetEncoding(encodingName); - using (StreamWriter outputWriter = new(Console.OpenStandardOutput(), encoding)) - { - outputWriter.WriteLine("stdout_line"); - } - - using (StreamWriter errorWriter = new(Console.OpenStandardError(), encoding)) - { - errorWriter.WriteLine("stderr_line"); - } - - return SuccessExitCode; - } - - public static int WriteMultiByteLinesToBothStreamsWithEncoding(string encodingName) - { - Encoding encoding = Encoding.GetEncoding(encodingName); - // Use characters that require multiple bytes in UTF-8 and exercise decoder state across reads: - // CJK characters (U+4E16 U+754C = "世界") require 3 bytes each in UTF-8, 2 bytes in UTF-16, 4 bytes in UTF-32. - using (StreamWriter outputWriter = new(Console.OpenStandardOutput(), encoding)) - { - outputWriter.WriteLine("hello_世界_stdout"); - } - - using (StreamWriter errorWriter = new(Console.OpenStandardError(), encoding)) - { - errorWriter.WriteLine("hello_世界_stderr"); - } - - return SuccessExitCode; - } - public static int SelfTerminate() { Process.GetCurrentProcess().Kill(); From 8936fe6ec85b7a61720fa014b50ba5a157dc2f3e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 28 Apr 2026 17:34:51 +0000 Subject: [PATCH 19/25] Add missing using System.IO to ProcessStreamingTests.cs to fix build Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/869a5549-5949-464f-87b5-fdec953f9edd Co-authored-by: adamsitnik <6011991+adamsitnik@users.noreply.github.com> --- .../System.Diagnostics.Process/tests/ProcessStreamingTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs index 977ad6b65f1261..d3c88fc9582363 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Generic; +using System.IO; using System.Text; using System.Threading; using System.Threading.Tasks; From d19dd3e4c9f89801dbdb3b5f71ed55edf7891e57 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 28 Apr 2026 19:11:03 +0000 Subject: [PATCH 20/25] Fix multi-byte split test to use raw bytes; add mixed line endings test Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/fb89045e-df31-40c7-84fc-9f8df3a0e2da Co-authored-by: adamsitnik <6011991+adamsitnik@users.noreply.github.com> --- .../tests/ProcessStreamingTests.cs | 58 +++++++++++++++---- 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs index d3c88fc9582363..46e8c68e042035 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs @@ -521,17 +521,24 @@ public async Task ReadAllLines_WorksWithMultiByteCharacters(string encodingName, using Process process = CreateProcess(static (string encodingArg) => { Encoding enc = Encoding.GetEncoding(encodingArg); - // Use characters that require multiple bytes in UTF-8 and exercise decoder state across reads: - // CJK characters (U+4E16 U+754C = "世界") require 3 bytes each in UTF-8, 2 bytes in UTF-16, 4 bytes in UTF-32. - using (StreamWriter outputWriter = new(Console.OpenStandardOutput(), enc)) - { - outputWriter.WriteLine("hello_\u4e16\u754c_stdout"); - } - - using (StreamWriter errorWriter = new(Console.OpenStandardError(), enc)) - { - errorWriter.WriteLine("hello_\u4e16\u754c_stderr"); - } + // Write raw encoded bytes split at the midpoint of the byte array so the split + // lands inside a multi-byte character, exercising decoder state across reads. + // CJK chars (U+4E16 U+754C = "世界"): 3 bytes each in UTF-8, 2 in UTF-16, 4 in UTF-32. + byte[] outBytes = enc.GetBytes("hello_\u4e16\u754c_stdout\n"); + int outSplit = outBytes.Length / 2; + Stream stdout = Console.OpenStandardOutput(); + stdout.Write(outBytes, 0, outSplit); + stdout.Flush(); + stdout.Write(outBytes, outSplit, outBytes.Length - outSplit); + stdout.Flush(); + + byte[] errBytes = enc.GetBytes("hello_\u4e16\u754c_stderr\n"); + int errSplit = errBytes.Length / 2; + Stream stderr = Console.OpenStandardError(); + stderr.Write(errBytes, 0, errSplit); + stderr.Flush(); + stderr.Write(errBytes, errSplit, errBytes.Length - errSplit); + stderr.Flush(); return RemoteExecutor.SuccessExitCode; }, encodingName); @@ -552,6 +559,35 @@ public async Task ReadAllLines_WorksWithMultiByteCharacters(string encodingName, Assert.True(process.WaitForExit(WaitInMS)); } + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData(true)] + [InlineData(false)] + public async Task ReadAllLines_HandlesMixedLineEndings(bool useAsync) + { + using Process process = CreateProcess(static () => + { + // Write stdout with all three line-terminator styles in one stream: + // \r\n (Windows), \n (Unix), bare \r (classic Mac), and a final chunk with no terminator. + Stream stdout = Console.OpenStandardOutput(); + byte[] data = Encoding.UTF8.GetBytes("lineA\r\nlineB\nlineC\rlineD"); + stdout.Write(data); + stdout.Flush(); + return RemoteExecutor.SuccessExitCode; + }); + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardError = true; + process.Start(); + + List capturedOutput = new(); + List capturedError = new(); + + await EnumerateLines(process, useAsync, capturedOutput, capturedError); + + Assert.Equal(new[] { "lineA", "lineB", "lineC", "lineD" }, capturedOutput); + Assert.Empty(capturedError); + Assert.True(process.WaitForExit(WaitInMS)); + } + private Process StartLinePrintingProcess(string stdOutText, string stdErrText) { Process process = CreateProcess((stdOut, stdErr) => From c8bda3e843ecd32cc6615694835bf204572d5c37 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 28 Apr 2026 22:13:15 +0000 Subject: [PATCH 21/25] Fix partial UTF-32 BOM across reads; restore non-blocking mode in finally; add test Agent-Logs-Url: https://github.com/dotnet/runtime/sessions/8ed9ed48-a800-4d97-ac90-68590832e24e Co-authored-by: adamsitnik <6011991+adamsitnik@users.noreply.github.com> --- .../Diagnostics/Process.Multiplexing.Unix.cs | 23 ++++- .../Process.Multiplexing.Windows.cs | 14 +++- .../Diagnostics/Process.Multiplexing.cs | 83 ++++++++++++++++++- .../tests/ProcessStreamingTests.cs | 36 +++++++- 4 files changed, 145 insertions(+), 11 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs index 6f78461dd2e2a8..dda6a12f5b0cdc 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs @@ -59,6 +59,14 @@ private IEnumerable ReadPipesToLines( bool outputDone = false, errorDone = false; bool outputPreambleChecked = false, errorPreambleChecked = false; + // Four-byte BOM accumulation buffers: bytes are gathered here until we have + // enough to unambiguously detect the encoding (needed to distinguish + // UTF-32 LE BOM FF FE 00 00 from a UTF-16 LE BOM FF FE when the first read + // delivers fewer than four bytes). + byte[] outputBomAccum = new byte[4]; + byte[] errorBomAccum = new byte[4]; + int outputBomAccumLen = 0, errorBomAccumLen = 0; + List lines = new(); while (!outputDone || !errorDone) @@ -81,13 +89,15 @@ private IEnumerable ReadPipesToLines( { HandlePipeLineRead(currentHandle, ref errorDecoder, ref errorEncoding, byteBuffer, ref errorCharBuffer, ref errorCharStart, ref errorCharEnd, - ref errorPreambleChecked, ref errorDone, standardError: true, lines); + ref errorPreambleChecked, ref errorDone, standardError: true, lines, + errorBomAccum, ref errorBomAccumLen); } else { HandlePipeLineRead(currentHandle, ref outputDecoder, ref outputEncoding, byteBuffer, ref outputCharBuffer, ref outputCharStart, ref outputCharEnd, - ref outputPreambleChecked, ref outputDone, standardError: false, lines); + ref outputPreambleChecked, ref outputDone, standardError: false, lines, + outputBomAccum, ref outputBomAccumLen); } } @@ -104,11 +114,13 @@ private IEnumerable ReadPipesToLines( { if (outputRefAdded) { + Interop.Sys.Fcntl.DangerousSetIsNonBlocking(outputHandle.DangerousGetHandle().ToInt32(), 0); outputHandle.DangerousRelease(); } if (errorRefAdded) { + Interop.Sys.Fcntl.DangerousSetIsNonBlocking(errorHandle.DangerousGetHandle().ToInt32(), 0); errorHandle.DangerousRelease(); } @@ -217,15 +229,18 @@ private static void HandlePipeLineRead( ref bool preambleChecked, ref bool done, bool standardError, - List lines) + List lines, + byte[] bomAccum, + ref int bomAccumLen) { int bytesRead = ReadNonBlocking(handle, byteBuffer, 0); if (bytesRead > 0) { - DecodeBytesAndParseLines(ref decoder, ref encoding, byteBuffer, bytesRead, ref charBuffer, ref charStart, ref charEnd, ref preambleChecked, standardError, lines); + DecodeBytesAndParseLines(ref decoder, ref encoding, byteBuffer, bytesRead, ref charBuffer, ref charStart, ref charEnd, ref preambleChecked, bomAccum, ref bomAccumLen, standardError, lines); } else if (bytesRead == 0) { + FlushBomAccumulation(ref decoder, ref encoding, bomAccum, bomAccumLen, ref preambleChecked, ref charBuffer, ref charStart, ref charEnd, standardError, lines); done = FlushDecoderAndEmitRemainingChars(decoder, ref charBuffer, ref charStart, ref charEnd, standardError, lines); } // bytesRead < 0 means EAGAIN — nothing available yet, let poll retry. diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs index 0715bafbd3f738..56d3b9f2062a27 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs @@ -63,6 +63,14 @@ private IEnumerable ReadPipesToLines( int errorCharStart = 0, errorCharEnd = 0; bool outputPreambleChecked = false, errorPreambleChecked = false; + // Four-byte BOM accumulation buffers: bytes are gathered here until we have + // enough to unambiguously detect the encoding (needed to distinguish + // UTF-32 LE BOM FF FE 00 00 from a UTF-16 LE BOM FF FE when the first read + // delivers fewer than four bytes). + byte[] outputBomAccum = new byte[4]; + byte[] errorBomAccum = new byte[4]; + int outputBomAccumLen = 0, errorBomAccumLen = 0; + unsafe { outputDone = !QueueRead(outputHandle, outputPin.GetAddressOfArrayData(), @@ -101,12 +109,12 @@ private IEnumerable ReadPipesToLines( if (isError) { DecodeBytesAndParseLines(ref errorDecoder, ref errorEncoding, errorByteBuffer, bytesRead, ref errorCharBuffer, ref errorCharStart, - ref errorCharEnd, ref errorPreambleChecked, isError, lines); + ref errorCharEnd, ref errorPreambleChecked, errorBomAccum, ref errorBomAccumLen, isError, lines); } else { DecodeBytesAndParseLines(ref outputDecoder, ref outputEncoding, outputByteBuffer, bytesRead, ref outputCharBuffer, ref outputCharStart, - ref outputCharEnd, ref outputPreambleChecked, isError, lines); + ref outputCharEnd, ref outputPreambleChecked, outputBomAccum, ref outputBomAccumLen, isError, lines); } unsafe @@ -131,10 +139,12 @@ private IEnumerable ReadPipesToLines( { if (isError) { + FlushBomAccumulation(ref errorDecoder, ref errorEncoding, errorBomAccum, errorBomAccumLen, ref errorPreambleChecked, ref errorCharBuffer, ref errorCharStart, ref errorCharEnd, isError, lines); errorDone = FlushDecoderAndEmitRemainingChars(errorDecoder, ref errorCharBuffer, ref errorCharStart, ref errorCharEnd, isError, lines); } else { + FlushBomAccumulation(ref outputDecoder, ref outputEncoding, outputBomAccum, outputBomAccumLen, ref outputPreambleChecked, ref outputCharBuffer, ref outputCharStart, ref outputCharEnd, isError, lines); outputDone = FlushDecoderAndEmitRemainingChars(outputDecoder, ref outputCharBuffer, ref outputCharStart, ref outputCharEnd, isError, lines); } diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs index 8d46ff81b506e6..f8c1f42c5d412f 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs @@ -342,19 +342,94 @@ private static void EmitRemainingCharsAsLine( } } - private static void DecodeBytesAndParseLines(ref Decoder decoder, ref Encoding encoding, byte[] byteBuffer, int bytesRead, ref char[] charBuffer, ref int charStart, ref int charEnd, ref bool preambleChecked, bool standardError, List lines) + /// + /// Decodes bytes from (and any previously accumulated BOM bytes) + /// into , then parses complete lines. On the first call(s), + /// bytes are accumulated into until four are available so that + /// can unambiguously distinguish a UTF-32 LE BOM + /// (FF FE 00 00) from a UTF-16 LE BOM (FF FE) even when the first OS read + /// delivers only two bytes. + /// + private static void DecodeBytesAndParseLines( + ref Decoder decoder, ref Encoding encoding, + byte[] byteBuffer, int bytesRead, + ref char[] charBuffer, ref int charStart, ref int charEnd, + ref bool preambleChecked, + byte[] bomAccum, ref int bomAccumLen, + bool standardError, List lines) { - int byteOffset = 0; if (!preambleChecked) { + // Accumulate initial bytes into bomAccum until we have 4, enough to unambiguously + // detect all supported BOMs (UTF-8 = 3 bytes, UTF-16 LE/BE = 2 bytes, but FF FE + // could be the start of a 4-byte UTF-32 LE BOM, so we need all 4 before deciding). + int prevBomAccumLen = bomAccumLen; + if (bomAccumLen < 4 && bytesRead > 0) + { + int toCopy = Math.Min(4 - bomAccumLen, bytesRead); + byteBuffer.AsSpan(0, toCopy).CopyTo(bomAccum.AsSpan(bomAccumLen)); + bomAccumLen += toCopy; + } + + if (bomAccumLen < 4) + { + // Not enough bytes yet for a definitive BOM/preamble check. Defer to the next read. + return; + } + preambleChecked = true; - byteOffset = SkipPreambleOrDetectEncoding(byteBuffer, bytesRead, ref encoding, ref decoder); + int bomSkip = SkipPreambleOrDetectEncoding(bomAccum, bomAccumLen, ref encoding, ref decoder); + + // Decode the accumulated BOM bytes (minus the BOM prefix to skip). + int bomToDecode = bomAccumLen - bomSkip; + if (bomToDecode > 0) + { + DecodeAndAppendChars(decoder, bomAccum, bomSkip, bomToDecode, flush: false, ref charBuffer, ref charStart, ref charEnd); + } + + // Decode remaining bytes from the current read (those not consumed into bomAccum). + int consumedFromByteBuffer = bomAccumLen - prevBomAccumLen; + int remaining = bytesRead - consumedFromByteBuffer; + if (remaining > 0) + { + DecodeAndAppendChars(decoder, byteBuffer, consumedFromByteBuffer, remaining, flush: false, ref charBuffer, ref charStart, ref charEnd); + } + + ParseLinesFromCharBuffer(charBuffer, ref charStart, charEnd, standardError, lines); + return; } - DecodeAndAppendChars(decoder, byteBuffer, byteOffset, bytesRead - byteOffset, flush: false, ref charBuffer, ref charStart, ref charEnd); + DecodeAndAppendChars(decoder, byteBuffer, 0, bytesRead, flush: false, ref charBuffer, ref charStart, ref charEnd); ParseLinesFromCharBuffer(charBuffer, ref charStart, charEnd, standardError, lines); } + /// + /// Resolves any bytes accumulated in when the pipe closes + /// before four bytes have been gathered. The accumulated bytes are BOM-checked and decoded + /// into ; any complete lines are added to + /// . This must be called before + /// at EOF. + /// + private static void FlushBomAccumulation( + ref Decoder decoder, ref Encoding encoding, + byte[] bomAccum, int bomAccumLen, + ref bool preambleChecked, + ref char[] charBuffer, ref int charStart, ref int charEnd, + bool standardError, List lines) + { + if (!preambleChecked && bomAccumLen > 0) + { + preambleChecked = true; + int bomSkip = SkipPreambleOrDetectEncoding(bomAccum, bomAccumLen, ref encoding, ref decoder); + int toDecode = bomAccumLen - bomSkip; + if (toDecode > 0) + { + DecodeAndAppendChars(decoder, bomAccum, bomSkip, toDecode, flush: false, ref charBuffer, ref charStart, ref charEnd); + ParseLinesFromCharBuffer(charBuffer, ref charStart, charEnd, standardError, lines); + } + } + } + private static bool FlushDecoderAndEmitRemainingChars(Decoder decoder, ref char[] charBuffer, ref int charStart, ref int charEnd, bool standardError, List lines) { DecodeAndAppendChars(decoder, Array.Empty(), 0, 0, flush: true, ref charBuffer, ref charStart, ref charEnd); diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs index 46e8c68e042035..fe18f8c601abd1 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs @@ -588,7 +588,41 @@ public async Task ReadAllLines_HandlesMixedLineEndings(bool useAsync) Assert.True(process.WaitForExit(WaitInMS)); } - private Process StartLinePrintingProcess(string stdOutText, string stdErrText) + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData(true)] + [InlineData(false)] + public async Task ReadAllLines_HandlesPartialBomAcrossReads(bool useAsync) + { + // Write a UTF-32 LE BOM (FF FE 00 00) as two separate flushed writes so the + // first read can deliver only the first two BOM bytes. Without BOM accumulation, + // FF FE would be misclassified as a UTF-16 LE BOM and the content would be + // decoded with the wrong encoding. + using Process process = CreateProcess(static () => + { + Stream stdout = Console.OpenStandardOutput(); + stdout.Write(new byte[] { 0xFF, 0xFE }); // First half of UTF-32 LE BOM + stdout.Flush(); + stdout.Write(new byte[] { 0x00, 0x00 }); // Second half of BOM + stdout.Write(Encoding.UTF32.GetBytes("hello\n")); // Content (no BOM from GetBytes) + stdout.Flush(); + return RemoteExecutor.SuccessExitCode; + }); + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardError = true; + process.StartInfo.StandardOutputEncoding = Encoding.UTF32; + process.Start(); + + List capturedOutput = new(); + List capturedError = new(); + + await EnumerateLines(process, useAsync, capturedOutput, capturedError); + + Assert.Equal(new[] { "hello" }, capturedOutput); + Assert.Empty(capturedError); + Assert.True(process.WaitForExit(WaitInMS)); + } + + { Process process = CreateProcess((stdOut, stdErr) => { From 5f740c5093fc685df6e0613436d1d941cd0fbd99 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 29 Apr 2026 14:18:57 +0200 Subject: [PATCH 22/25] Revert "Fix partial UTF-32 BOM across reads; restore non-blocking mode in finally; add test" This reverts commit c8bda3e843ecd32cc6615694835bf204572d5c37. --- .../Diagnostics/Process.Multiplexing.Unix.cs | 23 +---- .../Process.Multiplexing.Windows.cs | 14 +--- .../Diagnostics/Process.Multiplexing.cs | 83 +------------------ .../tests/ProcessStreamingTests.cs | 36 +------- 4 files changed, 11 insertions(+), 145 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs index dda6a12f5b0cdc..6f78461dd2e2a8 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs @@ -59,14 +59,6 @@ private IEnumerable ReadPipesToLines( bool outputDone = false, errorDone = false; bool outputPreambleChecked = false, errorPreambleChecked = false; - // Four-byte BOM accumulation buffers: bytes are gathered here until we have - // enough to unambiguously detect the encoding (needed to distinguish - // UTF-32 LE BOM FF FE 00 00 from a UTF-16 LE BOM FF FE when the first read - // delivers fewer than four bytes). - byte[] outputBomAccum = new byte[4]; - byte[] errorBomAccum = new byte[4]; - int outputBomAccumLen = 0, errorBomAccumLen = 0; - List lines = new(); while (!outputDone || !errorDone) @@ -89,15 +81,13 @@ private IEnumerable ReadPipesToLines( { HandlePipeLineRead(currentHandle, ref errorDecoder, ref errorEncoding, byteBuffer, ref errorCharBuffer, ref errorCharStart, ref errorCharEnd, - ref errorPreambleChecked, ref errorDone, standardError: true, lines, - errorBomAccum, ref errorBomAccumLen); + ref errorPreambleChecked, ref errorDone, standardError: true, lines); } else { HandlePipeLineRead(currentHandle, ref outputDecoder, ref outputEncoding, byteBuffer, ref outputCharBuffer, ref outputCharStart, ref outputCharEnd, - ref outputPreambleChecked, ref outputDone, standardError: false, lines, - outputBomAccum, ref outputBomAccumLen); + ref outputPreambleChecked, ref outputDone, standardError: false, lines); } } @@ -114,13 +104,11 @@ private IEnumerable ReadPipesToLines( { if (outputRefAdded) { - Interop.Sys.Fcntl.DangerousSetIsNonBlocking(outputHandle.DangerousGetHandle().ToInt32(), 0); outputHandle.DangerousRelease(); } if (errorRefAdded) { - Interop.Sys.Fcntl.DangerousSetIsNonBlocking(errorHandle.DangerousGetHandle().ToInt32(), 0); errorHandle.DangerousRelease(); } @@ -229,18 +217,15 @@ private static void HandlePipeLineRead( ref bool preambleChecked, ref bool done, bool standardError, - List lines, - byte[] bomAccum, - ref int bomAccumLen) + List lines) { int bytesRead = ReadNonBlocking(handle, byteBuffer, 0); if (bytesRead > 0) { - DecodeBytesAndParseLines(ref decoder, ref encoding, byteBuffer, bytesRead, ref charBuffer, ref charStart, ref charEnd, ref preambleChecked, bomAccum, ref bomAccumLen, standardError, lines); + DecodeBytesAndParseLines(ref decoder, ref encoding, byteBuffer, bytesRead, ref charBuffer, ref charStart, ref charEnd, ref preambleChecked, standardError, lines); } else if (bytesRead == 0) { - FlushBomAccumulation(ref decoder, ref encoding, bomAccum, bomAccumLen, ref preambleChecked, ref charBuffer, ref charStart, ref charEnd, standardError, lines); done = FlushDecoderAndEmitRemainingChars(decoder, ref charBuffer, ref charStart, ref charEnd, standardError, lines); } // bytesRead < 0 means EAGAIN — nothing available yet, let poll retry. diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs index 56d3b9f2062a27..0715bafbd3f738 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs @@ -63,14 +63,6 @@ private IEnumerable ReadPipesToLines( int errorCharStart = 0, errorCharEnd = 0; bool outputPreambleChecked = false, errorPreambleChecked = false; - // Four-byte BOM accumulation buffers: bytes are gathered here until we have - // enough to unambiguously detect the encoding (needed to distinguish - // UTF-32 LE BOM FF FE 00 00 from a UTF-16 LE BOM FF FE when the first read - // delivers fewer than four bytes). - byte[] outputBomAccum = new byte[4]; - byte[] errorBomAccum = new byte[4]; - int outputBomAccumLen = 0, errorBomAccumLen = 0; - unsafe { outputDone = !QueueRead(outputHandle, outputPin.GetAddressOfArrayData(), @@ -109,12 +101,12 @@ private IEnumerable ReadPipesToLines( if (isError) { DecodeBytesAndParseLines(ref errorDecoder, ref errorEncoding, errorByteBuffer, bytesRead, ref errorCharBuffer, ref errorCharStart, - ref errorCharEnd, ref errorPreambleChecked, errorBomAccum, ref errorBomAccumLen, isError, lines); + ref errorCharEnd, ref errorPreambleChecked, isError, lines); } else { DecodeBytesAndParseLines(ref outputDecoder, ref outputEncoding, outputByteBuffer, bytesRead, ref outputCharBuffer, ref outputCharStart, - ref outputCharEnd, ref outputPreambleChecked, outputBomAccum, ref outputBomAccumLen, isError, lines); + ref outputCharEnd, ref outputPreambleChecked, isError, lines); } unsafe @@ -139,12 +131,10 @@ private IEnumerable ReadPipesToLines( { if (isError) { - FlushBomAccumulation(ref errorDecoder, ref errorEncoding, errorBomAccum, errorBomAccumLen, ref errorPreambleChecked, ref errorCharBuffer, ref errorCharStart, ref errorCharEnd, isError, lines); errorDone = FlushDecoderAndEmitRemainingChars(errorDecoder, ref errorCharBuffer, ref errorCharStart, ref errorCharEnd, isError, lines); } else { - FlushBomAccumulation(ref outputDecoder, ref outputEncoding, outputBomAccum, outputBomAccumLen, ref outputPreambleChecked, ref outputCharBuffer, ref outputCharStart, ref outputCharEnd, isError, lines); outputDone = FlushDecoderAndEmitRemainingChars(outputDecoder, ref outputCharBuffer, ref outputCharStart, ref outputCharEnd, isError, lines); } diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs index f8c1f42c5d412f..8d46ff81b506e6 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs @@ -342,94 +342,19 @@ private static void EmitRemainingCharsAsLine( } } - /// - /// Decodes bytes from (and any previously accumulated BOM bytes) - /// into , then parses complete lines. On the first call(s), - /// bytes are accumulated into until four are available so that - /// can unambiguously distinguish a UTF-32 LE BOM - /// (FF FE 00 00) from a UTF-16 LE BOM (FF FE) even when the first OS read - /// delivers only two bytes. - /// - private static void DecodeBytesAndParseLines( - ref Decoder decoder, ref Encoding encoding, - byte[] byteBuffer, int bytesRead, - ref char[] charBuffer, ref int charStart, ref int charEnd, - ref bool preambleChecked, - byte[] bomAccum, ref int bomAccumLen, - bool standardError, List lines) + private static void DecodeBytesAndParseLines(ref Decoder decoder, ref Encoding encoding, byte[] byteBuffer, int bytesRead, ref char[] charBuffer, ref int charStart, ref int charEnd, ref bool preambleChecked, bool standardError, List lines) { + int byteOffset = 0; if (!preambleChecked) { - // Accumulate initial bytes into bomAccum until we have 4, enough to unambiguously - // detect all supported BOMs (UTF-8 = 3 bytes, UTF-16 LE/BE = 2 bytes, but FF FE - // could be the start of a 4-byte UTF-32 LE BOM, so we need all 4 before deciding). - int prevBomAccumLen = bomAccumLen; - if (bomAccumLen < 4 && bytesRead > 0) - { - int toCopy = Math.Min(4 - bomAccumLen, bytesRead); - byteBuffer.AsSpan(0, toCopy).CopyTo(bomAccum.AsSpan(bomAccumLen)); - bomAccumLen += toCopy; - } - - if (bomAccumLen < 4) - { - // Not enough bytes yet for a definitive BOM/preamble check. Defer to the next read. - return; - } - preambleChecked = true; - int bomSkip = SkipPreambleOrDetectEncoding(bomAccum, bomAccumLen, ref encoding, ref decoder); - - // Decode the accumulated BOM bytes (minus the BOM prefix to skip). - int bomToDecode = bomAccumLen - bomSkip; - if (bomToDecode > 0) - { - DecodeAndAppendChars(decoder, bomAccum, bomSkip, bomToDecode, flush: false, ref charBuffer, ref charStart, ref charEnd); - } - - // Decode remaining bytes from the current read (those not consumed into bomAccum). - int consumedFromByteBuffer = bomAccumLen - prevBomAccumLen; - int remaining = bytesRead - consumedFromByteBuffer; - if (remaining > 0) - { - DecodeAndAppendChars(decoder, byteBuffer, consumedFromByteBuffer, remaining, flush: false, ref charBuffer, ref charStart, ref charEnd); - } - - ParseLinesFromCharBuffer(charBuffer, ref charStart, charEnd, standardError, lines); - return; + byteOffset = SkipPreambleOrDetectEncoding(byteBuffer, bytesRead, ref encoding, ref decoder); } - DecodeAndAppendChars(decoder, byteBuffer, 0, bytesRead, flush: false, ref charBuffer, ref charStart, ref charEnd); + DecodeAndAppendChars(decoder, byteBuffer, byteOffset, bytesRead - byteOffset, flush: false, ref charBuffer, ref charStart, ref charEnd); ParseLinesFromCharBuffer(charBuffer, ref charStart, charEnd, standardError, lines); } - /// - /// Resolves any bytes accumulated in when the pipe closes - /// before four bytes have been gathered. The accumulated bytes are BOM-checked and decoded - /// into ; any complete lines are added to - /// . This must be called before - /// at EOF. - /// - private static void FlushBomAccumulation( - ref Decoder decoder, ref Encoding encoding, - byte[] bomAccum, int bomAccumLen, - ref bool preambleChecked, - ref char[] charBuffer, ref int charStart, ref int charEnd, - bool standardError, List lines) - { - if (!preambleChecked && bomAccumLen > 0) - { - preambleChecked = true; - int bomSkip = SkipPreambleOrDetectEncoding(bomAccum, bomAccumLen, ref encoding, ref decoder); - int toDecode = bomAccumLen - bomSkip; - if (toDecode > 0) - { - DecodeAndAppendChars(decoder, bomAccum, bomSkip, toDecode, flush: false, ref charBuffer, ref charStart, ref charEnd); - ParseLinesFromCharBuffer(charBuffer, ref charStart, charEnd, standardError, lines); - } - } - } - private static bool FlushDecoderAndEmitRemainingChars(Decoder decoder, ref char[] charBuffer, ref int charStart, ref int charEnd, bool standardError, List lines) { DecodeAndAppendChars(decoder, Array.Empty(), 0, 0, flush: true, ref charBuffer, ref charStart, ref charEnd); diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs index fe18f8c601abd1..46e8c68e042035 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs @@ -588,41 +588,7 @@ public async Task ReadAllLines_HandlesMixedLineEndings(bool useAsync) Assert.True(process.WaitForExit(WaitInMS)); } - [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - [InlineData(true)] - [InlineData(false)] - public async Task ReadAllLines_HandlesPartialBomAcrossReads(bool useAsync) - { - // Write a UTF-32 LE BOM (FF FE 00 00) as two separate flushed writes so the - // first read can deliver only the first two BOM bytes. Without BOM accumulation, - // FF FE would be misclassified as a UTF-16 LE BOM and the content would be - // decoded with the wrong encoding. - using Process process = CreateProcess(static () => - { - Stream stdout = Console.OpenStandardOutput(); - stdout.Write(new byte[] { 0xFF, 0xFE }); // First half of UTF-32 LE BOM - stdout.Flush(); - stdout.Write(new byte[] { 0x00, 0x00 }); // Second half of BOM - stdout.Write(Encoding.UTF32.GetBytes("hello\n")); // Content (no BOM from GetBytes) - stdout.Flush(); - return RemoteExecutor.SuccessExitCode; - }); - process.StartInfo.RedirectStandardOutput = true; - process.StartInfo.RedirectStandardError = true; - process.StartInfo.StandardOutputEncoding = Encoding.UTF32; - process.Start(); - - List capturedOutput = new(); - List capturedError = new(); - - await EnumerateLines(process, useAsync, capturedOutput, capturedError); - - Assert.Equal(new[] { "hello" }, capturedOutput); - Assert.Empty(capturedError); - Assert.True(process.WaitForExit(WaitInMS)); - } - - + private Process StartLinePrintingProcess(string stdOutText, string stdErrText) { Process process = CreateProcess((stdOut, stdErr) => { From 32043d78c305961a51e243c5dceeeaeab47dcab8 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 29 Apr 2026 15:13:56 +0200 Subject: [PATCH 23/25] keep reading until we have enough bytes to recognize the encoding, emit unconsumed bytes at the end --- .../Diagnostics/Process.Multiplexing.Unix.cs | 44 +++++-- .../Process.Multiplexing.Windows.cs | 51 ++++++-- .../Diagnostics/Process.Multiplexing.cs | 40 +++---- .../tests/ProcessStreamingTests.cs | 113 +++++++++++++----- 4 files changed, 171 insertions(+), 77 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs index 6f78461dd2e2a8..df7209149c78f4 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs @@ -28,7 +28,8 @@ private IEnumerable ReadPipesToLines( SafePipeHandle outputHandle = GetSafeHandleFromStreamReader(_standardOutput!); SafePipeHandle errorHandle = GetSafeHandleFromStreamReader(_standardError!); - byte[] byteBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); + byte[] outputByteBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); + byte[] errorByteBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); char[] outputCharBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); char[] errorCharBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); bool outputRefAdded = false, errorRefAdded = false; @@ -56,6 +57,7 @@ private IEnumerable ReadPipesToLines( Decoder errorDecoder = errorEncoding.GetDecoder(); int outputCharStart = 0, outputCharEnd = 0; int errorCharStart = 0, errorCharEnd = 0; + int unconsumedOutputBytesCount = 0, unconsumedErrorBytesCount = 0; bool outputDone = false, errorDone = false; bool outputPreambleChecked = false, errorPreambleChecked = false; @@ -79,15 +81,17 @@ private IEnumerable ReadPipesToLines( // Use explicit branching to avoid ref locals across yield points. if (isError) { - HandlePipeLineRead(currentHandle, ref errorDecoder, ref errorEncoding, byteBuffer, + HandlePipeLineRead(currentHandle, ref errorDecoder, ref errorEncoding, + errorByteBuffer, ref unconsumedErrorBytesCount, ref errorCharBuffer, ref errorCharStart, ref errorCharEnd, - ref errorPreambleChecked, ref errorDone, standardError: true, lines); + ref errorPreambleChecked, ref errorDone, isError, lines); } else { - HandlePipeLineRead(currentHandle, ref outputDecoder, ref outputEncoding, byteBuffer, + HandlePipeLineRead(currentHandle, ref outputDecoder, ref outputEncoding, + outputByteBuffer, ref unconsumedOutputBytesCount, ref outputCharBuffer, ref outputCharStart, ref outputCharEnd, - ref outputPreambleChecked, ref outputDone, standardError: false, lines); + ref outputPreambleChecked, ref outputDone, isError, lines); } } @@ -112,7 +116,8 @@ private IEnumerable ReadPipesToLines( errorHandle.DangerousRelease(); } - ArrayPool.Shared.Return(byteBuffer); + ArrayPool.Shared.Return(outputByteBuffer); + ArrayPool.Shared.Return(errorByteBuffer); ArrayPool.Shared.Return(outputCharBuffer); ArrayPool.Shared.Return(errorCharBuffer); } @@ -211,6 +216,7 @@ private static void HandlePipeLineRead( ref Decoder decoder, ref Encoding encoding, byte[] byteBuffer, + ref int unconsumedBytesCount, ref char[] charBuffer, ref int charStart, ref int charEnd, @@ -219,14 +225,34 @@ private static void HandlePipeLineRead( bool standardError, List lines) { - int bytesRead = ReadNonBlocking(handle, byteBuffer, 0); + int bytesRead = ReadNonBlocking(handle, byteBuffer, offset: unconsumedBytesCount); if (bytesRead > 0) { - DecodeBytesAndParseLines(ref decoder, ref encoding, byteBuffer, bytesRead, ref charBuffer, ref charStart, ref charEnd, ref preambleChecked, standardError, lines); + ReadOnlySpan bytes = byteBuffer.AsSpan(0, unconsumedBytesCount + bytesRead); + + if (!preambleChecked) + { + if (bytes.Length >= MaxEncodingBytesLength) + { + bytes = bytes.Slice(SkipPreambleOrDetectEncoding(bytes, ref encoding, ref decoder)); + preambleChecked = true; + unconsumedBytesCount = 0; + } + else + { + unconsumedBytesCount += bytesRead; + } + } + + if (preambleChecked) + { + DecodeBytesAndParseLines(decoder, bytes, ref charBuffer, ref charStart, ref charEnd, standardError, lines); + } } else if (bytesRead == 0) { - done = FlushDecoderAndEmitRemainingChars(decoder, ref charBuffer, ref charStart, ref charEnd, standardError, lines); + done = FlushDecoderAndEmitRemainingChars(decoder, byteBuffer.AsSpan(0, unconsumedBytesCount), + ref charBuffer, ref charStart, ref charEnd, standardError, lines); } // bytesRead < 0 means EAGAIN — nothing available yet, let poll retry. } diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs index 0715bafbd3f738..77fed61f80f8ff 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs @@ -61,6 +61,7 @@ private IEnumerable ReadPipesToLines( Decoder errorDecoder = errorEncoding.GetDecoder(); int outputCharStart = 0, outputCharEnd = 0; int errorCharStart = 0, errorCharEnd = 0; + int unconsumedOutputBytesCount = 0, unconsumedErrorBytesCount = 0; bool outputPreambleChecked = false, errorPreambleChecked = false; unsafe @@ -98,15 +99,37 @@ private IEnumerable ReadPipesToLines( if (bytesRead > 0) { - if (isError) + ReadOnlySpan bytes = new ReadOnlySpan( + isError ? errorByteBuffer : outputByteBuffer, + 0, + (isError ? unconsumedErrorBytesCount : unconsumedOutputBytesCount) + bytesRead); + + ref bool preambleChecked = ref (isError ? ref errorPreambleChecked : ref outputPreambleChecked); + ref Encoding currentEncoding = ref (isError ? ref errorEncoding : ref outputEncoding); + ref Decoder currentDecoder = ref (isError ? ref errorDecoder : ref outputDecoder); + ref int unconsumedBytesCount = ref (isError ? ref unconsumedErrorBytesCount : ref unconsumedOutputBytesCount); + + if (!preambleChecked) { - DecodeBytesAndParseLines(ref errorDecoder, ref errorEncoding, errorByteBuffer, bytesRead, ref errorCharBuffer, ref errorCharStart, - ref errorCharEnd, ref errorPreambleChecked, isError, lines); + if (bytes.Length >= 4) + { + bytes = bytes.Slice(SkipPreambleOrDetectEncoding(bytes, ref currentEncoding, ref currentDecoder)); + preambleChecked = true; + unconsumedBytesCount = 0; + } + else + { + unconsumedBytesCount += bytesRead; + } } - else + + if (preambleChecked) { - DecodeBytesAndParseLines(ref outputDecoder, ref outputEncoding, outputByteBuffer, bytesRead, ref outputCharBuffer, ref outputCharStart, - ref outputCharEnd, ref outputPreambleChecked, isError, lines); + DecodeBytesAndParseLines(currentDecoder, bytes, + ref isError ? ref errorCharBuffer : ref outputCharBuffer, + ref isError ? ref errorCharStart : ref outputCharStart, + ref isError ? ref errorCharEnd : ref outputCharEnd, + isError, lines); } unsafe @@ -114,12 +137,14 @@ private IEnumerable ReadPipesToLines( ResetOverlapped(currentEvent, (NativeOverlapped*)currentOverlappedNint); byte* pinPointer = isError - ? errorPin.GetAddressOfArrayData() - : outputPin.GetAddressOfArrayData(); - byte[] currentByteBuffer = isError ? errorByteBuffer : outputByteBuffer; + ? (errorPin.GetAddressOfArrayData() + unconsumedErrorBytesCount) + : (outputPin.GetAddressOfArrayData() + unconsumedOutputBytesCount); + int currentByteLength = isError + ? errorByteBuffer.Length - unconsumedErrorBytesCount + : outputByteBuffer.Length - unconsumedOutputBytesCount; if (!QueueRead(currentHandle, pinPointer, - currentByteBuffer.Length, + currentByteLength, (NativeOverlapped*)currentOverlappedNint, currentEvent)) { bytesRead = 0; // EOF during QueueRead @@ -131,11 +156,13 @@ private IEnumerable ReadPipesToLines( { if (isError) { - errorDone = FlushDecoderAndEmitRemainingChars(errorDecoder, ref errorCharBuffer, ref errorCharStart, ref errorCharEnd, isError, lines); + errorDone = FlushDecoderAndEmitRemainingChars(errorDecoder, errorByteBuffer.AsSpan(0, unconsumedErrorBytesCount), + ref errorCharBuffer, ref errorCharStart, ref errorCharEnd, isError, lines); } else { - outputDone = FlushDecoderAndEmitRemainingChars(outputDecoder, ref outputCharBuffer, ref outputCharStart, ref outputCharEnd, isError, lines); + outputDone = FlushDecoderAndEmitRemainingChars(outputDecoder, outputByteBuffer.AsSpan(0, unconsumedOutputBytesCount), + ref outputCharBuffer, ref outputCharStart, ref outputCharEnd, isError, lines); } currentEvent.Reset(); diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs index 8d46ff81b506e6..d45765f81ed5c9 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs @@ -18,6 +18,7 @@ public partial class Process { /// Initial buffer size for reading process output. private const int InitialReadAllBufferSize = 4096; + private const int MaxEncodingBytesLength = 4; /// /// Reads all standard output and standard error of the process as text. @@ -164,15 +165,13 @@ public IEnumerable ReadAllLines(TimeSpan? timeout = default) /// private static void DecodeAndAppendChars( Decoder decoder, - byte[] byteBuffer, - int byteIndex, - int byteCount, + ReadOnlySpan byteBuffer, bool flush, ref char[] charBuffer, ref int charStartIndex, ref int charEndIndex) { - int charCount = decoder.GetCharCount(byteBuffer, byteIndex, byteCount, flush); + int charCount = decoder.GetCharCount(byteBuffer, flush); // If there isn't enough room at the end, compact the consumed space at the start first // so that if growth is still needed, RentLargerBuffer copies only the unconsumed data. @@ -189,7 +188,7 @@ private static void DecodeAndAppendChars( RentLargerBuffer(ref charBuffer, charEndIndex); } - int decoded = decoder.GetChars(byteBuffer, byteIndex, byteCount, charBuffer, charEndIndex, flush); + int decoded = decoder.GetChars(byteBuffer, charBuffer.AsSpan(charEndIndex), flush); charEndIndex += decoded; } @@ -200,18 +199,20 @@ private static void DecodeAndAppendChars( /// If a different encoding's BOM is detected, updates and /// and returns the BOM length to skip. /// - private static int SkipPreambleOrDetectEncoding(byte[] byteBuffer, int byteCount, ref Encoding encoding, ref Decoder decoder) + private static int SkipPreambleOrDetectEncoding(ReadOnlySpan byteBuffer, ref Encoding encoding, ref Decoder decoder) { + Debug.Assert(byteBuffer.Length >= MaxEncodingBytesLength); + // Check for the encoding's own preamble first (like StreamReader.IsPreamble). ReadOnlySpan preamble = encoding.Preamble; - if (preamble.Length > 0 && byteCount >= preamble.Length - && byteBuffer.AsSpan(0, preamble.Length).SequenceEqual(preamble)) + if (preamble.Length > 0 && byteBuffer.Length >= preamble.Length + && byteBuffer.Slice(0, preamble.Length).SequenceEqual(preamble)) { return preamble.Length; } // No preamble match — check for BOM from other encodings (like StreamReader.DetectEncoding). - if (byteCount >= 2) + if (byteBuffer.Length >= 2) { ushort firstTwoBytes = BinaryPrimitives.ReadUInt16LittleEndian(byteBuffer); @@ -225,7 +226,7 @@ private static int SkipPreambleOrDetectEncoding(byte[] byteBuffer, int byteCount if (firstTwoBytes == 0xFEFF) { - if (byteCount >= 4 && byteBuffer[2] == 0 && byteBuffer[3] == 0) + if (byteBuffer.Length >= 4 && byteBuffer[2] == 0 && byteBuffer[3] == 0) { encoding = Encoding.UTF32; decoder = encoding.GetDecoder(); @@ -237,14 +238,14 @@ private static int SkipPreambleOrDetectEncoding(byte[] byteBuffer, int byteCount return 2; } - if (byteCount >= 3 && firstTwoBytes == 0xBBEF && byteBuffer[2] == 0xBF) + if (byteBuffer.Length >= 3 && firstTwoBytes == 0xBBEF && byteBuffer[2] == 0xBF) { encoding = Encoding.UTF8; decoder = encoding.GetDecoder(); return 3; } - if (byteCount >= 4 && firstTwoBytes == 0 && byteBuffer[2] == 0xFE && byteBuffer[3] == 0xFF) + if (byteBuffer.Length >= 4 && firstTwoBytes == 0 && byteBuffer[2] == 0xFE && byteBuffer[3] == 0xFF) { encoding = new UTF32Encoding(bigEndian: true, byteOrderMark: true); decoder = encoding.GetDecoder(); @@ -342,22 +343,15 @@ private static void EmitRemainingCharsAsLine( } } - private static void DecodeBytesAndParseLines(ref Decoder decoder, ref Encoding encoding, byte[] byteBuffer, int bytesRead, ref char[] charBuffer, ref int charStart, ref int charEnd, ref bool preambleChecked, bool standardError, List lines) + private static void DecodeBytesAndParseLines(Decoder decoder, ReadOnlySpan byteBuffer, ref char[] charBuffer, ref int charStart, ref int charEnd, bool standardError, List lines) { - int byteOffset = 0; - if (!preambleChecked) - { - preambleChecked = true; - byteOffset = SkipPreambleOrDetectEncoding(byteBuffer, bytesRead, ref encoding, ref decoder); - } - - DecodeAndAppendChars(decoder, byteBuffer, byteOffset, bytesRead - byteOffset, flush: false, ref charBuffer, ref charStart, ref charEnd); + DecodeAndAppendChars(decoder, byteBuffer, flush: false, ref charBuffer, ref charStart, ref charEnd); ParseLinesFromCharBuffer(charBuffer, ref charStart, charEnd, standardError, lines); } - private static bool FlushDecoderAndEmitRemainingChars(Decoder decoder, ref char[] charBuffer, ref int charStart, ref int charEnd, bool standardError, List lines) + private static bool FlushDecoderAndEmitRemainingChars(Decoder decoder, ReadOnlySpan unconsumedBytes, ref char[] charBuffer, ref int charStart, ref int charEnd, bool standardError, List lines) { - DecodeAndAppendChars(decoder, Array.Empty(), 0, 0, flush: true, ref charBuffer, ref charStart, ref charEnd); + DecodeAndAppendChars(decoder, unconsumedBytes, flush: true, ref charBuffer, ref charStart, ref charEnd); EmitRemainingCharsAsLine(charBuffer, ref charStart, ref charEnd, standardError, lines); return true; } diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs index 46e8c68e042035..44f8391fe31175 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs @@ -214,10 +214,7 @@ public async Task ReadAllLines_ReadsBothOutputAndError(string standardOutput, st string.IsNullOrEmpty(standardOutput) ? DontPrintAnything : standardOutput, string.IsNullOrEmpty(standardError) ? DontPrintAnything : standardError); - List capturedOutput = new(); - List capturedError = new(); - - await EnumerateLines(process, useAsync, capturedOutput, capturedError); + (List capturedOutput, List capturedError) = await EnumerateLines(process, useAsync); if (string.IsNullOrEmpty(standardOutput)) { @@ -263,10 +260,7 @@ public async Task ReadAllLines_ReadsInterleavedOutput(bool useAsync) process.StartInfo.RedirectStandardError = true; process.Start(); - List capturedOutput = new(); - List capturedError = new(); - - await EnumerateLines(process, useAsync, capturedOutput, capturedError); + (List capturedOutput, List capturedError) = await EnumerateLines(process, useAsync); List expectedOutput = new(); List expectedError = new(); @@ -302,10 +296,7 @@ public async Task ReadAllLines_ReadsLargeOutput(bool useAsync) process.StartInfo.RedirectStandardError = true; process.Start(); - List capturedOutput = new(); - List capturedError = new(); - - await EnumerateLines(process, useAsync, capturedOutput, capturedError); + (List capturedOutput, List capturedError) = await EnumerateLines(process, useAsync); Assert.Equal(lineCount, capturedOutput.Count); for (int i = 0; i < lineCount; i++) @@ -341,10 +332,7 @@ public async Task ReadAllLines_ReadsVeryLongLines(bool useAsync) process.StartInfo.RedirectStandardError = true; process.Start(); - List capturedOutput = new(); - List capturedError = new(); - - await EnumerateLines(process, useAsync, capturedOutput, capturedError); + (List capturedOutput, List capturedError) = await EnumerateLines(process, useAsync); Assert.Equal(lineCount, capturedOutput.Count); Assert.Equal(lineCount, capturedError.Count); @@ -407,10 +395,7 @@ public async Task ReadAllLines_ProcessOutputLineProperties(bool useAsync) { using Process process = StartLinePrintingProcess("stdout_line", "stderr_line"); - List capturedOutput = new(); - List capturedError = new(); - - await EnumerateLines(process, useAsync, capturedOutput, capturedError); + (List capturedOutput, List capturedError) = await EnumerateLines(process, useAsync); Assert.Single(capturedOutput, line => line == "stdout_line"); Assert.Single(capturedError, line => line == "stderr_line"); @@ -496,10 +481,7 @@ public async Task ReadAllLines_WorksWithNonDefaultEncodings(string encodingName, process.StartInfo.StandardErrorEncoding = encoding; process.Start(); - List capturedOutput = new(); - List capturedError = new(); - - await EnumerateLines(process, useAsync, capturedOutput, capturedError); + (List capturedOutput, List capturedError) = await EnumerateLines(process, useAsync); Assert.Equal(new[] { "stdout_line" }, capturedOutput); Assert.Equal(new[] { "stderr_line" }, capturedError); @@ -548,10 +530,7 @@ public async Task ReadAllLines_WorksWithMultiByteCharacters(string encodingName, process.StartInfo.StandardErrorEncoding = encoding; process.Start(); - List capturedOutput = new(); - List capturedError = new(); - - await EnumerateLines(process, useAsync, capturedOutput, capturedError); + (List capturedOutput, List capturedError) = await EnumerateLines(process, useAsync); Assert.Equal(new[] { "hello_\u4e16\u754c_stdout" }, capturedOutput); Assert.Equal(new[] { "hello_\u4e16\u754c_stderr" }, capturedError); @@ -578,16 +557,79 @@ public async Task ReadAllLines_HandlesMixedLineEndings(bool useAsync) process.StartInfo.RedirectStandardError = true; process.Start(); - List capturedOutput = new(); - List capturedError = new(); - - await EnumerateLines(process, useAsync, capturedOutput, capturedError); + (List capturedOutput, List capturedError) = await EnumerateLines(process, useAsync); Assert.Equal(new[] { "lineA", "lineB", "lineC", "lineD" }, capturedOutput); Assert.Empty(capturedError); Assert.True(process.WaitForExit(WaitInMS)); } + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData(true)] + [InlineData(false)] + public async Task ReadAllLines_HandlesPartialBomAcrossReads(bool useAsync) + { + // Write a UTF-32 LE BOM (FF FE 00 00) as two separate flushed writes so the + // first read can deliver only the first two BOM bytes. Without BOM accumulation, + // FF FE would be misclassified as a UTF-16 LE BOM and the content would be + // decoded with the wrong encoding. + using Process process = CreateProcess(static () => + { + Stream stdout = Console.OpenStandardOutput(); + stdout.Write([0xFF, 0xFE]); // First half of UTF-32 LE BOM + stdout.Flush(); + stdout.Write([0x00, 0x00]); // Second half of BOM + stdout.Write(Encoding.UTF32.GetBytes("hello\n")); // Content (no BOM from GetBytes) + stdout.Flush(); + return RemoteExecutor.SuccessExitCode; + }); + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardError = true; + process.StartInfo.StandardOutputEncoding = Encoding.UTF32; + process.Start(); + + (List capturedOutput, List capturedError) = await EnumerateLines(process, useAsync); + + Assert.Equal(new[] { "hello" }, capturedOutput); + Assert.Empty(capturedError); + Assert.True(process.WaitForExit(WaitInMS)); + } + + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData(true)] + [InlineData(false)] + public async Task ReadAllLines_LessThanFourBytes(bool useAsync) + { + using Process process = CreateProcess(static () => + { + Stream stdout = Console.OpenStandardOutput(); + stdout.Write([(byte)'h']); + stdout.Flush(); + stdout.Write([(byte)'i']); + stdout.Flush(); + + Stream error = Console.OpenStandardError(); + error.Write([(byte)'b']); + error.Flush(); + error.Write([(byte)'y']); + error.Flush(); + error.Write([(byte)'e']); + error.Flush(); + + return RemoteExecutor.SuccessExitCode; + }); + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardError = true; + process.StartInfo.StandardOutputEncoding = Encoding.UTF8; + process.Start(); + + (List capturedOutput, List capturedError) = await EnumerateLines(process, useAsync); + + Assert.Equal(new[] { "hi" }, capturedOutput); + Assert.Equal(new[] { "bye" }, capturedError); + Assert.True(process.WaitForExit(WaitInMS)); + } + private Process StartLinePrintingProcess(string stdOutText, string stdErrText) { Process process = CreateProcess((stdOut, stdErr) => @@ -616,8 +658,11 @@ private Process StartLinePrintingProcess(string stdOutText, string stdErrText) /// Helper that wraps both the sync and async line-reading APIs, populating /// the provided output and error lists. /// - private static async Task EnumerateLines(Process process, bool useAsync, List capturedOutput, List capturedError) + private static async Task<(List capturedOutput, List capturedError)> EnumerateLines(Process process, bool useAsync) { + List capturedOutput = new(); + List capturedError = new(); + if (useAsync) { await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) @@ -646,6 +691,8 @@ private static async Task EnumerateLines(Process process, bool useAsync, List Date: Wed, 29 Apr 2026 16:51:34 +0200 Subject: [PATCH 24/25] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../src/System/Diagnostics/Process.Multiplexing.Windows.cs | 2 +- .../System.Diagnostics.Process/tests/ProcessStreamingTests.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs index 77fed61f80f8ff..855206f4fec28a 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs @@ -111,7 +111,7 @@ private IEnumerable ReadPipesToLines( if (!preambleChecked) { - if (bytes.Length >= 4) + if (bytes.Length >= MaxEncodingBytesLength) { bytes = bytes.Slice(SkipPreambleOrDetectEncoding(bytes, ref currentEncoding, ref currentDecoder)); preambleChecked = true; diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs index 44f8391fe31175..15a6d65249e197 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs @@ -655,8 +655,8 @@ private Process StartLinePrintingProcess(string stdOutText, string stdErrText) } /// - /// Helper that wraps both the sync and async line-reading APIs, populating - /// the provided output and error lists. + /// Helper that wraps both the sync and async line-reading APIs and returns + /// the captured output and error lines. /// private static async Task<(List capturedOutput, List capturedError)> EnumerateLines(Process process, bool useAsync) { From 6340d76bdb52aa3d7f0469557a08127fedfeb837 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 29 Apr 2026 17:13:10 +0200 Subject: [PATCH 25/25] address code review feedback: handle preamble/encoding for very short output/error --- .../src/System/Diagnostics/Process.Multiplexing.Unix.cs | 2 +- .../System/Diagnostics/Process.Multiplexing.Windows.cs | 4 ++-- .../src/System/Diagnostics/Process.Multiplexing.cs | 9 ++++++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs index df7209149c78f4..d48cc914a766b6 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs @@ -251,7 +251,7 @@ private static void HandlePipeLineRead( } else if (bytesRead == 0) { - done = FlushDecoderAndEmitRemainingChars(decoder, byteBuffer.AsSpan(0, unconsumedBytesCount), + done = FlushDecoderAndEmitRemainingChars(preambleChecked, encoding, decoder, byteBuffer.AsSpan(0, unconsumedBytesCount), ref charBuffer, ref charStart, ref charEnd, standardError, lines); } // bytesRead < 0 means EAGAIN — nothing available yet, let poll retry. diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs index 855206f4fec28a..4a134a7a77b501 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs @@ -156,12 +156,12 @@ private IEnumerable ReadPipesToLines( { if (isError) { - errorDone = FlushDecoderAndEmitRemainingChars(errorDecoder, errorByteBuffer.AsSpan(0, unconsumedErrorBytesCount), + errorDone = FlushDecoderAndEmitRemainingChars(errorPreambleChecked, errorEncoding, errorDecoder, errorByteBuffer.AsSpan(0, unconsumedErrorBytesCount), ref errorCharBuffer, ref errorCharStart, ref errorCharEnd, isError, lines); } else { - outputDone = FlushDecoderAndEmitRemainingChars(outputDecoder, outputByteBuffer.AsSpan(0, unconsumedOutputBytesCount), + outputDone = FlushDecoderAndEmitRemainingChars(outputPreambleChecked, outputEncoding, outputDecoder, outputByteBuffer.AsSpan(0, unconsumedOutputBytesCount), ref outputCharBuffer, ref outputCharStart, ref outputCharEnd, isError, lines); } diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs index d45765f81ed5c9..6223af6cae0cb6 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs @@ -201,8 +201,6 @@ private static void DecodeAndAppendChars( /// private static int SkipPreambleOrDetectEncoding(ReadOnlySpan byteBuffer, ref Encoding encoding, ref Decoder decoder) { - Debug.Assert(byteBuffer.Length >= MaxEncodingBytesLength); - // Check for the encoding's own preamble first (like StreamReader.IsPreamble). ReadOnlySpan preamble = encoding.Preamble; if (preamble.Length > 0 && byteBuffer.Length >= preamble.Length @@ -349,8 +347,13 @@ private static void DecodeBytesAndParseLines(Decoder decoder, ReadOnlySpan ParseLinesFromCharBuffer(charBuffer, ref charStart, charEnd, standardError, lines); } - private static bool FlushDecoderAndEmitRemainingChars(Decoder decoder, ReadOnlySpan unconsumedBytes, ref char[] charBuffer, ref int charStart, ref int charEnd, bool standardError, List lines) + private static bool FlushDecoderAndEmitRemainingChars(bool preambleChecked, Encoding encoding, Decoder decoder, ReadOnlySpan unconsumedBytes, ref char[] charBuffer, ref int charStart, ref int charEnd, bool standardError, List lines) { + if (!preambleChecked && unconsumedBytes.Length > 0) + { + unconsumedBytes = unconsumedBytes.Slice(SkipPreambleOrDetectEncoding(unconsumedBytes, ref encoding, ref decoder)); + } + DecodeAndAppendChars(decoder, unconsumedBytes, flush: true, ref charBuffer, ref charStart, ref charEnd); EmitRemainingCharsAsLine(charBuffer, ref charStart, ref charEnd, standardError, lines); return true;