diff --git a/src/libraries/System.Diagnostics.Process/ref/System.Diagnostics.Process.cs b/src/libraries/System.Diagnostics.Process/ref/System.Diagnostics.Process.cs index 0907c2dc7b69f1..85682d07a03c30 100644 --- a/src/libraries/System.Diagnostics.Process/ref/System.Diagnostics.Process.cs +++ b/src/libraries/System.Diagnostics.Process/ref/System.Diagnostics.Process.cs @@ -170,6 +170,7 @@ public static void LeaveDebugMode() { } protected void OnExited() { } public (byte[] StandardOutput, byte[] StandardError) ReadAllBytes(System.TimeSpan? timeout = default(System.TimeSpan?)) { throw null; } public System.Threading.Tasks.Task<(byte[] StandardOutput, byte[] StandardError)> ReadAllBytesAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public System.Collections.Generic.IEnumerable ReadAllLines(System.TimeSpan? timeout = default(System.TimeSpan?)) { throw null; } public System.Collections.Generic.IAsyncEnumerable ReadAllLinesAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public (string StandardOutput, string StandardError) ReadAllText(System.TimeSpan? timeout = default(System.TimeSpan?)) { throw null; } public System.Threading.Tasks.Task<(string StandardOutput, string StandardError)> ReadAllTextAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs index 27393f1d50a8f2..d48cc914a766b6 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Unix.cs @@ -1,10 +1,12 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Buffers; +using System.Collections.Generic; using System.ComponentModel; using System.IO; using System.IO.Pipes; -using System.Runtime.InteropServices; +using System.Text; using Microsoft.Win32.SafeHandles; namespace System.Diagnostics @@ -13,6 +15,248 @@ public partial class Process { private static SafePipeHandle GetSafeHandleFromStreamReader(StreamReader reader) => ((AnonymousPipeClientStream)reader.BaseStream).SafePipeHandle; + /// + /// Reads from both standard output and standard error pipes as lines of text using Unix + /// poll-based multiplexing with non-blocking reads. + /// Buffers are rented from the pool and returned when enumeration completes. + /// + private IEnumerable ReadPipesToLines( + int timeoutMs, + Encoding outputEncoding, + Encoding errorEncoding) + { + SafePipeHandle outputHandle = GetSafeHandleFromStreamReader(_standardOutput!); + SafePipeHandle errorHandle = GetSafeHandleFromStreamReader(_standardError!); + + byte[] outputByteBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); + byte[] errorByteBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); + char[] outputCharBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); + char[] errorCharBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); + bool outputRefAdded = false, errorRefAdded = false; + + try + { + outputHandle.DangerousAddRef(ref outputRefAdded); + errorHandle.DangerousAddRef(ref errorRefAdded); + + int outputFd = outputHandle.DangerousGetHandle().ToInt32(); + int errorFd = errorHandle.DangerousGetHandle().ToInt32(); + + if (Interop.Sys.Fcntl.DangerousSetIsNonBlocking(outputFd, 1) != 0 + || Interop.Sys.Fcntl.DangerousSetIsNonBlocking(errorFd, 1) != 0) + { + throw new Win32Exception(); + } + + // Cannot use stackalloc in an iterator method; use a regular array. + Interop.PollEvent[] pollFds = new Interop.PollEvent[2]; + + long deadline = timeoutMs >= 0 ? Environment.TickCount64 + timeoutMs : long.MaxValue; + + Decoder outputDecoder = outputEncoding.GetDecoder(); + Decoder errorDecoder = errorEncoding.GetDecoder(); + int outputCharStart = 0, outputCharEnd = 0; + int errorCharStart = 0, errorCharEnd = 0; + int unconsumedOutputBytesCount = 0, unconsumedErrorBytesCount = 0; + bool outputDone = false, errorDone = false; + bool outputPreambleChecked = false, errorPreambleChecked = false; + + List lines = new(); + + while (!outputDone || !errorDone) + { + int numFds = PollForPipeActivity(pollFds, errorFd, outputFd, errorDone, outputDone, deadline, timeoutMs, out int errorIndex, out int outputIndex); + + // Process error pipe first (lower index) when both have data available. + for (int i = 0; i < numFds; i++) + { + if (pollFds[i].TriggeredEvents == Interop.PollEvents.POLLNONE) + { + continue; + } + + bool isError = i == errorIndex; + SafePipeHandle currentHandle = isError ? errorHandle : outputHandle; + + // Use explicit branching to avoid ref locals across yield points. + if (isError) + { + HandlePipeLineRead(currentHandle, ref errorDecoder, ref errorEncoding, + errorByteBuffer, ref unconsumedErrorBytesCount, + ref errorCharBuffer, ref errorCharStart, ref errorCharEnd, + ref errorPreambleChecked, ref errorDone, isError, lines); + } + else + { + HandlePipeLineRead(currentHandle, ref outputDecoder, ref outputEncoding, + outputByteBuffer, ref unconsumedOutputBytesCount, + ref outputCharBuffer, ref outputCharStart, ref outputCharEnd, + ref outputPreambleChecked, ref outputDone, isError, lines); + } + } + + // Yield parsed lines outside of any ref-local scope. + foreach (ProcessOutputLine line in lines) + { + yield return line; + } + + lines.Clear(); + } + } + finally + { + if (outputRefAdded) + { + outputHandle.DangerousRelease(); + } + + if (errorRefAdded) + { + errorHandle.DangerousRelease(); + } + + ArrayPool.Shared.Return(outputByteBuffer); + ArrayPool.Shared.Return(errorByteBuffer); + ArrayPool.Shared.Return(outputCharBuffer); + ArrayPool.Shared.Return(errorCharBuffer); + } + } + + /// + /// Populates the poll fd array with the active pipe file descriptors. + /// Error is added first so it gets serviced first when both have data. + /// Returns the number of active file descriptors. + /// + private static int PreparePollFds( + Span pollFds, + int errorFd, int outputFd, + bool errorDone, bool outputDone, + out int errorIndex, out int outputIndex) + { + int numFds = 0; + errorIndex = -1; + outputIndex = -1; + + if (!errorDone) + { + errorIndex = numFds; + pollFds[numFds].FileDescriptor = errorFd; + pollFds[numFds].Events = Interop.PollEvents.POLLIN; + pollFds[numFds].TriggeredEvents = Interop.PollEvents.POLLNONE; + numFds++; + } + + if (!outputDone) + { + outputIndex = numFds; + pollFds[numFds].FileDescriptor = outputFd; + pollFds[numFds].Events = Interop.PollEvents.POLLIN; + pollFds[numFds].TriggeredEvents = Interop.PollEvents.POLLNONE; + numFds++; + } + + return numFds; + } + + /// + /// Prepares the poll fd array, checks the remaining timeout, calls poll(2), and handles + /// errors. Returns the number of polled fds, or 0 if poll was interrupted (EINTR) and + /// the caller should retry. + /// + private static int PollForPipeActivity( + Span pollFds, + int errorFd, int outputFd, + bool errorDone, bool outputDone, + long deadline, int timeoutMs, + out int errorIndex, out int outputIndex) + { + int numFds = PreparePollFds(pollFds, errorFd, outputFd, errorDone, outputDone, out errorIndex, out outputIndex); + + if (!TryGetRemainingTimeout(deadline, timeoutMs, out int pollTimeout)) + { + throw new TimeoutException(); + } + + uint triggered = 0; + Interop.Error pollError; + unsafe + { + fixed (Interop.PollEvent* pPollFds = pollFds) + { + pollError = Interop.Sys.Poll(pPollFds, (uint)numFds, pollTimeout, &triggered); + } + } + + if (pollError != Interop.Error.SUCCESS) + { + if (pollError == Interop.Error.EINTR) + { + return 0; + } + + throw new Win32Exception(Interop.Sys.ConvertErrorPalToPlatform(pollError)); + } + + if (triggered == 0) + { + throw new TimeoutException(); + } + + return numFds; + } + + /// + /// Handles a poll notification for a single pipe: reads bytes, decodes to chars, + /// strips BOM on first decode, parses lines, compacts the char buffer, and sets + /// to on EOF. + /// + private static void HandlePipeLineRead( + SafePipeHandle handle, + ref Decoder decoder, + ref Encoding encoding, + byte[] byteBuffer, + ref int unconsumedBytesCount, + ref char[] charBuffer, + ref int charStart, + ref int charEnd, + ref bool preambleChecked, + ref bool done, + bool standardError, + List lines) + { + int bytesRead = ReadNonBlocking(handle, byteBuffer, offset: unconsumedBytesCount); + if (bytesRead > 0) + { + ReadOnlySpan bytes = byteBuffer.AsSpan(0, unconsumedBytesCount + bytesRead); + + if (!preambleChecked) + { + if (bytes.Length >= MaxEncodingBytesLength) + { + bytes = bytes.Slice(SkipPreambleOrDetectEncoding(bytes, ref encoding, ref decoder)); + preambleChecked = true; + unconsumedBytesCount = 0; + } + else + { + unconsumedBytesCount += bytesRead; + } + } + + if (preambleChecked) + { + DecodeBytesAndParseLines(decoder, bytes, ref charBuffer, ref charStart, ref charEnd, standardError, lines); + } + } + else if (bytesRead == 0) + { + done = FlushDecoderAndEmitRemainingChars(preambleChecked, encoding, decoder, byteBuffer.AsSpan(0, unconsumedBytesCount), + ref charBuffer, ref charStart, ref charEnd, standardError, lines); + } + // bytesRead < 0 means EAGAIN — nothing available yet, let poll retry. + } + /// /// Reads from both standard output and standard error pipes using Unix poll-based multiplexing /// with non-blocking reads. @@ -43,59 +287,7 @@ private static void ReadPipes( bool outputDone = false, errorDone = false; while (!outputDone || !errorDone) { - int numFds = 0; - - int outputIndex = -1; - int errorIndex = -1; - - if (!outputDone) - { - outputIndex = numFds; - pollFds[numFds].FileDescriptor = outputFd; - pollFds[numFds].Events = Interop.PollEvents.POLLIN; - pollFds[numFds].TriggeredEvents = Interop.PollEvents.POLLNONE; - numFds++; - } - - if (!errorDone) - { - errorIndex = numFds; - pollFds[numFds].FileDescriptor = errorFd; - pollFds[numFds].Events = Interop.PollEvents.POLLIN; - pollFds[numFds].TriggeredEvents = Interop.PollEvents.POLLNONE; - numFds++; - } - - int pollTimeout; - if (!TryGetRemainingTimeout(deadline, timeoutMs, out pollTimeout)) - { - throw new TimeoutException(); - } - - unsafe - { - uint triggered; - fixed (Interop.PollEvent* pPollFds = pollFds) - { - Interop.Error error = Interop.Sys.Poll(pPollFds, (uint)numFds, pollTimeout, &triggered); - if (error != Interop.Error.SUCCESS) - { - if (error == Interop.Error.EINTR) - { - // We don't re-issue the poll immediately because we need to check - // if we've already exceeded the overall timeout. - continue; - } - - throw new Win32Exception(Interop.Sys.ConvertErrorPalToPlatform(error)); - } - - if (triggered == 0) - { - throw new TimeoutException(); - } - } - } + int numFds = PollForPipeActivity(pollFds, errorFd, outputFd, errorDone, outputDone, deadline, timeoutMs, out int errorIndex, out int outputIndex); for (int i = 0; i < numFds; i++) { diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs index fb30cc07fb254c..4a134a7a77b501 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.Windows.cs @@ -1,9 +1,12 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Buffers; +using System.Collections.Generic; using System.ComponentModel; using System.IO; using System.Runtime.InteropServices; +using System.Text; using System.Threading; using Microsoft.Win32.SafeHandles; @@ -13,6 +16,196 @@ public partial class Process { private static SafeFileHandle GetSafeHandleFromStreamReader(StreamReader reader) => ((FileStream)reader.BaseStream).SafeFileHandle; + /// + /// Reads from both standard output and standard error pipes as lines of text using Windows + /// overlapped IO with wait handles for single-threaded synchronous multiplexing. + /// Buffers are rented from the pool and returned when enumeration completes. + /// + private IEnumerable ReadPipesToLines( + int timeoutMs, + Encoding outputEncoding, + Encoding errorEncoding) + { + SafeFileHandle outputHandle = GetSafeHandleFromStreamReader(_standardOutput!); + SafeFileHandle errorHandle = GetSafeHandleFromStreamReader(_standardError!); + + byte[] outputByteBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); + byte[] errorByteBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); + char[] outputCharBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); + char[] errorCharBuffer = ArrayPool.Shared.Rent(InitialReadAllBufferSize); + PinnedGCHandle outputPin = default, errorPin = default; + // NativeOverlapped* can't be used as iterator state machine fields (pointers aren't + // allowed in managed types). Store as nint and cast back inside scoped unsafe blocks. + nint outputOverlappedNint = 0, errorOverlappedNint = 0; + EventWaitHandle? outputEvent = null, errorEvent = null; + bool outputDone = true, errorDone = true; + + try + { + outputPin = new PinnedGCHandle(outputByteBuffer); + errorPin = new PinnedGCHandle(errorByteBuffer); + + outputEvent = new EventWaitHandle(initialState: false, EventResetMode.ManualReset); + errorEvent = new EventWaitHandle(initialState: false, EventResetMode.ManualReset); + + unsafe + { + outputOverlappedNint = (nint)AllocateOverlapped(outputEvent); + errorOverlappedNint = (nint)AllocateOverlapped(errorEvent); + } + + // Error output gets index 0 so WaitAny services it first when both are signaled. + WaitHandle[] waitHandles = [errorEvent, outputEvent]; + + Decoder outputDecoder = outputEncoding.GetDecoder(); + Decoder errorDecoder = errorEncoding.GetDecoder(); + int outputCharStart = 0, outputCharEnd = 0; + int errorCharStart = 0, errorCharEnd = 0; + int unconsumedOutputBytesCount = 0, unconsumedErrorBytesCount = 0; + bool outputPreambleChecked = false, errorPreambleChecked = false; + + unsafe + { + outputDone = !QueueRead(outputHandle, outputPin.GetAddressOfArrayData(), + outputByteBuffer.Length, (NativeOverlapped*)outputOverlappedNint, outputEvent); + errorDone = !QueueRead(errorHandle, errorPin.GetAddressOfArrayData(), + errorByteBuffer.Length, (NativeOverlapped*)errorOverlappedNint, errorEvent); + } + + long deadline = timeoutMs >= 0 ? Environment.TickCount64 + timeoutMs : long.MaxValue; + List lines = new(); + + while (!outputDone || !errorDone) + { + int waitResult = TryGetRemainingTimeout(deadline, timeoutMs, out int remainingMilliseconds) + ? WaitHandle.WaitAny(waitHandles, remainingMilliseconds) + : WaitHandle.WaitTimeout; + + if (waitResult == WaitHandle.WaitTimeout) + { + throw new TimeoutException(); + } + + bool isError = waitResult == 0; + nint currentOverlappedNint = isError ? errorOverlappedNint : outputOverlappedNint; + SafeFileHandle currentHandle = isError ? errorHandle : outputHandle; + EventWaitHandle currentEvent = isError ? errorEvent! : outputEvent!; + + int bytesRead; + unsafe + { + bytesRead = GetOverlappedResultForPipe(currentHandle, (NativeOverlapped*)currentOverlappedNint); + } + + if (bytesRead > 0) + { + ReadOnlySpan bytes = new ReadOnlySpan( + isError ? errorByteBuffer : outputByteBuffer, + 0, + (isError ? unconsumedErrorBytesCount : unconsumedOutputBytesCount) + bytesRead); + + ref bool preambleChecked = ref (isError ? ref errorPreambleChecked : ref outputPreambleChecked); + ref Encoding currentEncoding = ref (isError ? ref errorEncoding : ref outputEncoding); + ref Decoder currentDecoder = ref (isError ? ref errorDecoder : ref outputDecoder); + ref int unconsumedBytesCount = ref (isError ? ref unconsumedErrorBytesCount : ref unconsumedOutputBytesCount); + + if (!preambleChecked) + { + if (bytes.Length >= MaxEncodingBytesLength) + { + bytes = bytes.Slice(SkipPreambleOrDetectEncoding(bytes, ref currentEncoding, ref currentDecoder)); + preambleChecked = true; + unconsumedBytesCount = 0; + } + else + { + unconsumedBytesCount += bytesRead; + } + } + + if (preambleChecked) + { + DecodeBytesAndParseLines(currentDecoder, bytes, + ref isError ? ref errorCharBuffer : ref outputCharBuffer, + ref isError ? ref errorCharStart : ref outputCharStart, + ref isError ? ref errorCharEnd : ref outputCharEnd, + isError, lines); + } + + unsafe + { + ResetOverlapped(currentEvent, (NativeOverlapped*)currentOverlappedNint); + + byte* pinPointer = isError + ? (errorPin.GetAddressOfArrayData() + unconsumedErrorBytesCount) + : (outputPin.GetAddressOfArrayData() + unconsumedOutputBytesCount); + int currentByteLength = isError + ? errorByteBuffer.Length - unconsumedErrorBytesCount + : outputByteBuffer.Length - unconsumedOutputBytesCount; + + if (!QueueRead(currentHandle, pinPointer, + currentByteLength, + (NativeOverlapped*)currentOverlappedNint, currentEvent)) + { + bytesRead = 0; // EOF during QueueRead + } + } + } + + if (bytesRead == 0) // EOF + { + if (isError) + { + errorDone = FlushDecoderAndEmitRemainingChars(errorPreambleChecked, errorEncoding, errorDecoder, errorByteBuffer.AsSpan(0, unconsumedErrorBytesCount), + ref errorCharBuffer, ref errorCharStart, ref errorCharEnd, isError, lines); + } + else + { + outputDone = FlushDecoderAndEmitRemainingChars(outputPreambleChecked, outputEncoding, outputDecoder, outputByteBuffer.AsSpan(0, unconsumedOutputBytesCount), + ref outputCharBuffer, ref outputCharStart, ref outputCharEnd, isError, lines); + } + + currentEvent.Reset(); + } + + // Yield parsed lines outside of any unsafe or ref-local scope. + foreach (ProcessOutputLine line in lines) + { + yield return line; + } + + lines.Clear(); + } + } + finally + { + unsafe + { + if (outputOverlappedNint != 0) + { + CancelPendingIOIfNeeded(outputHandle, outputDone, (NativeOverlapped*)outputOverlappedNint); + NativeMemory.Free((void*)outputOverlappedNint); + } + + if (errorOverlappedNint != 0) + { + CancelPendingIOIfNeeded(errorHandle, errorDone, (NativeOverlapped*)errorOverlappedNint); + NativeMemory.Free((void*)errorOverlappedNint); + } + } + + outputEvent?.Dispose(); + errorEvent?.Dispose(); + outputPin.Dispose(); + errorPin.Dispose(); + + ArrayPool.Shared.Return(outputByteBuffer); + ArrayPool.Shared.Return(errorByteBuffer); + ArrayPool.Shared.Return(outputCharBuffer); + ArrayPool.Shared.Return(errorCharBuffer); + } + } + /// /// Reads from both standard output and standard error pipes using Windows overlapped IO /// with wait handles for single-threaded synchronous multiplexing. diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs index 5b11679d82c451..6223af6cae0cb6 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.Multiplexing.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Buffers; +using System.Buffers.Binary; using System.Collections.Generic; using System.IO; using System.Runtime.CompilerServices; @@ -17,6 +18,7 @@ public partial class Process { /// Initial buffer size for reading process output. private const int InitialReadAllBufferSize = 4096; + private const int MaxEncodingBytesLength = 4; /// /// Reads all standard output and standard error of the process as text. @@ -113,6 +115,250 @@ public partial class Process } } + /// + /// Reads all standard output and standard error of the process as lines of text, + /// interleaving them as they become available. + /// + /// + /// The maximum amount of time to wait for the streams to be fully read. + /// When , waits indefinitely. + /// + /// + /// An enumerable of instances representing the lines + /// read from standard output and standard error. + /// + /// + /// Lines from standard output and standard error are yielded as they become available. + /// When data is available in both standard output and standard error, standard error + /// is processed first. + /// + /// + /// Standard output or standard error has not been redirected. + /// -or- + /// A redirected stream has already been used for synchronous or asynchronous reading. + /// + /// + /// The operation did not complete within the specified . + /// + /// + /// The process has been disposed. + /// + public IEnumerable ReadAllLines(TimeSpan? timeout = default) + { + ValidateReadAllState(); + + int timeoutMs = timeout.HasValue + ? ProcessUtils.ToTimeoutMilliseconds(timeout.Value) + : Timeout.Infinite; + + Encoding outputEncoding = _startInfo?.StandardOutputEncoding ?? GetStandardOutputEncoding(); + Encoding errorEncoding = _startInfo?.StandardErrorEncoding ?? GetStandardOutputEncoding(); + + return ReadPipesToLines(timeoutMs, outputEncoding, errorEncoding); + } + + /// + /// Decodes bytes from the byte buffer using the and appends the + /// resulting characters to the char buffer, growing it if necessary. + /// To flush the decoder at EOF, pass an empty byte array with set to + /// . + /// + private static void DecodeAndAppendChars( + Decoder decoder, + ReadOnlySpan byteBuffer, + bool flush, + ref char[] charBuffer, + ref int charStartIndex, + ref int charEndIndex) + { + int charCount = decoder.GetCharCount(byteBuffer, flush); + + // If there isn't enough room at the end, compact the consumed space at the start first + // so that if growth is still needed, RentLargerBuffer copies only the unconsumed data. + if (charEndIndex + charCount > charBuffer.Length && charStartIndex > 0) + { + int remaining = charEndIndex - charStartIndex; + Array.Copy(charBuffer, charStartIndex, charBuffer, 0, remaining); + charStartIndex = 0; + charEndIndex = remaining; + } + + while (charEndIndex + charCount > charBuffer.Length) + { + RentLargerBuffer(ref charBuffer, charEndIndex); + } + + int decoded = decoder.GetChars(byteBuffer, charBuffer.AsSpan(charEndIndex), flush); + charEndIndex += decoded; + } + + /// + /// Checks for the encoding's preamble or a BOM from a different encoding at the start of + /// the byte buffer, mimicking behavior. + /// If the encoding's own preamble is found, returns the number of bytes to skip. + /// If a different encoding's BOM is detected, updates and + /// and returns the BOM length to skip. + /// + private static int SkipPreambleOrDetectEncoding(ReadOnlySpan byteBuffer, ref Encoding encoding, ref Decoder decoder) + { + // Check for the encoding's own preamble first (like StreamReader.IsPreamble). + ReadOnlySpan preamble = encoding.Preamble; + if (preamble.Length > 0 && byteBuffer.Length >= preamble.Length + && byteBuffer.Slice(0, preamble.Length).SequenceEqual(preamble)) + { + return preamble.Length; + } + + // No preamble match — check for BOM from other encodings (like StreamReader.DetectEncoding). + if (byteBuffer.Length >= 2) + { + ushort firstTwoBytes = BinaryPrimitives.ReadUInt16LittleEndian(byteBuffer); + + if (firstTwoBytes == 0xFFFE) + { + // Big Endian Unicode + encoding = Encoding.BigEndianUnicode; + decoder = encoding.GetDecoder(); + return 2; + } + + if (firstTwoBytes == 0xFEFF) + { + if (byteBuffer.Length >= 4 && byteBuffer[2] == 0 && byteBuffer[3] == 0) + { + encoding = Encoding.UTF32; + decoder = encoding.GetDecoder(); + return 4; + } + + encoding = Encoding.Unicode; + decoder = encoding.GetDecoder(); + return 2; + } + + if (byteBuffer.Length >= 3 && firstTwoBytes == 0xBBEF && byteBuffer[2] == 0xBF) + { + encoding = Encoding.UTF8; + decoder = encoding.GetDecoder(); + return 3; + } + + if (byteBuffer.Length >= 4 && firstTwoBytes == 0 && byteBuffer[2] == 0xFE && byteBuffer[3] == 0xFF) + { + encoding = new UTF32Encoding(bigEndian: true, byteOrderMark: true); + decoder = encoding.GetDecoder(); + return 4; + } + } + + return 0; + } + + /// + /// Scans the char buffer from to for complete + /// lines (delimited by \r, \n, or \r\n), adds each as a + /// to , and advances + /// past the consumed data. + /// This matches behavior used by the async path. + /// + private static void ParseLinesFromCharBuffer( + char[] buffer, + ref int startIndex, + int endIndex, + bool standardError, + List lines) + { + while (startIndex < endIndex) + { + int remaining = endIndex - startIndex; + int lineEnd = buffer.AsSpan(startIndex, remaining).IndexOfAny('\r', '\n'); + if (lineEnd == -1) + { + break; + } + + char terminator = buffer[startIndex + lineEnd]; + + // If we found '\r', we need to check for a following '\n' to treat \r\n as one terminator. + // If '\n' isn't available yet (end of current data), stop and wait for more data. + if (terminator == '\r') + { + if (startIndex + lineEnd + 1 >= endIndex) + { + // The '\r' is at the very end of available data — we can't tell yet + // whether it's a standalone '\r' or part of '\r\n'. Wait for more data. + break; + } + + lines.Add(new ProcessOutputLine( + new string(buffer, startIndex, lineEnd), + standardError)); + + // Skip \r and also \n if it immediately follows. + startIndex += lineEnd + 1; + if (startIndex < endIndex && buffer[startIndex] == '\n') + { + startIndex++; + } + } + else + { + // terminator == '\n' + lines.Add(new ProcessOutputLine( + new string(buffer, startIndex, lineEnd), + standardError)); + + startIndex += lineEnd + 1; + } + } + } + + /// + /// Emits any remaining characters in the buffer as a final line when an EOF is reached. + /// A trailing \r is stripped to match behavior. + /// + private static void EmitRemainingCharsAsLine( + char[] buffer, + ref int startIndex, + ref int endIndex, + bool standardError, + List lines) + { + if (startIndex < endIndex) + { + int length = endIndex - startIndex; + if (length > 0 && buffer[startIndex + length - 1] == '\r') + { + length--; + } + + lines.Add(new ProcessOutputLine( + new string(buffer, startIndex, length), + standardError)); + + startIndex = 0; + endIndex = 0; + } + } + + private static void DecodeBytesAndParseLines(Decoder decoder, ReadOnlySpan byteBuffer, ref char[] charBuffer, ref int charStart, ref int charEnd, bool standardError, List lines) + { + DecodeAndAppendChars(decoder, byteBuffer, flush: false, ref charBuffer, ref charStart, ref charEnd); + ParseLinesFromCharBuffer(charBuffer, ref charStart, charEnd, standardError, lines); + } + + private static bool FlushDecoderAndEmitRemainingChars(bool preambleChecked, Encoding encoding, Decoder decoder, ReadOnlySpan unconsumedBytes, ref char[] charBuffer, ref int charStart, ref int charEnd, bool standardError, List lines) + { + if (!preambleChecked && unconsumedBytes.Length > 0) + { + unconsumedBytes = unconsumedBytes.Slice(SkipPreambleOrDetectEncoding(unconsumedBytes, ref encoding, ref decoder)); + } + + DecodeAndAppendChars(decoder, unconsumedBytes, flush: true, ref charBuffer, ref charStart, ref charEnd); + EmitRemainingCharsAsLine(charBuffer, ref charStart, ref charEnd, standardError, lines); + return true; + } + /// /// Asynchronously reads all standard output and standard error of the process as text. /// @@ -419,17 +665,17 @@ private void ReadPipesToBuffers( } /// - /// Rents a larger buffer from the array pool and copies the existing data to it. + /// Rents a larger buffer from the array pool, copies existing data, and returns the old buffer to the pool. /// - private static void RentLargerBuffer(ref byte[] buffer, int bytesRead) + private static void RentLargerBuffer(ref T[] buffer, int dataLength) { int newSize = (int)Math.Min((long)buffer.Length * 2, Array.MaxLength); newSize = Math.Max(buffer.Length + 1, newSize); - byte[] newBuffer = ArrayPool.Shared.Rent(newSize); - Buffer.BlockCopy(buffer, 0, newBuffer, 0, bytesRead); - byte[] oldBuffer = buffer; + T[] newBuffer = ArrayPool.Shared.Rent(newSize); + Array.Copy(buffer, newBuffer, dataLength); + T[] oldBuffer = buffer; buffer = newBuffer; - ArrayPool.Shared.Return(oldBuffer); + ArrayPool.Shared.Return(oldBuffer); } private static bool TryGetRemainingTimeout(long deadline, int originalTimeout, out int remainingTimeoutMs) diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs index 4b2967e1085611..15a6d65249e197 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessStreamingTests.cs @@ -2,6 +2,8 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Generic; +using System.IO; +using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.DotNet.RemoteExecutor; @@ -13,65 +15,109 @@ public class ProcessStreamingTests : ProcessTestBase { private const string DontPrintAnything = "DO_NOT_PRINT"; - [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - public async Task ReadAllLinesAsync_ThrowsAfterDispose() + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData(true)] + [InlineData(false)] + public async Task ReadAllLines_ThrowsAfterDispose(bool useAsync) { - Process process = CreateProcess(RemotelyInvokable.Dummy); + using Process process = CreateProcess(static () => RemoteExecutor.SuccessExitCode); process.Start(); Assert.True(process.WaitForExit(WaitInMS)); process.Dispose(); - await Assert.ThrowsAsync(async () => + if (useAsync) { - await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + await Assert.ThrowsAsync(async () => { - } - }); + await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + { + } + }); + } + else + { + Assert.Throws(() => + { + foreach (ProcessOutputLine line in process.ReadAllLines()) + { + } + }); + } } - [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - public async Task ReadAllLinesAsync_ThrowsWhenNoStreamsRedirected() + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData(true)] + [InlineData(false)] + public async Task ReadAllLines_ThrowsWhenNoStreamsRedirected(bool useAsync) { - Process process = CreateProcess(RemotelyInvokable.Dummy); + using Process process = CreateProcess(static () => RemoteExecutor.SuccessExitCode); process.Start(); - await Assert.ThrowsAsync(async () => + if (useAsync) { - await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + await Assert.ThrowsAsync(async () => { - } - }); + await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + { + } + }); + } + else + { + Assert.Throws(() => + { + foreach (ProcessOutputLine line in process.ReadAllLines()) + { + } + }); + } Assert.True(process.WaitForExit(WaitInMS)); } [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - [InlineData(true)] - [InlineData(false)] - public async Task ReadAllLinesAsync_ThrowsWhenOnlyOutputOrErrorIsRedirected(bool standardOutput) + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public async Task ReadAllLines_ThrowsWhenOnlyOutputOrErrorIsRedirected(bool standardOutput, bool useAsync) { - Process process = CreateProcess(RemotelyInvokable.Dummy); + using Process process = CreateProcess(static () => RemoteExecutor.SuccessExitCode); process.StartInfo.RedirectStandardOutput = standardOutput; process.StartInfo.RedirectStandardError = !standardOutput; process.Start(); - await Assert.ThrowsAsync(async () => + if (useAsync) { - await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + await Assert.ThrowsAsync(async () => { - } - }); + await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + { + } + }); + } + else + { + Assert.Throws(() => + { + foreach (ProcessOutputLine line in process.ReadAllLines()) + { + } + }); + } Assert.True(process.WaitForExit(WaitInMS)); } [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - [InlineData(true)] - [InlineData(false)] - public async Task ReadAllLinesAsync_ThrowsWhenOutputOrErrorIsInSyncMode(bool standardOutput) + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public async Task ReadAllLines_ThrowsWhenOutputOrErrorIsInSyncMode(bool standardOutput, bool useAsync) { - Process process = CreateProcess(RemotelyInvokable.Dummy); + using Process process = CreateProcess(static () => RemoteExecutor.SuccessExitCode); process.StartInfo.RedirectStandardOutput = true; process.StartInfo.RedirectStandardError = true; process.Start(); @@ -79,22 +125,36 @@ public async Task ReadAllLinesAsync_ThrowsWhenOutputOrErrorIsInSyncMode(bool sta // Access the StreamReader property to set the stream to sync mode _ = standardOutput ? process.StandardOutput : process.StandardError; - await Assert.ThrowsAsync(async () => + if (useAsync) { - await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + await Assert.ThrowsAsync(async () => { - } - }); + await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + { + } + }); + } + else + { + Assert.Throws(() => + { + foreach (ProcessOutputLine line in process.ReadAllLines()) + { + } + }); + } Assert.True(process.WaitForExit(WaitInMS)); } [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - [InlineData(true)] - [InlineData(false)] - public async Task ReadAllLinesAsync_ThrowsWhenOutputOrErrorIsInAsyncMode(bool standardOutput) + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public async Task ReadAllLines_ThrowsWhenOutputOrErrorIsInAsyncMode(bool standardOutput, bool useAsync) { - Process process = CreateProcess(RemotelyInvokable.StreamBody); + using Process process = CreateProcess(static () => RemoteExecutor.SuccessExitCode); process.StartInfo.RedirectStandardOutput = true; process.StartInfo.RedirectStandardError = true; process.Start(); @@ -108,12 +168,24 @@ public async Task ReadAllLinesAsync_ThrowsWhenOutputOrErrorIsInAsyncMode(bool st process.BeginErrorReadLine(); } - await Assert.ThrowsAsync(async () => + if (useAsync) { - await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + await Assert.ThrowsAsync(async () => { - } - }); + await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + { + } + }); + } + else + { + Assert.Throws(() => + { + foreach (ProcessOutputLine line in process.ReadAllLines()) + { + } + }); + } if (standardOutput) { @@ -128,30 +200,21 @@ await Assert.ThrowsAsync(async () => } [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - [InlineData("hello", "world")] - [InlineData("just output", "")] - [InlineData("", "just error")] - [InlineData("", "")] - public async Task ReadAllLinesAsync_ReadsBothOutputAndError(string standardOutput, string standardError) + [InlineData("hello", "world", true)] + [InlineData("hello", "world", false)] + [InlineData("just output", "", true)] + [InlineData("just output", "", false)] + [InlineData("", "just error", true)] + [InlineData("", "just error", false)] + [InlineData("", "", true)] + [InlineData("", "", false)] + public async Task ReadAllLines_ReadsBothOutputAndError(string standardOutput, string standardError, bool useAsync) { using Process process = StartLinePrintingProcess( string.IsNullOrEmpty(standardOutput) ? DontPrintAnything : standardOutput, string.IsNullOrEmpty(standardError) ? DontPrintAnything : standardError); - List capturedOutput = new(); - List capturedError = new(); - - await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) - { - if (line.StandardError) - { - capturedError.Add(line.Content); - } - else - { - capturedOutput.Add(line.Content); - } - } + (List capturedOutput, List capturedError) = await EnumerateLines(process, useAsync); if (string.IsNullOrEmpty(standardOutput)) { @@ -174,8 +237,10 @@ public async Task ReadAllLinesAsync_ReadsBothOutputAndError(string standardOutpu Assert.True(process.WaitForExit(WaitInMS)); } - [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - public async Task ReadAllLinesAsync_ReadsInterleavedOutput() + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData(true)] + [InlineData(false)] + public async Task ReadAllLines_ReadsInterleavedOutput(bool useAsync) { const int iterations = 100; using Process process = CreateProcess(() => @@ -195,20 +260,7 @@ public async Task ReadAllLinesAsync_ReadsInterleavedOutput() process.StartInfo.RedirectStandardError = true; process.Start(); - List capturedOutput = new(); - List capturedError = new(); - - await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) - { - if (line.StandardError) - { - capturedError.Add(line.Content); - } - else - { - capturedOutput.Add(line.Content); - } - } + (List capturedOutput, List capturedError) = await EnumerateLines(process, useAsync); List expectedOutput = new(); List expectedError = new(); @@ -224,8 +276,10 @@ public async Task ReadAllLinesAsync_ReadsInterleavedOutput() Assert.True(process.WaitForExit(WaitInMS)); } - [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - public async Task ReadAllLinesAsync_ReadsLargeOutput() + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData(true)] + [InlineData(false)] + public async Task ReadAllLines_ReadsLargeOutput(bool useAsync) { const int lineCount = 1000; using Process process = CreateProcess(() => @@ -242,34 +296,62 @@ public async Task ReadAllLinesAsync_ReadsLargeOutput() process.StartInfo.RedirectStandardError = true; process.Start(); - List capturedOutput = new(); - List capturedError = new(); + (List capturedOutput, List capturedError) = await EnumerateLines(process, useAsync); - await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + Assert.Equal(lineCount, capturedOutput.Count); + for (int i = 0; i < lineCount; i++) { - if (line.StandardError) - { - capturedError.Add(line.Content); - } - else + Assert.Equal($"line{i}", capturedOutput[i]); + } + + Assert.Empty(capturedError); + Assert.True(process.WaitForExit(WaitInMS)); + } + + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData(true)] + [InlineData(false)] + public async Task ReadAllLines_ReadsVeryLongLines(bool useAsync) + { + const int lineLength = 8192; + const int lineCount = 3; + using Process process = CreateProcess(() => + { + for (int i = 0; i < lineCount; i++) { - capturedOutput.Add(line.Content); + Console.Out.WriteLine(new string((char)('A' + i), lineLength)); + Console.Out.Flush(); + Console.Error.WriteLine(new string((char)('a' + i), lineLength)); + Console.Error.Flush(); } - } + + return RemoteExecutor.SuccessExitCode; + }); + + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardError = true; + process.Start(); + + (List capturedOutput, List capturedError) = await EnumerateLines(process, useAsync); + + Assert.Equal(lineCount, capturedOutput.Count); + Assert.Equal(lineCount, capturedError.Count); for (int i = 0; i < lineCount; i++) { - Assert.Equal($"line{i}", capturedOutput[i]); + Assert.Equal(new string((char)('A' + i), lineLength), capturedOutput[i]); + Assert.Equal(new string((char)('a' + i), lineLength), capturedError[i]); } - Assert.Empty(capturedError); Assert.True(process.WaitForExit(WaitInMS)); } - [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - public async Task ReadAllLinesAsync_ThrowsOperationCanceledOnCancellation() + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData(true)] + [InlineData(false)] + public async Task ReadAllLines_ThrowsOnCancellationOrTimeout(bool useAsync) { - Process process = CreateProcess(RemotelyInvokable.ReadLine); + using Process process = CreateProcess(RemotelyInvokable.ReadLine); process.StartInfo.RedirectStandardOutput = true; process.StartInfo.RedirectStandardError = true; process.StartInfo.RedirectStandardInput = true; @@ -277,14 +359,26 @@ public async Task ReadAllLinesAsync_ThrowsOperationCanceledOnCancellation() try { - using CancellationTokenSource cts = new(TimeSpan.FromMilliseconds(100)); + if (useAsync) + { + using CancellationTokenSource cts = new(TimeSpan.FromMilliseconds(100)); - await Assert.ThrowsAnyAsync(async () => + await Assert.ThrowsAnyAsync(async () => + { + await foreach (ProcessOutputLine line in process.ReadAllLinesAsync(cts.Token)) + { + } + }); + } + else { - await foreach (ProcessOutputLine line in process.ReadAllLinesAsync(cts.Token)) + Assert.Throws(() => { - } - }); + foreach (ProcessOutputLine line in process.ReadAllLines(TimeSpan.FromMilliseconds(100))) + { + } + }); + } } finally { @@ -294,26 +388,25 @@ await Assert.ThrowsAnyAsync(async () => Assert.True(process.WaitForExit(WaitInMS)); } - [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - public async Task ReadAllLinesAsync_ProcessOutputLineProperties() + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData(true)] + [InlineData(false)] + public async Task ReadAllLines_ProcessOutputLineProperties(bool useAsync) { using Process process = StartLinePrintingProcess("stdout_line", "stderr_line"); - List allLines = new(); + (List capturedOutput, List capturedError) = await EnumerateLines(process, useAsync); - await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) - { - allLines.Add(line); - } - - Assert.Single(allLines, line => line.Content == "stdout_line" && !line.StandardError); - Assert.Single(allLines, line => line.Content == "stderr_line" && line.StandardError); + Assert.Single(capturedOutput, line => line == "stdout_line"); + Assert.Single(capturedError, line => line == "stderr_line"); Assert.True(process.WaitForExit(WaitInMS)); } - [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - public async Task ReadAllLinesAsync_StopsCleanlyWhenConsumerBreaksEarly() + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData(true)] + [InlineData(false)] + public async Task ReadAllLines_StopsCleanlyWhenConsumerBreaksEarly(bool useAsync) { using Process process = CreateProcess(() => { @@ -333,10 +426,21 @@ public async Task ReadAllLinesAsync_StopsCleanlyWhenConsumerBreaksEarly() ProcessOutputLine? firstLine = null; - await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + if (useAsync) { - firstLine = line; - break; // stop after first line + await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + { + firstLine = line; + break; + } + } + else + { + foreach (ProcessOutputLine line in process.ReadAllLines()) + { + firstLine = line; + break; + } } Assert.NotNull(firstLine); @@ -345,6 +449,187 @@ public async Task ReadAllLinesAsync_StopsCleanlyWhenConsumerBreaksEarly() Assert.True(process.WaitForExit(WaitInMS)); } + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData("utf-8", true)] + [InlineData("utf-8", false)] + [InlineData("utf-16", true)] + [InlineData("utf-16", false)] + [InlineData("utf-32", true)] + [InlineData("utf-32", false)] + public async Task ReadAllLines_WorksWithNonDefaultEncodings(string encodingName, bool useAsync) + { + Encoding encoding = Encoding.GetEncoding(encodingName); + + using Process process = CreateProcess(static (string encodingArg) => + { + Encoding enc = Encoding.GetEncoding(encodingArg); + using (StreamWriter outputWriter = new(Console.OpenStandardOutput(), enc)) + { + outputWriter.WriteLine("stdout_line"); + } + + using (StreamWriter errorWriter = new(Console.OpenStandardError(), enc)) + { + errorWriter.WriteLine("stderr_line"); + } + + return RemoteExecutor.SuccessExitCode; + }, encodingName); + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardError = true; + process.StartInfo.StandardOutputEncoding = encoding; + process.StartInfo.StandardErrorEncoding = encoding; + process.Start(); + + (List capturedOutput, List capturedError) = await EnumerateLines(process, useAsync); + + Assert.Equal(new[] { "stdout_line" }, capturedOutput); + Assert.Equal(new[] { "stderr_line" }, capturedError); + + Assert.True(process.WaitForExit(WaitInMS)); + } + + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData("utf-8", true)] + [InlineData("utf-8", false)] + [InlineData("utf-16", true)] + [InlineData("utf-16", false)] + [InlineData("utf-32", true)] + [InlineData("utf-32", false)] + public async Task ReadAllLines_WorksWithMultiByteCharacters(string encodingName, bool useAsync) + { + Encoding encoding = Encoding.GetEncoding(encodingName); + + using Process process = CreateProcess(static (string encodingArg) => + { + Encoding enc = Encoding.GetEncoding(encodingArg); + // Write raw encoded bytes split at the midpoint of the byte array so the split + // lands inside a multi-byte character, exercising decoder state across reads. + // CJK chars (U+4E16 U+754C = "世界"): 3 bytes each in UTF-8, 2 in UTF-16, 4 in UTF-32. + byte[] outBytes = enc.GetBytes("hello_\u4e16\u754c_stdout\n"); + int outSplit = outBytes.Length / 2; + Stream stdout = Console.OpenStandardOutput(); + stdout.Write(outBytes, 0, outSplit); + stdout.Flush(); + stdout.Write(outBytes, outSplit, outBytes.Length - outSplit); + stdout.Flush(); + + byte[] errBytes = enc.GetBytes("hello_\u4e16\u754c_stderr\n"); + int errSplit = errBytes.Length / 2; + Stream stderr = Console.OpenStandardError(); + stderr.Write(errBytes, 0, errSplit); + stderr.Flush(); + stderr.Write(errBytes, errSplit, errBytes.Length - errSplit); + stderr.Flush(); + + return RemoteExecutor.SuccessExitCode; + }, encodingName); + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardError = true; + process.StartInfo.StandardOutputEncoding = encoding; + process.StartInfo.StandardErrorEncoding = encoding; + process.Start(); + + (List capturedOutput, List capturedError) = await EnumerateLines(process, useAsync); + + Assert.Equal(new[] { "hello_\u4e16\u754c_stdout" }, capturedOutput); + Assert.Equal(new[] { "hello_\u4e16\u754c_stderr" }, capturedError); + + Assert.True(process.WaitForExit(WaitInMS)); + } + + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData(true)] + [InlineData(false)] + public async Task ReadAllLines_HandlesMixedLineEndings(bool useAsync) + { + using Process process = CreateProcess(static () => + { + // Write stdout with all three line-terminator styles in one stream: + // \r\n (Windows), \n (Unix), bare \r (classic Mac), and a final chunk with no terminator. + Stream stdout = Console.OpenStandardOutput(); + byte[] data = Encoding.UTF8.GetBytes("lineA\r\nlineB\nlineC\rlineD"); + stdout.Write(data); + stdout.Flush(); + return RemoteExecutor.SuccessExitCode; + }); + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardError = true; + process.Start(); + + (List capturedOutput, List capturedError) = await EnumerateLines(process, useAsync); + + Assert.Equal(new[] { "lineA", "lineB", "lineC", "lineD" }, capturedOutput); + Assert.Empty(capturedError); + Assert.True(process.WaitForExit(WaitInMS)); + } + + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData(true)] + [InlineData(false)] + public async Task ReadAllLines_HandlesPartialBomAcrossReads(bool useAsync) + { + // Write a UTF-32 LE BOM (FF FE 00 00) as two separate flushed writes so the + // first read can deliver only the first two BOM bytes. Without BOM accumulation, + // FF FE would be misclassified as a UTF-16 LE BOM and the content would be + // decoded with the wrong encoding. + using Process process = CreateProcess(static () => + { + Stream stdout = Console.OpenStandardOutput(); + stdout.Write([0xFF, 0xFE]); // First half of UTF-32 LE BOM + stdout.Flush(); + stdout.Write([0x00, 0x00]); // Second half of BOM + stdout.Write(Encoding.UTF32.GetBytes("hello\n")); // Content (no BOM from GetBytes) + stdout.Flush(); + return RemoteExecutor.SuccessExitCode; + }); + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardError = true; + process.StartInfo.StandardOutputEncoding = Encoding.UTF32; + process.Start(); + + (List capturedOutput, List capturedError) = await EnumerateLines(process, useAsync); + + Assert.Equal(new[] { "hello" }, capturedOutput); + Assert.Empty(capturedError); + Assert.True(process.WaitForExit(WaitInMS)); + } + + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [InlineData(true)] + [InlineData(false)] + public async Task ReadAllLines_LessThanFourBytes(bool useAsync) + { + using Process process = CreateProcess(static () => + { + Stream stdout = Console.OpenStandardOutput(); + stdout.Write([(byte)'h']); + stdout.Flush(); + stdout.Write([(byte)'i']); + stdout.Flush(); + + Stream error = Console.OpenStandardError(); + error.Write([(byte)'b']); + error.Flush(); + error.Write([(byte)'y']); + error.Flush(); + error.Write([(byte)'e']); + error.Flush(); + + return RemoteExecutor.SuccessExitCode; + }); + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardError = true; + process.StartInfo.StandardOutputEncoding = Encoding.UTF8; + process.Start(); + + (List capturedOutput, List capturedError) = await EnumerateLines(process, useAsync); + + Assert.Equal(new[] { "hi" }, capturedOutput); + Assert.Equal(new[] { "bye" }, capturedError); + Assert.True(process.WaitForExit(WaitInMS)); + } + private Process StartLinePrintingProcess(string stdOutText, string stdErrText) { Process process = CreateProcess((stdOut, stdErr) => @@ -368,5 +653,46 @@ private Process StartLinePrintingProcess(string stdOutText, string stdErrText) return process; } + + /// + /// Helper that wraps both the sync and async line-reading APIs and returns + /// the captured output and error lines. + /// + private static async Task<(List capturedOutput, List capturedError)> EnumerateLines(Process process, bool useAsync) + { + List capturedOutput = new(); + List capturedError = new(); + + if (useAsync) + { + await foreach (ProcessOutputLine line in process.ReadAllLinesAsync()) + { + if (line.StandardError) + { + capturedError.Add(line.Content); + } + else + { + capturedOutput.Add(line.Content); + } + } + } + else + { + foreach (ProcessOutputLine line in process.ReadAllLines()) + { + if (line.StandardError) + { + capturedError.Add(line.Content); + } + else + { + capturedOutput.Add(line.Content); + } + } + } + + return (capturedOutput, capturedError); + } } }