From 026755478c4a6d760e2e08ef9d85fcac1abbb7e7 Mon Sep 17 00:00:00 2001 From: Cory Nelson Date: Thu, 30 Dec 2021 14:56:30 -0800 Subject: [PATCH] Improve header parser performance. --- .../src/System.Net.Http.csproj | 4 +- .../ChunkedEncodingReadStream.cs | 57 ++- .../Http/SocketsHttpHandler/HttpConnection.cs | 390 ++++++++++++++---- .../SocketsHttpHandler/SimdPaddedArray.cs | 61 +++ 4 files changed, 407 insertions(+), 105 deletions(-) create mode 100644 src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/SimdPaddedArray.cs diff --git a/src/libraries/System.Net.Http/src/System.Net.Http.csproj b/src/libraries/System.Net.Http/src/System.Net.Http.csproj index 8b5076b5ca9e37..61778c733914a8 100644 --- a/src/libraries/System.Net.Http/src/System.Net.Http.csproj +++ b/src/libraries/System.Net.Http/src/System.Net.Http.csproj @@ -1,4 +1,4 @@ - + win true @@ -80,6 +80,7 @@ + @@ -668,6 +669,7 @@ + diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/ChunkedEncodingReadStream.cs b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/ChunkedEncodingReadStream.cs index e4ecc124a1e469..297f87c8b0e586 100644 --- a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/ChunkedEncodingReadStream.cs +++ b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/ChunkedEncodingReadStream.cs @@ -21,18 +21,21 @@ private sealed class ChunkedEncodingReadStream : HttpContentReadStream /// infinite chunk length is sent. This value is arbitrary and can be changed as needed. /// private const int MaxChunkBytesAllowed = 16 * 1024; - /// How long a trailing header can be. This value is arbitrary and can be changed as needed. - private const int MaxTrailingHeaderLength = 16 * 1024; /// The number of bytes remaining in the chunk. private ulong _chunkBytesRemaining; /// The current state of the parsing state machine for the chunked response. private ParsingState _state = ParsingState.ExpectChunkHeader; private readonly HttpResponseMessage _response; + private readonly int _trailingHeaderBytesAllowed; public ChunkedEncodingReadStream(HttpConnection connection, HttpResponseMessage response) : base(connection) { Debug.Assert(response != null, "The HttpResponseMessage cannot be null."); _response = response; + + // _allowedReadLineBytes is reset when reading chunk indicators. + // save the original here, which will be the total header bytes minus already received headers. + _trailingHeaderBytesAllowed = connection._allowedReadLineBytes; } public override int Read(Span buffer) @@ -361,6 +364,7 @@ private int ReadChunksFromConnectionBuffer(Span buffer, CancellationTokenR } else { + _connection._allowedReadLineBytes = _trailingHeaderBytesAllowed; _state = ParsingState.ConsumeTrailers; goto case ParsingState.ConsumeTrailers; } @@ -406,39 +410,24 @@ private int ReadChunksFromConnectionBuffer(Span buffer, CancellationTokenR case ParsingState.ConsumeTrailers: Debug.Assert(_chunkBytesRemaining == 0, $"Expected {nameof(_chunkBytesRemaining)} == 0, got {_chunkBytesRemaining}"); - while (true) + // Consume receive buffer. If the stream is disposed, pass a null response to avoid + // processing headers for a connection returned to the pool. + + if (_connection!.ParseHeaders(IsDisposed ? null : _response, isFromTrailer: true)) { - _connection._allowedReadLineBytes = MaxTrailingHeaderLength; - if (!_connection.TryReadNextLine(out currentLine)) - { - break; - } - - if (currentLine.IsEmpty) - { - // Dispose of the registration and then check whether cancellation has been - // requested. This is necessary to make determinstic a race condition between - // cancellation being requested and unregistering from the token. Otherwise, - // it's possible cancellation could be requested just before we unregister and - // we then return a connection to the pool that has been or will be disposed - // (e.g. if a timer is used and has already queued its callback but the - // callback hasn't yet run). - cancellationRegistration.Dispose(); - CancellationHelper.ThrowIfCancellationRequested(cancellationRegistration.Token); - - _state = ParsingState.Done; - _connection.CompleteResponse(); - _connection = null; - - break; - } - // Parse the trailer. - else if (!IsDisposed) - { - // Make sure that we don't inadvertently consume trailing headers - // while draining a connection that's being returned back to the pool. - HttpConnection.ParseHeaderNameValue(_connection, currentLine, _response, isFromTrailer: true); - } + // Dispose of the registration and then check whether cancellation has been + // requested. This is necessary to make determinstic a race condition between + // cancellation being requested and unregistering from the token. Otherwise, + // it's possible cancellation could be requested just before we unregister and + // we then return a connection to the pool that has been or will be disposed + // (e.g. if a timer is used and has already queued its callback but the + // callback hasn't yet run). + cancellationRegistration.Dispose(); + CancellationHelper.ThrowIfCancellationRequested(cancellationRegistration.Token); + + _state = ParsingState.Done; + _connection.CompleteResponse(); + _connection = null; } return default; diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnection.cs b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnection.cs index 12b6b4bf3c55a0..515e2a5ed43ee5 100644 --- a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnection.cs +++ b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnection.cs @@ -10,7 +10,10 @@ using System.Net.Http.Headers; using System.Net.Security; using System.Net.Sockets; +using System.Numerics; using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -57,7 +60,7 @@ internal sealed partial class HttpConnection : HttpConnectionBase private ValueTask? _readAheadTask; private int _readAheadTaskLock; // 0 == free, 1 == held - private byte[] _readBuffer; + private SimdPaddedArray _readBuffer; private int _readOffset; private int _readLength; @@ -88,7 +91,7 @@ public HttpConnection( _transportContext = transportContext; _writeBuffer = new byte[InitialWriteBufferSize]; - _readBuffer = new byte[InitialReadBufferSize]; + _readBuffer = new SimdPaddedArray(InitialReadBufferSize); _weakThisRef = new WeakReference(this); @@ -179,7 +182,7 @@ public bool PrepareForReuse(bool async) try { #pragma warning disable CA2012 // we're very careful to ensure the ValueTask is only consumed once, even though it's stored into a field - _readAheadTask = _stream.ReadAsync(new Memory(_readBuffer)); + _readAheadTask = _stream.ReadAsync(_readBuffer.AsMemory()); #pragma warning restore CA2012 return !_readAheadTask.Value.IsCompleted; } @@ -219,7 +222,7 @@ async ValueTask ReadAheadWithZeroByteReadAsync() await _stream.ReadAsync(Memory.Empty).ConfigureAwait(false); // We don't know for sure that the stream actually has data available, so we need to issue a real read now. - return await _stream.ReadAsync(new Memory(_readBuffer)).ConfigureAwait(false); + return await _stream.ReadAsync(_readBuffer.AsMemory()).ConfigureAwait(false); } } @@ -246,7 +249,7 @@ async ValueTask ReadAheadWithZeroByteReadAsync() private int ReadBufferSize => _readBuffer.Length; - private ReadOnlyMemory RemainingBuffer => new ReadOnlyMemory(_readBuffer, _readOffset, _readLength - _readOffset); + private ReadOnlyMemory RemainingBuffer => _readBuffer.AsMemory(_readOffset, _readLength - _readOffset); private void ConsumeFromRemainingBuffer(int bytesToConsume) { @@ -637,14 +640,9 @@ public async Task SendAsyncCore(HttpRequestMessage request, } // Parse the response headers. Logic after this point depends on being able to examine headers in the response object. - while (true) + while (!ParseHeaders(response, isFromTrailer: false)) { - ReadOnlyMemory line = await ReadNextResponseHeaderLineAsync(async, foldedHeadersAllowed: true).ConfigureAwait(false); - if (IsLineEmpty(line)) - { - break; - } - ParseHeaderNameValue(this, line.Span, response, isFromTrailer: false); + await FillAsync(async).ConfigureAwait(false); } if (HttpTelemetry.Log.IsEnabled()) HttpTelemetry.Log.ResponseHeadersStop(); @@ -1027,69 +1025,321 @@ private static void ParseStatusLine(ReadOnlySpan line, HttpResponseMessage } } - private static void ParseHeaderNameValue(HttpConnection connection, ReadOnlySpan line, HttpResponseMessage response, bool isFromTrailer) + internal bool ParseHeaders(HttpResponseMessage? response, bool isFromTrailer) { - Debug.Assert(line.Length > 0); + Span buffer = _readBuffer.AsSpan(_readOffset, _readLength - _readOffset); - int pos = 0; - while (line[pos] != (byte)':' && line[pos] != (byte)' ') + (bool finished, int bytesConsumed) = + Avx2.IsSupported ? ParseHeadersAvx2(buffer, response, isFromTrailer) : + ParseHeadersPortable(buffer, response, isFromTrailer); + + _readOffset += bytesConsumed; + _allowedReadLineBytes -= bytesConsumed; + ThrowIfExceededAllowedReadLineBytes(); + + return finished; + } + + private (bool finished, int bytesConsumed) ParseHeadersPortable(Span buffer, HttpResponseMessage? response, bool isFromTrailer) + { + int originalBufferLength = buffer.Length; + + while (true) { - pos++; - if (pos == line.Length) + if (buffer.Length == 0) goto needMore; + + int colIdx = buffer.IndexOfAny((byte)':', (byte)'\n'); + if (colIdx < 0) goto needMore; + + if (buffer[colIdx] == '\n') { - // Invalid header line that doesn't contain ':'. - throw new HttpRequestException(SR.Format(SR.net_http_invalid_response_header_line, Encoding.ASCII.GetString(line))); + if ((colIdx == 1 && buffer[0] == '\r') || colIdx == 0) + { + return (finished: true, bytesConsumed: originalBufferLength - buffer.Length + colIdx + 1); + } + + buffer = buffer.Slice(0, colIdx); + throw new HttpRequestException(SR.Format(SR.net_http_invalid_response_header_line, Encoding.ASCII.GetString(buffer))); + } + + int lfIdx = colIdx + 1; + if (lfIdx >= buffer.Length) goto needMore; + + Span valueStart = buffer.Slice(lfIdx); + Span valueIter = valueStart; + + while (true) + { + lfIdx = valueIter.IndexOf((byte)'\n'); + if (lfIdx < 0) goto needMore; + + int crOrLfIdx = lfIdx - 1; + if (crOrLfIdx < 0 || valueIter[crOrLfIdx] != '\r') + { + crOrLfIdx = lfIdx; + } + + int spIdx = lfIdx + 1; + if (spIdx >= valueIter.Length) goto needMore; + + if (valueIter[spIdx] is not (byte)'\t' and not (byte)' ') + { + // Found the end of the header value. + + if (response is not null) + { + ReadOnlySpan headerName = buffer.Slice(0, colIdx); + ReadOnlySpan headerValue = valueStart.Slice(0, valueStart.Length - valueIter.Length + crOrLfIdx); + AddResponseHeader(headerName, headerValue, response, isFromTrailer); + } + + buffer = valueStart.Slice(valueStart.Length - valueIter.Length + spIdx); + break; + } + + // Found an obs-fold (CRLFHT/CRLFSP). + // Replace with SPSPSP and keep looking for the final newline. + // Also handles LFHT and LFSP. + + valueIter[crOrLfIdx] = (byte)' '; + valueIter[lfIdx] = (byte)' '; + valueIter[spIdx] = (byte)' '; + + valueIter = valueIter.Slice(spIdx + 1); } } - if (pos == 0) + needMore: + return (finished: false, bytesConsumed: originalBufferLength - buffer.Length); + } + + /// + /// This method REQUIRES (32-1) bytes of valid address space after . + /// This is done via . + /// It also assumes that it can always step backwards to a 32-byte aligned address. + /// + private unsafe (bool finished, int bytesConsumed) ParseHeadersAvx2(Span buffer, HttpResponseMessage? response, bool isFromTrailer) + { + Vector256 maskCol = Vector256.Create((byte)':'); + Vector256 maskLF = Vector256.Create((byte)'\n'); + + fixed (byte* vectorBegin = buffer) { - // Invalid empty header name. - throw new HttpRequestException(SR.Format(SR.net_http_invalid_response_header_name, "")); - } + byte* vectorEnd = vectorBegin + buffer.Length; + byte* nameStart = vectorBegin; + + // The following goto, needMore, afterNeedMore are all here + // to reduce code size on the hot path as much as possible. + + // Various points need to jump to needMore, so it must be in this + // root scope. The if here is inverted and manually implemented to allow this. + + if (vectorBegin != vectorEnd) + { + goto afterNeedMore; + } + + needMore: + return (finished: false, bytesConsumed: (int)(nameStart - vectorBegin)); + + afterNeedMore: + // align to a 32 byte address for optimal loads. + byte* vectorIter = (byte*)((nint)vectorBegin & ~(32 - 1)); + + Vector256 vector = Avx.LoadAlignedVector256(vectorIter); + uint foundLF = (uint)Avx2.MoveMask(Avx2.CompareEqual(vector, maskLF)); + uint foundCol = foundLF | (uint)Avx2.MoveMask(Avx2.CompareEqual(vector, maskCol)); + + while (true) + { + // Find the end of the header name, a colon. + // Or, a CRLF or LF indicating end of headers. + + byte* col; + do + { + int colIdx; + while ((colIdx = BitOperations.TrailingZeroCount(foundCol)) == Vector256.Count) + { + vectorIter += Vector256.Count; + if (vectorIter >= vectorEnd) + { + goto needMore; + } + + // vectorIter may be past the end of buffer[^1]. This is why padding is required. + + vector = Avx.LoadAlignedVector256(vectorIter); + foundLF = (uint)Avx2.MoveMask(Avx2.CompareEqual(vector, maskLF)); + foundCol = foundLF | (uint)Avx2.MoveMask(Avx2.CompareEqual(vector, maskCol)); + } - if (!HeaderDescriptor.TryGet(line.Slice(0, pos), out HeaderDescriptor descriptor)) + foundCol ^= 1u << colIdx; + col = vectorIter + colIdx; + } + // Branch will not be taken unless padding bytes prior to vectorBegin match. + while (col < nameStart); + + if (col >= vectorEnd) + { + // Branch will not be taken unless padding bytes beyond vectorEnd match. + goto needMore; + } + + if (*col == '\n') + { + if ((col == nameStart + 1 && *nameStart == '\r') || col == nameStart) + { + // End of headers. + return (finished: true, bytesConsumed: (int)(col - vectorBegin + 1)); + } + else + { + // Found a newline without a colon. + buffer = new Span(nameStart, (int)(col - nameStart)); + throw new HttpRequestException(SR.Format(SR.net_http_invalid_response_header_line, Encoding.ASCII.GetString(buffer))); + } + } + + // Found the start of header value. + byte* valueStart = col + 1; + if (valueStart == vectorEnd) + { + goto needMore; + } + + // Find the end of the value, a CRLF or LF. + byte* crOrLf; + byte* lf; + while (true) + { + do + { + int lfIdx; + while ((lfIdx = BitOperations.TrailingZeroCount(foundLF)) == Vector256.Count) + { + vectorIter += Vector256.Count; + if (vectorIter >= vectorEnd) + { + goto needMore; + } + + vector = Avx.LoadAlignedVector256(vectorIter); + foundLF = (uint)Avx2.MoveMask(Avx2.CompareEqual(vector, maskLF)); + foundCol = foundLF | (uint)Avx2.MoveMask(Avx2.CompareEqual(vector, maskCol)); + } + + uint clearMask = ~(1u << lfIdx); + foundLF &= clearMask; + foundCol &= clearMask; + lf = vectorIter + lfIdx; + } + // Branch will not be taken unless padding bytes prior to vectorBegin match. + while (lf < col); + + if (lf >= vectorEnd) + { + // Branch will not be taken unless padding bytes beyond vectorEnd match. + goto needMore; + } + + crOrLf = lf - 1; + if (crOrLf < valueStart || *crOrLf != '\r') + { + // Cold path: lone LF without a CR. + crOrLf = lf; + } + + byte* ht = lf + 1; + if (ht == vectorEnd) + { + goto needMore; + } + + if (*ht is not (byte)' ' and not (byte)'\t') + { + // Found the end of the header value. + + if (response is not null) + { + // Hot path: response will only be null when draining before headers are observed. + Span name = new Span(nameStart, (int)(col - nameStart)); + Span value = new Span(valueStart, (int)(crOrLf - valueStart)); + AddResponseHeader(name, value, response, isFromTrailer); + } + + nameStart = ht; + break; + } + + // Found an obs-fold: replace all bytes with SP. + // Cold path: obs-fold is generally unused. + *crOrLf = (byte)' '; + *lf = (byte)' '; + *ht = (byte)' '; + } // Loop: continue searching for CRLF. + } // Loop: found a header, now search for the next name. + } // Pin + } + + private void AddResponseHeader(ReadOnlySpan name, ReadOnlySpan value, HttpResponseMessage response, bool isFromTrailer) + { + // Skip trailing whitespace and check for null length. + while (true) { - // Invalid header name. - throw new HttpRequestException(SR.Format(SR.net_http_invalid_response_header_name, Encoding.ASCII.GetString(line.Slice(0, pos)))); + int spIdx = name.Length - 1; + + if (spIdx < 0) + { + throw new HttpRequestException(SR.Format(SR.net_http_invalid_response_header_name, "")); + } + + if (name[spIdx] != ' ') + { + // hot path. + break; + } + + name = name.Slice(0, spIdx); } - if (isFromTrailer && descriptor.KnownHeader != null && (descriptor.KnownHeader.HeaderType & HttpHeaderType.NonTrailing) == HttpHeaderType.NonTrailing) + // Skip leading OWS for value. + // hot path: loop body runs only once. + while (value.Length != 0 && value[0] is (byte)' ' or (byte)'\t') { - // Disallowed trailer fields. - // A recipient MUST ignore fields that are forbidden to be sent in a trailer. - if (NetEventSource.Log.IsEnabled()) connection.Trace($"Stripping forbidden {descriptor.Name} from trailer headers."); - return; + value = value.Slice(1); } - // Eat any trailing whitespace - while (line[pos] == (byte)' ') + // Skip trailing whitespace for value. + while (true) { - pos++; - if (pos == line.Length) + int spIdx = value.Length - 1; + if (spIdx < 0 || value[spIdx] != ' ') { - // Invalid header line that doesn't contain ':'. - throw new HttpRequestException(SR.Format(SR.net_http_invalid_response_header_line, Encoding.ASCII.GetString(line))); + // hot path. + break; } + + value = value.Slice(0, spIdx); } - if (line[pos++] != ':') + if (!HeaderDescriptor.TryGet(name, out HeaderDescriptor descriptor)) { - // Invalid header line that doesn't contain ':'. - throw new HttpRequestException(SR.Format(SR.net_http_invalid_response_header_line, Encoding.ASCII.GetString(line))); + // Invalid header name. + throw new HttpRequestException(SR.Format(SR.net_http_invalid_response_header_name, Encoding.ASCII.GetString(name))); } - // Skip whitespace after colon - while (pos < line.Length && (line[pos] == (byte)' ' || line[pos] == (byte)'\t')) + if (isFromTrailer && descriptor.KnownHeader != null && (descriptor.KnownHeader.HeaderType & HttpHeaderType.NonTrailing) == HttpHeaderType.NonTrailing) { - pos++; + // Disallowed trailer fields. + // A recipient MUST ignore fields that are forbidden to be sent in a trailer. + if (NetEventSource.Log.IsEnabled()) Trace($"Stripping forbidden {descriptor.Name} from trailer headers."); + return; } - Debug.Assert(response.RequestMessage != null); - Encoding? valueEncoding = connection._pool.Settings._responseHeaderEncodingSelector?.Invoke(descriptor.Name, response.RequestMessage); + Encoding? valueEncoding = _pool.Settings._responseHeaderEncodingSelector?.Invoke(descriptor.Name, response.RequestMessage!); // Note we ignore the return value from TryAddWithoutValidation. If the header can't be added, we silently drop it. - ReadOnlySpan value = line.Slice(pos); if (isFromTrailer) { string headerValue = descriptor.GetHeaderValue(value, valueEncoding); @@ -1103,7 +1353,7 @@ private static void ParseHeaderNameValue(HttpConnection connection, ReadOnlySpan else { // Request headers returned on the response must be treated as custom headers. - string headerValue = connection.GetResponseHeaderValueWithCaching(descriptor, value, valueEncoding); + string headerValue = GetResponseHeaderValueWithCaching(descriptor, value, valueEncoding); response.Headers.TryAddWithoutValidation( (descriptor.HeaderType & HttpHeaderType.Request) == HttpHeaderType.Request ? descriptor.AsCustomHeader() : descriptor, headerValue); @@ -1452,7 +1702,7 @@ private ValueTask WriteToStreamAsync(ReadOnlyMemory source, bool async) private bool TryReadNextLine(out ReadOnlySpan line) { - var buffer = new ReadOnlySpan(_readBuffer, _readOffset, _readLength - _readOffset); + var buffer = _readBuffer.AsSpan(_readOffset, _readLength - _readOffset); int length = buffer.IndexOf((byte)'\n'); if (length < 0) { @@ -1480,7 +1730,7 @@ private async ValueTask> ReadNextResponseHeaderLineAsync(bo while (true) { int scanOffset = _readOffset + previouslyScannedBytes; - int lfIndex = Array.IndexOf(_readBuffer, (byte)'\n', scanOffset, _readLength - scanOffset); + int lfIndex = Array.IndexOf(_readBuffer.BackingArray, (byte)'\n', scanOffset, _readLength - scanOffset); if (lfIndex >= 0) { int startIndex = _readOffset; @@ -1524,7 +1774,7 @@ private async ValueTask> ReadNextResponseHeaderLineAsync(bo // Folded headers are only allowed within header field values, not within header field names, // so if we haven't seen a colon, this is invalid. - if (Array.IndexOf(_readBuffer, (byte)':', _readOffset, lfIndex - _readOffset) == -1) + if (Array.IndexOf(_readBuffer.BackingArray, (byte)':', _readOffset, lfIndex - _readOffset) == -1) { throw new HttpRequestException(SR.net_http_invalid_response_header_folder); } @@ -1554,7 +1804,7 @@ private async ValueTask> ReadNextResponseHeaderLineAsync(bo ThrowIfExceededAllowedReadLineBytes(); _readOffset = lfIndex + 1; - return new ReadOnlyMemory(_readBuffer, startIndex, length); + return _readBuffer.AsMemory(startIndex, length); } // Couldn't find LF. Read more. Note this may cause _readOffset to change. @@ -1587,8 +1837,8 @@ private async ValueTask InitialFillAsync(bool async) _readOffset = 0; _readLength = async ? - await _stream.ReadAsync(_readBuffer).ConfigureAwait(false) : - _stream.Read(_readBuffer); + await _stream.ReadAsync(_readBuffer.AsMemory()).ConfigureAwait(false) : + _stream.Read(_readBuffer.AsSpan()); if (NetEventSource.Log.IsEnabled()) Trace($"Received {_readLength} bytes."); } @@ -1611,27 +1861,27 @@ private async ValueTask FillAsync(bool async) { // There's some data in the buffer but it's not at the beginning. Shift it // down to make room for more. - Buffer.BlockCopy(_readBuffer, _readOffset, _readBuffer, 0, remaining); + Buffer.BlockCopy(_readBuffer.BackingArray, _readOffset, _readBuffer.BackingArray, 0, remaining); _readOffset = 0; _readLength = remaining; } - else if (remaining == _readBuffer.Length) + else if (remaining == ReadBufferSize) { // The whole buffer is full, but the caller is still requesting more data, // so increase the size of the buffer. Debug.Assert(_readOffset == 0); - Debug.Assert(_readLength == _readBuffer.Length); + Debug.Assert(_readLength == ReadBufferSize); - var newReadBuffer = new byte[_readBuffer.Length * 2]; - Buffer.BlockCopy(_readBuffer, 0, newReadBuffer, 0, remaining); + var newReadBuffer = new SimdPaddedArray(ReadBufferSize * 2); + Buffer.BlockCopy(_readBuffer.BackingArray, 0, newReadBuffer.BackingArray, 0, remaining); _readBuffer = newReadBuffer; _readOffset = 0; _readLength = remaining; } int bytesRead = async ? - await _stream.ReadAsync(new Memory(_readBuffer, _readLength, _readBuffer.Length - _readLength)).ConfigureAwait(false) : - _stream.Read(_readBuffer, _readLength, _readBuffer.Length - _readLength); + await _stream.ReadAsync(_readBuffer.AsMemory(_readLength, ReadBufferSize - _readLength)).ConfigureAwait(false) : + _stream.Read(_readBuffer.AsSpan(_readLength, ReadBufferSize - _readLength)); if (NetEventSource.Log.IsEnabled()) Trace($"Received {bytesRead} bytes."); if (bytesRead == 0) @@ -1646,7 +1896,7 @@ private void ReadFromBuffer(Span buffer) { Debug.Assert(buffer.Length <= _readLength - _readOffset); - new Span(_readBuffer, _readOffset, buffer.Length).CopyTo(buffer); + _readBuffer.AsSpan(_readOffset, buffer.Length).CopyTo(buffer); _readOffset += buffer.Length; } @@ -1730,7 +1980,7 @@ private int ReadBuffered(Span destination) // Do a buffered read directly against the underlying stream. Debug.Assert(_readAheadTask == null, "Read ahead task should have been consumed as part of the headers."); - int bytesRead = _stream.Read(_readBuffer, 0, destination.Length == 0 ? 0 : _readBuffer.Length); + int bytesRead = _stream.Read(_readBuffer.AsSpan(0, destination.Length == 0 ? 0 : _readBuffer.Length)); if (NetEventSource.Log.IsEnabled()) Trace($"Received {bytesRead} bytes."); _readLength = bytesRead; @@ -1800,11 +2050,11 @@ private ValueTask CopyFromBufferAsync(Stream destination, bool async, int count, if (async) { - return destination.WriteAsync(new ReadOnlyMemory(_readBuffer, offset, count), cancellationToken); + return destination.WriteAsync(_readBuffer.AsMemory(offset, count), cancellationToken); } else { - destination.Write(_readBuffer, offset, count); + destination.Write(_readBuffer.BackingArray, offset, count); return default; } } @@ -1879,7 +2129,7 @@ private async Task CopyToContentLengthAsync(Stream destination, bool async, ulon // use a temporary buffer from the ArrayPool so that the connection doesn't hog large // buffers from the pool for extended durations, especially if it's going to sit in the // connection pool for a prolonged period. - byte[]? origReadBuffer = null; + SimdPaddedArray? origReadBuffer = null; try { while (true) @@ -1901,14 +2151,14 @@ private async Task CopyToContentLengthAsync(Stream destination, bool async, ulon // larger than the one we already have, then grow the connection's read buffer to that size. if (origReadBuffer == null) { - byte[] currentReadBuffer = _readBuffer; + SimdPaddedArray currentReadBuffer = _readBuffer; if (remaining == currentReadBuffer.Length) { int desiredBufferSize = (int)Math.Min((ulong)bufferSize, length); if (desiredBufferSize > currentReadBuffer.Length) { origReadBuffer = currentReadBuffer; - _readBuffer = ArrayPool.Shared.Rent(desiredBufferSize); + _readBuffer = SimdPaddedArray.FromPool(desiredBufferSize); } } } @@ -1918,8 +2168,8 @@ private async Task CopyToContentLengthAsync(Stream destination, bool async, ulon { if (origReadBuffer != null) { - byte[] tmp = _readBuffer; - _readBuffer = origReadBuffer; + byte[] tmp = _readBuffer.BackingArray; + _readBuffer = origReadBuffer.GetValueOrDefault(); ArrayPool.Shared.Return(tmp); // _readOffset and _readLength may not be within range of the original diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/SimdPaddedArray.cs b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/SimdPaddedArray.cs new file mode 100644 index 00000000000000..2123a3f3e790c5 --- /dev/null +++ b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/SimdPaddedArray.cs @@ -0,0 +1,61 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Buffers; +using System.Diagnostics; +using System.Runtime.Intrinsics.X86; + +namespace System.Net.Http +{ + /// + /// An array that ensures valid address space before + /// and after the array's bytes, to support SIMD loads. + /// + internal readonly struct SimdPaddedArray + { + private static int PaddingLength => + Avx2.IsSupported ? 31 : + 0; + + private readonly byte[] _array; + + public int Length => _array.Length - PaddingLength; + public byte[] BackingArray => _array; + + + public SimdPaddedArray(int length) + : this(new byte[length + PaddingLength]) + { + } + + private SimdPaddedArray(byte[] array) + { + _array = array; + } + + public static SimdPaddedArray FromPool(int length) => new SimdPaddedArray(ArrayPool.Shared.Rent(length)); + + public Span AsSpan() => AsSpan(0, Length); + public Span AsSpan(int offset, int length) + { + Debug.Assert(length - offset <= Length); + return new Span(_array, offset, length); + } + + public Memory AsMemory() => AsMemory(0, Length); + public Memory AsMemory(int offset, int length) + { + Debug.Assert(length - offset <= Length); + return new Memory(_array, offset, length); + } + + public ref byte this[int index] + { + get + { + Debug.Assert(index < Length); + return ref _array[index]; + } + } + } +}