Skip to content
This repository was archived by the owner on Dec 18, 2018. It is now read-only.

SuperCharged MemoryPoolIterator #1138

Merged
halter73 merged 20 commits into from
Nov 22, 2016
Merged

SuperCharged MemoryPoolIterator #1138

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
4f7977d
Use PowerOfTwoToHighByte for FindFirstEqualByte
benaadams Oct 3, 2016
9eb01d1
Reduce Vector register pressure by not overlapping
benaadams Oct 5, 2016
972d978
Swap for vector loop unroll detection
benaadams Oct 15, 2016
97d4406
Use Vector<byte>.ctor workaround
benaadams Oct 15, 2016
00a6353
And=> xor, powerOfTwoToHighByte to const
benaadams Oct 16, 2016
28a21fa
Move vectors closer to use
benaadams Oct 16, 2016
8bcbfb9
Only LocateFirstFoundByte once
benaadams Nov 9, 2016
53f3611
MemoryPoolIterator fast paths
benaadams Nov 9, 2016
90c7be1
Add Request Parsing benchmark
benaadams Nov 15, 2016
12e2f30
MemoryPoolIterator byref structs
benaadams Nov 15, 2016
4cdcca2
Don't inline slow paths
benaadams Nov 16, 2016
7ab4442
Feedback + Cleanup
benaadams Nov 18, 2016
5041d6c
wasLastBlock adjustments
benaadams Nov 19, 2016
8ce47fb
Move perf, share code
benaadams Nov 19, 2016
2011a27
Use ternary op rather than lazy and
benaadams Nov 19, 2016
ef5ad3d
defaullt(MemoryPoolIterator) test coverage
benaadams Nov 19, 2016
dc90dd1
Add GetArraySegment test
benaadams Nov 19, 2016
ba0b7cc
Add 100% coverage Take tests
benaadams Nov 19, 2016
9ec4d88
Improve Skip coverage
benaadams Nov 19, 2016
2eba401
MemoryPoolIterator feedback
benaadams Nov 21, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,3 +30,5 @@ runtimes/
.build/
.testPublish/
launchSettings.json
BenchmarkDotNet.Artifacts/
BDN.Generated/
10 changes: 10 additions & 0 deletions KestrelHttpServer.sln
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,19 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "shared", "shared", "{0EF2AC
test\shared\DummyApplication.cs = test\shared\DummyApplication.cs
test\shared\HttpClientSlim.cs = test\shared\HttpClientSlim.cs
test\shared\LifetimeNotImplemented.cs = test\shared\LifetimeNotImplemented.cs
test\shared\MockConnection.cs = test\shared\MockConnection.cs
test\shared\MockFrameControl.cs = test\shared\MockFrameControl.cs
test\shared\MockSystemClock.cs = test\shared\MockSystemClock.cs
test\shared\SocketInputExtensions.cs = test\shared\SocketInputExtensions.cs
test\shared\TestApplicationErrorLogger.cs = test\shared\TestApplicationErrorLogger.cs
test\shared\TestConnection.cs = test\shared\TestConnection.cs
test\shared\TestKestrelTrace.cs = test\shared\TestKestrelTrace.cs
test\shared\TestServer.cs = test\shared\TestServer.cs
test\shared\TestServiceContext.cs = test\shared\TestServiceContext.cs
EndProjectSection
EndProject
Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Microsoft.AspNetCore.Server.Kestrel.Performance", "test\Microsoft.AspNetCore.Server.Kestrel.Performance\Microsoft.AspNetCore.Server.Kestrel.Performance.xproj", "{70567566-524C-4B67-9B59-E5C206D6C2EB}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -79,6 +84,10 @@ Global
{9559A5F1-080C-4909-B6CF-7E4B3DC55748}.Debug|Any CPU.Build.0 = Debug|Any CPU
{9559A5F1-080C-4909-B6CF-7E4B3DC55748}.Release|Any CPU.ActiveCfg = Release|Any CPU
{9559A5F1-080C-4909-B6CF-7E4B3DC55748}.Release|Any CPU.Build.0 = Release|Any CPU
{70567566-524C-4B67-9B59-E5C206D6C2EB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{70567566-524C-4B67-9B59-E5C206D6C2EB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{70567566-524C-4B67-9B59-E5C206D6C2EB}.Release|Any CPU.ActiveCfg = Release|Any CPU
{70567566-524C-4B67-9B59-E5C206D6C2EB}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand All @@ -92,5 +101,6 @@ Global
{5F64B3C3-0C2E-431A-B820-A81BBFC863DA} = {2D5D5227-4DBD-499A-96B1-76A36B03B750}
{9559A5F1-080C-4909-B6CF-7E4B3DC55748} = {D3273454-EA07-41D2-BF0B-FCC3675C2483}
{0EF2ACDF-012F-4472-A13A-4272419E2903} = {D3273454-EA07-41D2-BF0B-FCC3675C2483}
{70567566-524C-4B67-9B59-E5C206D6C2EB} = {D3273454-EA07-41D2-BF0B-FCC3675C2483}
EndGlobalSection
EndGlobal
78 changes: 39 additions & 39 deletions src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/Frame.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
using System.IO;
using System.Linq;
using System.Net;
using System.Numerics;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
Expand All @@ -24,6 +23,15 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
{
public abstract partial class Frame : IFrameControl
{
// byte types don't have a data type annotation so we pre-cast them; to avoid in-place casts
private const byte ByteCR = (byte)'\r';
private const byte ByteLF = (byte)'\n';
private const byte ByteColon = (byte)':';
private const byte ByteSpace = (byte)' ';
private const byte ByteTab = (byte)'\t';
private const byte ByteQuestionMark = (byte)'?';
private const byte BytePercentage = (byte)'%';

private static readonly ArraySegment<byte> _endChunkedResponseBytes = CreateAsciiByteArraySegment("0\r\n\r\n");
private static readonly ArraySegment<byte> _continueBytes = CreateAsciiByteArraySegment("HTTP/1.1 100 Continue\r\n\r\n");

Expand All @@ -35,14 +43,6 @@ public abstract partial class Frame : IFrameControl
private static readonly byte[] _bytesEndHeaders = Encoding.ASCII.GetBytes("\r\n\r\n");
private static readonly byte[] _bytesServer = Encoding.ASCII.GetBytes("\r\nServer: Kestrel");

private static Vector<byte> _vectorCRs = new Vector<byte>((byte)'\r');
private static Vector<byte> _vectorLFs = new Vector<byte>((byte)'\n');
private static Vector<byte> _vectorColons = new Vector<byte>((byte)':');
private static Vector<byte> _vectorSpaces = new Vector<byte>((byte)' ');
private static Vector<byte> _vectorTabs = new Vector<byte>((byte)'\t');
private static Vector<byte> _vectorQuestionMarks = new Vector<byte>((byte)'?');
private static Vector<byte> _vectorPercentages = new Vector<byte>((byte)'%');

private readonly object _onStartingSync = new Object();
private readonly object _onCompletedSync = new Object();

Expand Down Expand Up @@ -952,7 +952,7 @@ public RequestLineStatus TakeStartLine(SocketInput input)
_requestProcessingStatus = RequestProcessingStatus.RequestStarted;

int bytesScanned;
if (end.Seek(ref _vectorLFs, out bytesScanned, ServerOptions.Limits.MaxRequestLineSize) == -1)
if (end.Seek(ByteLF, out bytesScanned, ServerOptions.Limits.MaxRequestLineSize) == -1)
{
if (bytesScanned >= ServerOptions.Limits.MaxRequestLineSize)
{
Expand All @@ -969,13 +969,13 @@ public RequestLineStatus TakeStartLine(SocketInput input)
var begin = scan;
if (!begin.GetKnownMethod(out method))
{
if (scan.Seek(ref _vectorSpaces, ref end) == -1)
if (scan.Seek(ByteSpace, ref end) == -1)
{
RejectRequest(RequestRejectionReason.InvalidRequestLine,
Log.IsEnabled(LogLevel.Information) ? start.GetAsciiStringEscaped(end, MaxInvalidRequestLineChars) : string.Empty);
}

method = begin.GetAsciiString(scan);
method = begin.GetAsciiString(ref scan);

if (method == null)
{
Expand All @@ -1002,16 +1002,16 @@ public RequestLineStatus TakeStartLine(SocketInput input)
scan.Take();
begin = scan;
var needDecode = false;
var chFound = scan.Seek(ref _vectorSpaces, ref _vectorQuestionMarks, ref _vectorPercentages, ref end);
var chFound = scan.Seek(ByteSpace, ByteQuestionMark, BytePercentage, ref end);
if (chFound == -1)
{
RejectRequest(RequestRejectionReason.InvalidRequestLine,
Log.IsEnabled(LogLevel.Information) ? start.GetAsciiStringEscaped(end, MaxInvalidRequestLineChars) : string.Empty);
}
else if (chFound == '%')
else if (chFound == BytePercentage)
{
needDecode = true;
chFound = scan.Seek(ref _vectorSpaces, ref _vectorQuestionMarks, ref end);
chFound = scan.Seek(ByteSpace, ByteQuestionMark, ref end);
if (chFound == -1)
{
RejectRequest(RequestRejectionReason.InvalidRequestLine,
Expand All @@ -1023,28 +1023,28 @@ public RequestLineStatus TakeStartLine(SocketInput input)
var pathEnd = scan;

var queryString = "";
if (chFound == '?')
if (chFound == ByteQuestionMark)
{
begin = scan;
if (scan.Seek(ref _vectorSpaces, ref end) == -1)
if (scan.Seek(ByteSpace, ref end) == -1)
{
RejectRequest(RequestRejectionReason.InvalidRequestLine,
Log.IsEnabled(LogLevel.Information) ? start.GetAsciiStringEscaped(end, MaxInvalidRequestLineChars) : string.Empty);
}
queryString = begin.GetAsciiString(scan);
queryString = begin.GetAsciiString(ref scan);
}

var queryEnd = scan;

if (pathBegin.Peek() == ' ')
if (pathBegin.Peek() == ByteSpace)
{
RejectRequest(RequestRejectionReason.InvalidRequestLine,
Log.IsEnabled(LogLevel.Information) ? start.GetAsciiStringEscaped(end, MaxInvalidRequestLineChars) : string.Empty);
}

scan.Take();
begin = scan;
if (scan.Seek(ref _vectorCRs, ref end) == -1)
if (scan.Seek(ByteCR, ref end) == -1)
{
RejectRequest(RequestRejectionReason.InvalidRequestLine,
Log.IsEnabled(LogLevel.Information) ? start.GetAsciiStringEscaped(end, MaxInvalidRequestLineChars) : string.Empty);
Expand All @@ -1067,7 +1067,7 @@ public RequestLineStatus TakeStartLine(SocketInput input)
}

scan.Take(); // consume CR
if (scan.Take() != '\n')
if (scan.Take() != ByteLF)
{
RejectRequest(RequestRejectionReason.InvalidRequestLine,
Log.IsEnabled(LogLevel.Information) ? start.GetAsciiStringEscaped(end, MaxInvalidRequestLineChars) : string.Empty);
Expand All @@ -1081,16 +1081,16 @@ public RequestLineStatus TakeStartLine(SocketInput input)
if (needDecode)
{
// Read raw target before mutating memory.
rawTarget = pathBegin.GetAsciiString(queryEnd);
rawTarget = pathBegin.GetAsciiString(ref queryEnd);

// URI was encoded, unescape and then parse as utf8
pathEnd = UrlPathDecoder.Unescape(pathBegin, pathEnd);
requestUrlPath = pathBegin.GetUtf8String(pathEnd);
requestUrlPath = pathBegin.GetUtf8String(ref pathEnd);
}
else
{
// URI wasn't encoded, parse as ASCII
requestUrlPath = pathBegin.GetAsciiString(pathEnd);
requestUrlPath = pathBegin.GetAsciiString(ref pathEnd);

if (queryString.Length == 0)
{
Expand All @@ -1100,7 +1100,7 @@ public RequestLineStatus TakeStartLine(SocketInput input)
}
else
{
rawTarget = pathBegin.GetAsciiString(queryEnd);
rawTarget = pathBegin.GetAsciiString(ref queryEnd);
}
}

Expand Down Expand Up @@ -1208,7 +1208,7 @@ public bool TakeMessageHeaders(SocketInput input, FrameRequestHeaders requestHea
{
return false;
}
else if (ch == '\r')
else if (ch == ByteCR)
{
// Check for final CRLF.
end.Take();
Expand All @@ -1218,7 +1218,7 @@ public bool TakeMessageHeaders(SocketInput input, FrameRequestHeaders requestHea
{
return false;
}
else if (ch == '\n')
else if (ch == ByteLF)
{
ConnectionControl.CancelTimeout();
consumed = end;
Expand All @@ -1228,7 +1228,7 @@ public bool TakeMessageHeaders(SocketInput input, FrameRequestHeaders requestHea
// Headers don't end in CRLF line.
RejectRequest(RequestRejectionReason.HeadersCorruptedInvalidHeaderSequence);
}
else if (ch == ' ' || ch == '\t')
else if (ch == ByteSpace || ch == ByteTab)
{
RejectRequest(RequestRejectionReason.HeaderLineMustNotStartWithWhitespace);
}
Expand All @@ -1241,7 +1241,7 @@ public bool TakeMessageHeaders(SocketInput input, FrameRequestHeaders requestHea
}

int bytesScanned;
if (end.Seek(ref _vectorLFs, out bytesScanned, _remainingRequestHeadersBytesAllowed) == -1)
if (end.Seek(ByteLF, out bytesScanned, _remainingRequestHeadersBytesAllowed) == -1)
{
if (bytesScanned >= _remainingRequestHeadersBytesAllowed)
{
Expand All @@ -1254,7 +1254,7 @@ public bool TakeMessageHeaders(SocketInput input, FrameRequestHeaders requestHea
}

var beginName = scan;
if (scan.Seek(ref _vectorColons, ref end) == -1)
if (scan.Seek(ByteColon, ref end) == -1)
{
RejectRequest(RequestRejectionReason.NoColonCharacterFoundInHeaderLine);
}
Expand All @@ -1263,22 +1263,22 @@ public bool TakeMessageHeaders(SocketInput input, FrameRequestHeaders requestHea
scan.Take();

var validateName = beginName;
if (validateName.Seek(ref _vectorSpaces, ref _vectorTabs, ref endName) != -1)
if (validateName.Seek(ByteSpace, ByteTab, ref endName) != -1)
{
RejectRequest(RequestRejectionReason.WhitespaceIsNotAllowedInHeaderName);
}

var beginValue = scan;
ch = scan.Take();

while (ch == ' ' || ch == '\t')
while (ch == ByteSpace || ch == ByteTab)
{
beginValue = scan;
ch = scan.Take();
}

scan = beginValue;
if (scan.Seek(ref _vectorCRs, ref end) == -1)
if (scan.Seek(ByteCR, ref end) == -1)
{
RejectRequest(RequestRejectionReason.MissingCRInHeaderLine);
}
Expand All @@ -1287,7 +1287,7 @@ public bool TakeMessageHeaders(SocketInput input, FrameRequestHeaders requestHea
ch = scan.Take(); // expecting '\n'
end = scan;

if (ch != '\n')
if (ch != ByteLF)
{
RejectRequest(RequestRejectionReason.HeaderValueMustNotContainCR);
}
Expand All @@ -1297,7 +1297,7 @@ public bool TakeMessageHeaders(SocketInput input, FrameRequestHeaders requestHea
{
return false;
}
else if (next == ' ' || next == '\t')
else if (next == ByteSpace || next == ByteTab)
{
// From https://tools.ietf.org/html/rfc7230#section-3.2.4:
//
Expand Down Expand Up @@ -1330,18 +1330,18 @@ public bool TakeMessageHeaders(SocketInput input, FrameRequestHeaders requestHea
var endValue = scan;
do
{
ws.Seek(ref _vectorSpaces, ref _vectorTabs, ref _vectorCRs);
ws.Seek(ByteSpace, ByteTab, ByteCR);
endValue = ws;

ch = ws.Take();
while (ch == ' ' || ch == '\t')
while (ch == ByteSpace || ch == ByteTab)
{
ch = ws.Take();
}
} while (ch != '\r');
} while (ch != ByteCR);

var name = beginName.GetArraySegment(endName);
var value = beginValue.GetAsciiString(endValue);
var value = beginValue.GetAsciiString(ref endValue);

consumed = scan;
requestHeaders.Append(name.Array, name.Offset, name.Count, value);
Expand Down
8 changes: 3 additions & 5 deletions src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/MessageBody.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@

using System;
using System.IO;
using System.Numerics;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
Expand Down Expand Up @@ -390,9 +389,8 @@ protected override void OnConsumedBytes(int count)
/// </summary>
private class ForChunkedEncoding : MessageBody
{
// This causes an InvalidProgramException if made static
// https://github.com/dotnet/corefx/issues/8825
private Vector<byte> _vectorCRs = new Vector<byte>((byte)'\r');
// byte consts don't have a data type annotation so we pre-cast it
private const byte ByteCR = (byte)'\r';

private readonly SocketInput _input;
private readonly FrameRequestHeaders _requestHeaders;
Expand Down Expand Up @@ -613,7 +611,7 @@ private void ParseExtension()
// Just drain the data
do
{
if (scan.Seek(ref _vectorCRs) == -1)
if (scan.Seek(ByteCR) == -1)
{
// End marker not found yet
consumed = scan;
Expand Down
1 change: 1 addition & 0 deletions src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/Constants.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

using System.Runtime.InteropServices;
using System.Text;

namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure
{
Expand Down
Loading