Skip to content
This repository was archived by the owner on Dec 18, 2018. It is now read-only.

Use Ben's Magic Number for FindFirstEqualByte #1136

Closed
benaadams wants to merge 2 commits into from
Closed

Use Ben's Magic Number for FindFirstEqualByte #1136

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4924cf5
Use De Bruijn sequence for FindFirstEqualByte
benaadams Oct 1, 2016
311309c
Use BensMagicNumber to find byte
benaadams Oct 3, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 73 additions & 23 deletions src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,29 @@
using System;
using System.Diagnostics;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Threading;

namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure
{
public struct MemoryPoolIterator
{
// When this variable is set in KestrelEngine.JitReadonlyConsts it will
// deterministically convert the following readonly static values to jitted consts
// and embed the init code in that start up called function.
//
// This means when they are next used the have been pre-evaluated and the jit can
// either directly embed them or use them for branch elimiation.
internal static bool StaticReadonlysJitted;

// Ben Adam's Magic Number for finding set bytes
private const ulong BENS_MAGIC_NUMBER = 0x81018202830380;

// Convert these returns to jitted consts
Copy link
Copy Markdown

@jamesqo jamesqo Oct 2, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe use using static here instead?

Copy link
Copy Markdown
Contributor Author

@benaadams benaadams Oct 2, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure what that would get? Each one is from a different type and I'm trying to force their evaluation rather than specifically shorten their names.

Copy link
Copy Markdown
Contributor Author

@benaadams benaadams Oct 2, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Conversations over in #1129 (comment) you might be interested in

private static readonly bool IsHardwareAccelerated = Vector.IsHardwareAccelerated;
private static readonly bool IsLittleEndian = BitConverter.IsLittleEndian;
private static readonly int _vectorSpan = Vector<byte>.Count;
private static readonly int _vectorUlongSpan = Vector<ulong>.Count;

private MemoryPoolBlock _block;
private int _index;
Expand Down Expand Up @@ -122,7 +138,9 @@ public void Skip(int bytesToSkip)
{
if (wasLastBlock)
{
throw new InvalidOperationException("Attempted to skip more bytes than available.");
// Removed throw code to reduce inline size
// https://github.com/dotnet/coreclr/pull/6103
ThrowInvalidOperationException_SkippedMoreThanAvailable();
}
else
{
Expand Down Expand Up @@ -262,7 +280,7 @@ public unsafe int Seek(
// Need unit tests to test Vector path
#if !DEBUG
// Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079
if (Vector.IsHardwareAccelerated)
if (IsHardwareAccelerated)
{
#endif
if (following >= _vectorSpan)
Expand Down Expand Up @@ -308,7 +326,6 @@ public unsafe int Seek(
#if !DEBUG
}
#endif

var pCurrent = (block.DataFixedPtr + index);
var pEnd = pCurrent + Math.Min(following, limit - bytesScanned);
do
Expand Down Expand Up @@ -370,7 +387,7 @@ public unsafe int Seek(
// Need unit tests to test Vector path
#if !DEBUG
// Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079
if (Vector.IsHardwareAccelerated)
if (IsHardwareAccelerated)
{
#endif
if (following >= _vectorSpan)
Expand Down Expand Up @@ -483,7 +500,7 @@ public unsafe int Seek(
// Need unit tests to test Vector path
#if !DEBUG
// Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079
if (Vector.IsHardwareAccelerated)
if (IsHardwareAccelerated)
{
#endif
if (following >= _vectorSpan)
Expand Down Expand Up @@ -626,7 +643,7 @@ public unsafe int Seek(
// Need unit tests to test Vector path
#if !DEBUG
// Check will be Jitted away https://github.com/dotnet/coreclr/issues/1079
if (Vector.IsHardwareAccelerated)
if (IsHardwareAccelerated)
{
#endif
if (following >= _vectorSpan)
Expand Down Expand Up @@ -748,27 +765,43 @@ public unsafe int Seek(
/// <param name="byteEquals"></param >
/// <returns>The first index of the result vector</returns>
/// <exception cref="InvalidOperationException">byteEquals = 0</exception>
// Force inlining (70 IL bytes, 98 bytes asm w/ inlines) Issue: https://github.com/dotnet/coreclr/issues/7386
//[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static int FindFirstEqualByte(ref Vector<byte> byteEquals)
{
if (!BitConverter.IsLittleEndian) return FindFirstEqualByteSlow(ref byteEquals);

// Quasi-tree search
var vector64 = Vector.AsVectorInt64(byteEquals);
for (var i = 0; i < Vector<long>.Count; i++)
// Jitted const eliminates branch
if (IsLittleEndian)
{
var longValue = vector64[i];
if (longValue == 0) continue;
var vector64 = Vector.AsVectorUInt64(byteEquals);
ulong longValue = 0;
var i = 0;
// Should only be called when byte in Vector. Since range check
// can't be elminated, make loop one larger to throw if not found
// rather than doing the throw ourselves
for (; i < _vectorUlongSpan + 1; i++)
{
longValue = vector64[i];
if (longValue != 0)
{
break;
}
}

return (i << 3) +
((longValue & 0x00000000ffffffff) > 0
? (longValue & 0x000000000000ffff) > 0
? (longValue & 0x00000000000000ff) > 0 ? 0 : 1
: (longValue & 0x0000000000ff0000) > 0 ? 2 : 3
: (longValue & 0x0000ffff00000000) > 0
? (longValue & 0x000000ff00000000) > 0 ? 4 : 5
: (longValue & 0x00ff000000000000) > 0 ? 6 : 7);
// Single LEA instruction with jitted const (using function result)
return i * 8 + BensMagicNumberFindByte(longValue);
}
throw new InvalidOperationException();
else
{
return FindFirstEqualByteSlow(ref byteEquals);
}
}

// Force inlining (23 IL bytes)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int BensMagicNumberFindByte(ulong ulongValue)
{
var flag = (ulongValue & ((ulong)-(long)ulongValue)) >> 8;
return (int) ((flag * BENS_MAGIC_NUMBER) >> 55) & 7;
}

// Internal for testing
Expand Down Expand Up @@ -856,7 +889,9 @@ public int GetLength(MemoryPoolIterator end)
}
else if (block.Next == null)
{
throw new InvalidOperationException("end did not follow iterator");
// Removed throw code to reduce inline size
// https://github.com/dotnet/coreclr/pull/6103
ThrowInvalidOperationException_EndDidNotFollow();
}
else
{
Expand Down Expand Up @@ -1041,5 +1076,20 @@ public unsafe void CopyFromAscii(string data)
_block = block;
_index = blockIndex;
}

private static void ThrowInvalidOperationException_EndDidNotFollow()
{
// Removed throw code to reduce inline size
// https://github.com/dotnet/coreclr/pull/6103
throw new InvalidOperationException("end did not follow iterator");
}

private static void ThrowInvalidOperationException_SkippedMoreThanAvailable()
{
// Removed throw code to reduce inline size
// https://github.com/dotnet/coreclr/pull/6103
throw new InvalidOperationException("Attempted to skip more bytes than available.");
}

}
}
32 changes: 19 additions & 13 deletions ...crosoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIteratorExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,14 @@ public static class MemoryPoolIteratorExtensions
public const string Http10Version = "HTTP/1.0";
public const string Http11Version = "HTTP/1.1";

// When this variable is set in KestrelEngine.JitReadonlyConsts it will
// deterministically convert the following readonly static values to jitted consts
// and embed the init code in that start up called function.
//
// This means when they are next used the have been pre-evaluated and the jit can
// either directly embed them or use them for branch elimiation.
internal static bool StaticReadonlysJitted;

// readonly primitive statics can be Jit'd to consts https://github.com/dotnet/coreclr/issues/1079
private readonly static long _httpConnectMethodLong = GetAsciiStringAsLong("CONNECT ");
private readonly static long _httpDeleteMethodLong = GetAsciiStringAsLong("DELETE \0");
Expand All @@ -36,19 +44,17 @@ public static class MemoryPoolIteratorExtensions
private readonly static long _mask5Chars = GetMaskAsLong(new byte[] { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00 });
private readonly static long _mask4Chars = GetMaskAsLong(new byte[] { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 });

private readonly static Tuple<long, long, string>[] _knownMethods = new Tuple<long, long, string>[8];

static MemoryPoolIteratorExtensions()
{
_knownMethods[0] = Tuple.Create(_mask4Chars, _httpPutMethodLong, HttpMethods.Put);
_knownMethods[1] = Tuple.Create(_mask5Chars, _httpPostMethodLong, HttpMethods.Post);
_knownMethods[2] = Tuple.Create(_mask5Chars, _httpHeadMethodLong, HttpMethods.Head);
_knownMethods[3] = Tuple.Create(_mask6Chars, _httpTraceMethodLong, HttpMethods.Trace);
_knownMethods[4] = Tuple.Create(_mask6Chars, _httpPatchMethodLong, HttpMethods.Patch);
_knownMethods[5] = Tuple.Create(_mask7Chars, _httpDeleteMethodLong, HttpMethods.Delete);
_knownMethods[6] = Tuple.Create(_mask8Chars, _httpConnectMethodLong, HttpMethods.Connect);
_knownMethods[7] = Tuple.Create(_mask8Chars, _httpOptionsMethodLong, HttpMethods.Options);
}
private readonly static Tuple<long, long, string>[] _knownMethods = new Tuple<long, long, string>[]
{
Tuple.Create(_mask4Chars, _httpPutMethodLong, HttpMethods.Put),
Tuple.Create(_mask5Chars, _httpPostMethodLong, HttpMethods.Post),
Tuple.Create(_mask5Chars, _httpHeadMethodLong, HttpMethods.Head),
Tuple.Create(_mask6Chars, _httpTraceMethodLong, HttpMethods.Trace),
Tuple.Create(_mask6Chars, _httpPatchMethodLong, HttpMethods.Patch),
Tuple.Create(_mask7Chars, _httpDeleteMethodLong, HttpMethods.Delete),
Tuple.Create(_mask8Chars, _httpConnectMethodLong, HttpMethods.Connect),
Tuple.Create(_mask8Chars, _httpOptionsMethodLong, HttpMethods.Options)
};

private unsafe static long GetAsciiStringAsLong(string str)
{
Expand Down
8 changes: 8 additions & 0 deletions src/Microsoft.AspNetCore.Server.Kestrel/Internal/KestrelEngine.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ internal KestrelEngine(Libuv uv, ServiceContext context)

public void Start(int count)
{
JitReadonlyConsts();

for (var index = 0; index < count; index++)
{
Threads.Add(new KestrelThread(this));
Expand All @@ -49,6 +51,12 @@ public void Start(int count)
}
}

public static void JitReadonlyConsts()
{
MemoryPoolIterator.StaticReadonlysJitted = true;
MemoryPoolIteratorExtensions.StaticReadonlysJitted = true;
}

public void Dispose()
{
Task.WaitAll(Threads.Select(thread => thread.StopAsync(TimeSpan.FromSeconds(2.5))).ToArray());
Expand Down