From b3335785be6b1b5ac5cb831786c56ed62e0dadb0 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Tue, 4 Oct 2016 01:55:39 +0100 Subject: [PATCH 1/2] Don't spill Vectors from registers to stack --- .../Infrastructure/MemoryPoolIterator.cs | 105 +++++++++++------- 1 file changed, 64 insertions(+), 41 deletions(-) diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs index b0b6837f6..9bf942afc 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs @@ -269,40 +269,44 @@ public unsafe int Seek( { var byte0Equals = Vector.Equals(new Vector(array, index), byte0Vector); - if (byte0Equals.Equals(Vector.Zero)) + if (!byte0Equals.Equals(Vector.Zero)) { - if (bytesScanned + _vectorSpan >= limit) + _block = block; + + // Make a copy and pass it by ref. + // As a result byte0Equals will not be marked as addr-exposed and will be reg allocated + // Note that making a copy under h/w acceleration is equal to a reg-to-reg or reg-to-mem move + Vector tmp = byte0Equals; + var firstEqualByteIndex = FindFirstEqualByte(ref tmp); + var vectorBytesScanned = firstEqualByteIndex + 1; + + if (bytesScanned + vectorBytesScanned > limit) { - _block = block; // Ensure iterator is left at limit position _index = index + (limit - bytesScanned); bytesScanned = limit; return -1; } - bytesScanned += _vectorSpan; - following -= _vectorSpan; - index += _vectorSpan; - continue; - } - - _block = block; + _index = index + firstEqualByteIndex; + bytesScanned += vectorBytesScanned; - var firstEqualByteIndex = FindFirstEqualByte(ref byte0Equals); - var vectorBytesScanned = firstEqualByteIndex + 1; + return byte0; + } - if (bytesScanned + vectorBytesScanned > limit) + if (bytesScanned + _vectorSpan >= limit) { + _block = block; // Ensure iterator is left at limit position _index = index + (limit - bytesScanned); bytesScanned = limit; return -1; } - _index = index + firstEqualByteIndex; - bytesScanned += vectorBytesScanned; - - return byte0; + bytesScanned += _vectorSpan; + following -= _vectorSpan; + index += _vectorSpan; + continue; } // Need unit tests to test Vector path #if !DEBUG @@ -377,35 +381,38 @@ public unsafe int Seek( { var byte0Equals = Vector.Equals(new Vector(array, index), byte0Vector); - if (byte0Equals.Equals(Vector.Zero)) + if (!byte0Equals.Equals(Vector.Zero)) { - if (block == limit.Block && index + _vectorSpan > limit.Index) + _block = block; + + // Make a copy and pass it by ref. + // As a result byte0Equals will not be marked as addr-exposed and will be reg allocated + // Note that making a copy under h/w acceleration is equal to a reg-to-reg or reg-to-mem move + Vector tmp = byte0Equals; + var firstEqualByteIndex = FindFirstEqualByte(ref tmp); + + if (_block == limit.Block && index + firstEqualByteIndex > limit.Index) { - _block = block; // Ensure iterator is left at limit position _index = limit.Index; return -1; } - following -= _vectorSpan; - index += _vectorSpan; - continue; - } - - _block = block; + _index = index + firstEqualByteIndex; - var firstEqualByteIndex = FindFirstEqualByte(ref byte0Equals); + return byte0; + } - if (_block == limit.Block && index + firstEqualByteIndex > limit.Index) + if (block == limit.Block && index + _vectorSpan > limit.Index) { + _block = block; // Ensure iterator is left at limit position _index = limit.Index; return -1; } - _index = index + firstEqualByteIndex; - - return byte0; + following -= _vectorSpan; + index += _vectorSpan; } // Need unit tests to test Vector path #if !DEBUG @@ -487,17 +494,24 @@ public unsafe int Seek( #endif if (following >= _vectorSpan) { + Vector tmp; var data = new Vector(array, index); - var byte0Equals = Vector.Equals(data, byte0Vector); - var byte1Equals = Vector.Equals(data, byte1Vector); + var byte0Equals = Vector.Equals(data, byte0Vector); if (!byte0Equals.Equals(Vector.Zero)) { - byte0Index = FindFirstEqualByte(ref byte0Equals); + // Make a copy and pass it by ref. + // As a result byte0Equals will not be marked as addr-exposed and will be reg allocated + // Note that making a copy under h/w acceleration is equal to a reg-to-reg or reg-to-mem move + tmp = byte0Equals; + byte0Index = FindFirstEqualByte(ref tmp); } + + var byte1Equals = Vector.Equals(data, byte1Vector); if (!byte1Equals.Equals(Vector.Zero)) { - byte1Index = FindFirstEqualByte(ref byte1Equals); + tmp = byte1Equals; + byte1Index = FindFirstEqualByte(ref tmp); } if (byte0Index == int.MaxValue && byte1Index == int.MaxValue) @@ -630,22 +644,31 @@ public unsafe int Seek( #endif if (following >= _vectorSpan) { + Vector tmp; var data = new Vector(array, index); - var byte0Equals = Vector.Equals(data, byte0Vector); - var byte1Equals = Vector.Equals(data, byte1Vector); - var byte2Equals = Vector.Equals(data, byte2Vector); + var byte0Equals = Vector.Equals(data, byte0Vector); if (!byte0Equals.Equals(Vector.Zero)) { - byte0Index = FindFirstEqualByte(ref byte0Equals); + // Make a copy and pass it by ref. + // As a result byte0Equals will not be marked as addr-exposed and will be reg allocated + // Note that making a copy under h/w acceleration is equal to a reg-to-reg or reg-to-mem move + tmp = byte0Equals; + byte0Index = FindFirstEqualByte(ref tmp); } + + var byte1Equals = Vector.Equals(data, byte1Vector); if (!byte1Equals.Equals(Vector.Zero)) { - byte1Index = FindFirstEqualByte(ref byte1Equals); + tmp = byte1Equals; + byte1Index = FindFirstEqualByte(ref tmp); } + + var byte2Equals = Vector.Equals(data, byte2Vector); if (!byte2Equals.Equals(Vector.Zero)) { - byte2Index = FindFirstEqualByte(ref byte2Equals); + tmp = byte2Equals; + byte2Index = FindFirstEqualByte(ref tmp); } if (byte0Index == int.MaxValue && byte1Index == int.MaxValue && byte2Index == int.MaxValue) From d31cb8876c9a32577d030e3b0dc999a4d385e62c Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Wed, 5 Oct 2016 01:35:08 +0100 Subject: [PATCH 2/2] Pass the Vectors byval since that's byref anyway --- .../Infrastructure/MemoryPoolIterator.cs | 42 +++++-------------- 1 file changed, 10 insertions(+), 32 deletions(-) diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs index 9bf942afc..08718d6db 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs @@ -268,16 +268,11 @@ public unsafe int Seek( if (following >= _vectorSpan) { var byte0Equals = Vector.Equals(new Vector(array, index), byte0Vector); - if (!byte0Equals.Equals(Vector.Zero)) { _block = block; - // Make a copy and pass it by ref. - // As a result byte0Equals will not be marked as addr-exposed and will be reg allocated - // Note that making a copy under h/w acceleration is equal to a reg-to-reg or reg-to-mem move - Vector tmp = byte0Equals; - var firstEqualByteIndex = FindFirstEqualByte(ref tmp); + var firstEqualByteIndex = FindFirstEqualByte(byte0Equals); var vectorBytesScanned = firstEqualByteIndex + 1; if (bytesScanned + vectorBytesScanned > limit) @@ -385,11 +380,7 @@ public unsafe int Seek( { _block = block; - // Make a copy and pass it by ref. - // As a result byte0Equals will not be marked as addr-exposed and will be reg allocated - // Note that making a copy under h/w acceleration is equal to a reg-to-reg or reg-to-mem move - Vector tmp = byte0Equals; - var firstEqualByteIndex = FindFirstEqualByte(ref tmp); + var firstEqualByteIndex = FindFirstEqualByte(byte0Equals); if (_block == limit.Block && index + firstEqualByteIndex > limit.Index) { @@ -494,24 +485,18 @@ public unsafe int Seek( #endif if (following >= _vectorSpan) { - Vector tmp; var data = new Vector(array, index); var byte0Equals = Vector.Equals(data, byte0Vector); if (!byte0Equals.Equals(Vector.Zero)) { - // Make a copy and pass it by ref. - // As a result byte0Equals will not be marked as addr-exposed and will be reg allocated - // Note that making a copy under h/w acceleration is equal to a reg-to-reg or reg-to-mem move - tmp = byte0Equals; - byte0Index = FindFirstEqualByte(ref tmp); + byte0Index = FindFirstEqualByte(byte0Equals); } var byte1Equals = Vector.Equals(data, byte1Vector); if (!byte1Equals.Equals(Vector.Zero)) { - tmp = byte1Equals; - byte1Index = FindFirstEqualByte(ref tmp); + byte1Index = FindFirstEqualByte(byte1Equals); } if (byte0Index == int.MaxValue && byte1Index == int.MaxValue) @@ -644,31 +629,24 @@ public unsafe int Seek( #endif if (following >= _vectorSpan) { - Vector tmp; var data = new Vector(array, index); var byte0Equals = Vector.Equals(data, byte0Vector); if (!byte0Equals.Equals(Vector.Zero)) { - // Make a copy and pass it by ref. - // As a result byte0Equals will not be marked as addr-exposed and will be reg allocated - // Note that making a copy under h/w acceleration is equal to a reg-to-reg or reg-to-mem move - tmp = byte0Equals; - byte0Index = FindFirstEqualByte(ref tmp); + byte0Index = FindFirstEqualByte(byte0Equals); } var byte1Equals = Vector.Equals(data, byte1Vector); if (!byte1Equals.Equals(Vector.Zero)) { - tmp = byte1Equals; - byte1Index = FindFirstEqualByte(ref tmp); + byte1Index = FindFirstEqualByte(byte1Equals); } var byte2Equals = Vector.Equals(data, byte2Vector); if (!byte2Equals.Equals(Vector.Zero)) { - tmp = byte2Equals; - byte2Index = FindFirstEqualByte(ref tmp); + byte2Index = FindFirstEqualByte(byte2Equals); } if (byte0Index == int.MaxValue && byte1Index == int.MaxValue && byte2Index == int.MaxValue) @@ -770,9 +748,9 @@ public unsafe int Seek( /// /// The first index of the result vector /// byteEquals = 0 - internal static int FindFirstEqualByte(ref Vector byteEquals) + internal static int FindFirstEqualByte(Vector byteEquals) { - if (!BitConverter.IsLittleEndian) return FindFirstEqualByteSlow(ref byteEquals); + if (!BitConverter.IsLittleEndian) return FindFirstEqualByteSlow(byteEquals); // Quasi-tree search var vector64 = Vector.AsVectorInt64(byteEquals); @@ -794,7 +772,7 @@ internal static int FindFirstEqualByte(ref Vector byteEquals) } // Internal for testing - internal static int FindFirstEqualByteSlow(ref Vector byteEquals) + internal static int FindFirstEqualByteSlow(Vector byteEquals) { // Quasi-tree search var vector64 = Vector.AsVectorInt64(byteEquals);