Skip to content
This repository was archived by the owner on Dec 18, 2018. It is now read-only.

Faster CopyFrom(+Ascii) #527

Merged
halter73 merged 1 commit into from
Jan 12, 2016
Merged

Faster CopyFrom(+Ascii) #527

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/Microsoft.AspNet.Server.Kestrel/Http/Connection.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ private Libuv.uv_buf_t OnAlloc(UvStreamHandle handle, int suggestedSize)

return handle.Libuv.buf_init(
result.Pin() + result.End,
result.Data.Offset + result.Data.Count - result.End);
result.BlockEndOffset - result.End);
}

private static void ReadCallback(UvStreamHandle handle, int status, object state)
Expand Down
2 changes: 1 addition & 1 deletion src/Microsoft.AspNet.Server.Kestrel/Http/SocketInput.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public MemoryPoolBlock2 IncomingStart()
{
const int minimumSize = 2048;

if (_tail != null && minimumSize <= _tail.Data.Offset + _tail.Data.Count - _tail.End)
if (_tail != null && minimumSize <= _tail.BlockEndOffset - _tail.End)
{
_pinned = _tail;
}
Expand Down
2 changes: 1 addition & 1 deletion src/Microsoft.AspNet.Server.Kestrel/Http/SocketOutput.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -400,7 +400,7 @@ private static void BytesBetween(MemoryPoolIterator2 start, MemoryPoolIterator2
return;
}

bytes = start.Block.Data.Offset + start.Block.Data.Count - start.Index;
bytes = start.Block.BlockEndOffset - start.Index;
buffers = 1;

for (var block = start.Block.Next; block != end.Block; block = block.Next)
Expand Down
6 changes: 6 additions & 0 deletions src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolBlock2.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,11 @@ protected MemoryPoolBlock2()
/// </summary>
public byte[] Array => Data.Array;

/// <summary>
/// Fixed end offset of the block
/// </summary>
public int BlockEndOffset { get; private set; }

/// <summary>
/// The Start represents the offset into Array where the range of "active" bytes begins. At the point when the block is leased
/// the Start is guaranteed to be equal to Array.Offset. The value of Start may be assigned anywhere between Data.Offset and
Expand Down Expand Up @@ -144,6 +149,7 @@ public static MemoryPoolBlock2 Create(
Slab = slab,
Start = data.Offset,
End = data.Offset,
BlockEndOffset = data.Offset + data.Count
};
}

Expand Down
187 changes: 126 additions & 61 deletions src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -692,46 +692,52 @@ public void CopyFrom(ArraySegment<byte> buffer)
CopyFrom(buffer.Array, buffer.Offset, buffer.Count);
}

public void CopyFrom(byte[] data, int offset, int count)
public unsafe void CopyFrom(byte[] data, int offset, int count)
{
Debug.Assert(_block != null);
Debug.Assert(_block.Next == null);
Debug.Assert(_block.End == _index);
Copy link
Copy Markdown
Member

@halter73 halter73 Jan 4, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any reason for removing these asserts?


var pool = _block.Pool;

var block = _block;
var blockIndex = _index;
var bytesLeftInBlock = block.BlockEndOffset - blockIndex;

var bufferIndex = offset;
var remaining = count;
var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex;
if (bytesLeftInBlock >= count)
{
_index = blockIndex + count;
Buffer.BlockCopy(data, offset, block.Array, blockIndex, count);
block.End = _index;
return;
}

while (remaining > 0)
do
{
if (bytesLeftInBlock == 0)
{
var nextBlock = pool.Lease();
block.End = blockIndex;
var nextBlock = block.Pool.Lease();
blockIndex = nextBlock.Data.Offset;
bytesLeftInBlock = nextBlock.Data.Count;
block.Next = nextBlock;
block = nextBlock;

blockIndex = block.Data.Offset;
bytesLeftInBlock = block.Data.Count;
}

var bytesToCopy = remaining < bytesLeftInBlock ? remaining : bytesLeftInBlock;

Buffer.BlockCopy(data, bufferIndex, block.Array, blockIndex, bytesToCopy);

blockIndex += bytesToCopy;
bufferIndex += bytesToCopy;
remaining -= bytesToCopy;
bytesLeftInBlock -= bytesToCopy;
}

block.End = blockIndex;
_block = block;
_index = blockIndex;
if (count > bytesLeftInBlock)
{
count -= bytesLeftInBlock;
Buffer.BlockCopy(data, offset, block.Array, blockIndex, bytesLeftInBlock);
offset += bytesLeftInBlock;
block.End = blockIndex + bytesLeftInBlock;
bytesLeftInBlock = 0;
}
else
{
_index = blockIndex + count;
Buffer.BlockCopy(data, offset, block.Array, blockIndex, count);
block.End = _index;
_block = block;
return;
}
} while (true);
}

public unsafe void CopyFromAscii(string data)
Expand All @@ -740,64 +746,123 @@ public unsafe void CopyFromAscii(string data)
Debug.Assert(_block.Next == null);
Debug.Assert(_block.End == _index);

var pool = _block.Pool;
var block = _block;
var blockIndex = _index;
var length = data.Length;
var count = data.Length;

var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex;
var bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3;
var blockRemaining = block.BlockEndOffset - blockIndex;

fixed (char* pData = data)
fixed (char* pInput = data)
{
var input = pData;
var inputEnd = pData + length;
var inputEndMinusSpan = inputEnd - 3;
if (blockRemaining >= count)
{
_index = blockIndex + count;

fixed (byte* pOutput = &block.Data.Array[blockIndex])
{
CopyFromAscii(pInput, pOutput, count);
}

while (input < inputEnd)
block.End = _index;
return;
}

var input = pInput;
do
{
if (bytesLeftInBlock == 0)
if (blockRemaining == 0)
{
var nextBlock = pool.Lease();
block.End = blockIndex;
var nextBlock = block.Pool.Lease();
blockIndex = nextBlock.Data.Offset;
blockRemaining = nextBlock.Data.Count;
block.Next = nextBlock;
block = nextBlock;

blockIndex = block.Data.Offset;
bytesLeftInBlock = block.Data.Count;
bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3;
}

fixed (byte* pOutput = &block.Data.Array[block.End])
if (count > blockRemaining)
{
//this line is needed to allow output be an register var
var output = pOutput;
count -= blockRemaining;

var copied = 0;
for (; input < inputEndMinusSpan && copied < bytesLeftInBlockMinusSpan; copied += 4)
fixed (byte* pOutput = &block.Data.Array[blockIndex])
{
*(output) = (byte)*(input);
*(output + 1) = (byte)*(input + 1);
*(output + 2) = (byte)*(input + 2);
*(output + 3) = (byte)*(input + 3);
output += 4;
input += 4;
CopyFromAscii(input, pOutput, blockRemaining);
}
for (; input < inputEnd && copied < bytesLeftInBlock; copied++)

block.End = blockIndex + blockRemaining;
input += blockRemaining;
blockRemaining = 0;
}
else
{
_index = blockIndex + count;

fixed (byte* pOutput = &block.Data.Array[blockIndex])
{
*(output++) = (byte)*(input++);
CopyFromAscii(input, pOutput, count);
}

blockIndex += copied;
bytesLeftInBlockMinusSpan -= copied;
bytesLeftInBlock -= copied;
block.End = _index;
_block = block;
return;
}
}
} while (true);
}
}

block.End = blockIndex;
_block = block;
_index = blockIndex;
private unsafe static void CopyFromAscii(char* pInput, byte* pOutput, int count)
{
var i = 0;
//these line is needed to allow input/output be an register var
var input = pInput;
var output = pOutput;

while (i < count - 11)
{
i += 12;
*(output) = (byte)*(input);
*(output + 1) = (byte)*(input + 1);
*(output + 2) = (byte)*(input + 2);
*(output + 3) = (byte)*(input + 3);
*(output + 4) = (byte)*(input + 4);
*(output + 5) = (byte)*(input + 5);
*(output + 6) = (byte)*(input + 6);
*(output + 7) = (byte)*(input + 7);
*(output + 8) = (byte)*(input + 8);
*(output + 9) = (byte)*(input + 9);
*(output + 10) = (byte)*(input + 10);
*(output + 11) = (byte)*(input + 11);
output += 12;
input += 12;
}
if (i < count - 5)
{
i += 6;
*(output) = (byte)*(input);
*(output + 1) = (byte)*(input + 1);
*(output + 2) = (byte)*(input + 2);
*(output + 3) = (byte)*(input + 3);
*(output + 4) = (byte)*(input + 4);
*(output + 5) = (byte)*(input + 5);
output += 6;
input += 6;
}
if (i < count - 3)
{
i += 4;
*(output) = (byte)*(input);
*(output + 1) = (byte)*(input + 1);
*(output + 2) = (byte)*(input + 2);
*(output + 3) = (byte)*(input + 3);
output += 4;
input += 4;
}
while (i < count)
{
i++;
*output = (byte)*input;
output++;
input++;
}
}
}
}
2 changes: 1 addition & 1 deletion src/Microsoft.AspNet.Server.Kestrel/Networking/UvWriteReq.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ public unsafe void Write(
for (var index = 0; index < nBuffers; index++)
{
var blockStart = block == start.Block ? start.Index : block.Data.Offset;
var blockEnd = block == end.Block ? end.Index : block.Data.Offset + block.Data.Count;
var blockEnd = block == end.Block ? end.Index : block.BlockEndOffset;

// create and pin each segment being written
pBuffers[index] = Libuv.buf_init(
Expand Down
2 changes: 1 addition & 1 deletion test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolBlock2Tests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ public void CopyFromCorrectlyTraversesBlocks()

Assert.Null(block1.Next);

end.CopyFrom(new ArraySegment<byte>(buffer));
end.CopyFrom(buffer, 0, buffer.Length);

Assert.NotNull(block1.Next);

Expand Down
4 changes: 2 additions & 2 deletions test/Microsoft.AspNet.Server.KestrelTests/SocketOutputTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -308,11 +308,11 @@ public void ProducingStartAndProducingCompleteCanBeUsedDirectly()

// block 1
var start = socketOutput.ProducingStart();
start.Block.End = start.Block.Data.Offset + start.Block.Data.Count;
start.Block.End = start.Block.BlockEndOffset;

// block 2
var block2 = memory.Lease();
block2.End = block2.Data.Offset + block2.Data.Count;
block2.End = block2.BlockEndOffset;
start.Block.Next = block2;

var end = new MemoryPoolIterator2(block2, block2.End);
Expand Down