diff --git a/src/System.Memory/src/System.Memory.csproj b/src/System.Memory/src/System.Memory.csproj
index 4ff412887ec3..6badf10d9295 100644
--- a/src/System.Memory/src/System.Memory.csproj
+++ b/src/System.Memory/src/System.Memory.csproj
@@ -37,11 +37,13 @@
+
+
diff --git a/src/System.Memory/src/System/Pinnable.cs b/src/System.Memory/src/System/Pinnable.cs
index 0f9b02ad94c6..766811cc3a33 100644
--- a/src/System.Memory/src/System/Pinnable.cs
+++ b/src/System.Memory/src/System/Pinnable.cs
@@ -2,7 +2,6 @@
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
-using System.Diagnostics;
using System.Runtime.InteropServices;
namespace System
diff --git a/src/System.Memory/src/System/ReadOnlySpan.cs b/src/System.Memory/src/System/ReadOnlySpan.cs
index 0f3c434fdc10..ec443aa08908 100644
--- a/src/System.Memory/src/System/ReadOnlySpan.cs
+++ b/src/System.Memory/src/System/ReadOnlySpan.cs
@@ -28,8 +28,9 @@ public struct ReadOnlySpan
public ReadOnlySpan(T[] array)
{
if (array == null)
- ThrowHelper.ThrowArgumentNullException(ExceptionArgument.array);
- if (default(T) == null && array.GetType() != typeof(T[]))
+ ThrowHelper.ThrowArgumentNullException_Array();
+ // Check will be Jitted out for ValueTypes
+ if (!SpanHelpers.IsValueType() && array.GetType() != typeof(T[]))
ThrowHelper.ThrowArrayTypeMismatchException_ArrayTypeMustBeExactMatch(typeof(T));
_length = array.Length;
@@ -52,16 +53,13 @@ public ReadOnlySpan(T[] array)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public ReadOnlySpan(T[] array, int start)
{
- if (array == null)
- ThrowHelper.ThrowArgumentNullException(ExceptionArgument.array);
- if (default(T) == null && array.GetType() != typeof(T[]))
+ if (array == null || (uint)start > (uint)array.Length)
+ ThrowHelper.ThrowArgumentNullOrOutOfRangeException(array);
+ // Check will be Jitted out for ValueTypes
+ if (!SpanHelpers.IsValueType() && array.GetType() != typeof(T[]))
ThrowHelper.ThrowArrayTypeMismatchException_ArrayTypeMustBeExactMatch(typeof(T));
- int arrayLength = array.Length;
- if ((uint)start > (uint)arrayLength)
- ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start);
-
- _length = arrayLength - start;
+ _length = array.Length - start;
_pinnable = Unsafe.As>(array);
_byteOffset = SpanHelpers.PerTypeValues.ArrayAdjustment.Add(start);
}
@@ -82,12 +80,11 @@ public ReadOnlySpan(T[] array, int start)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public ReadOnlySpan(T[] array, int start, int length)
{
- if (array == null)
- ThrowHelper.ThrowArgumentNullException(ExceptionArgument.array);
- if (default(T) == null && array.GetType() != typeof(T[]))
+ if (array == null || (uint)start > (uint)array.Length || (uint)length > (uint)(array.Length - start))
+ ThrowHelper.ThrowArgumentNullOrOutOfRangeException(array);
+ // Check will be Jitted out for ValueTypes
+ if (!SpanHelpers.IsValueType() && array.GetType() != typeof(T[]))
ThrowHelper.ThrowArrayTypeMismatchException_ArrayTypeMustBeExactMatch(typeof(T));
- if ((uint)start > (uint)array.Length || (uint)length > (uint)(array.Length - start))
- ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start);
_length = length;
_pinnable = Unsafe.As>(array);
@@ -111,10 +108,11 @@ public ReadOnlySpan(T[] array, int start, int length)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public unsafe ReadOnlySpan(void* pointer, int length)
{
+ // Check will be Jitted out for valid types
if (SpanHelpers.IsReferenceOrContainsReferences())
ThrowHelper.ThrowArgumentException_InvalidTypeWithPointersNotSupported(typeof(T));
if (length < 0)
- ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start);
+ ThrowHelper.ThrowArgumentOutOfRangeException_Length();
_length = length;
_pinnable = null;
@@ -138,10 +136,8 @@ public unsafe ReadOnlySpan(void* pointer, int length)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static ReadOnlySpan DangerousCreate(object obj, ref T objectData, int length)
{
- if (obj == null)
- ThrowHelper.ThrowArgumentNullException(ExceptionArgument.obj);
- if (length < 0)
- ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.length);
+ if (obj == null || length < 0)
+ ThrowHelper.ThrowArgumentNullOrOutOfRangeException(length);
Pinnable pinnable = Unsafe.As>(obj);
IntPtr byteOffset = Unsafe.ByteOffset(ref pinnable.Data, ref objectData);
@@ -315,11 +311,10 @@ public override int GetHashCode()
public ReadOnlySpan Slice(int start)
{
if ((uint)start > (uint)_length)
- ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start);
+ ThrowHelper.ThrowArgumentOutOfRangeException_Start();
IntPtr newOffset = _byteOffset.Add(start);
- int length = _length - start;
- return new ReadOnlySpan(_pinnable, newOffset, length);
+ return new ReadOnlySpan(_pinnable, newOffset, _length - start);
}
///
@@ -334,7 +329,7 @@ public ReadOnlySpan Slice(int start)
public ReadOnlySpan Slice(int start, int length)
{
if ((uint)start > (uint)_length || (uint)length > (uint)(_length - start))
- ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start);
+ ThrowHelper.ThrowArgumentOutOfRangeException_Start();
IntPtr newOffset = _byteOffset.Add(start);
return new ReadOnlySpan(_pinnable, newOffset, length);
diff --git a/src/System.Memory/src/System/Span.cs b/src/System.Memory/src/System/Span.cs
index 7fd8348c99fd..ebb4d8d0c8a3 100644
--- a/src/System.Memory/src/System/Span.cs
+++ b/src/System.Memory/src/System/Span.cs
@@ -28,8 +28,9 @@ public struct Span
public Span(T[] array)
{
if (array == null)
- ThrowHelper.ThrowArgumentNullException(ExceptionArgument.array);
- if (default(T) == null && array.GetType() != typeof(T[]))
+ ThrowHelper.ThrowArgumentNullException_Array();
+ // Check will be Jitted out for ValueTypes
+ if (!SpanHelpers.IsValueType() && array.GetType() != typeof(T[]))
ThrowHelper.ThrowArrayTypeMismatchException_ArrayTypeMustBeExactMatch(typeof(T));
_length = array.Length;
@@ -52,16 +53,13 @@ public Span(T[] array)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Span(T[] array, int start)
{
- if (array == null)
- ThrowHelper.ThrowArgumentNullException(ExceptionArgument.array);
- if (default(T) == null && array.GetType() != typeof(T[]))
+ if (array == null || (uint)start > (uint)array.Length)
+ ThrowHelper.ThrowArgumentNullOrOutOfRangeException(array);
+ // Check will be Jitted out for ValueTypes
+ if (!SpanHelpers.IsValueType() && array.GetType() != typeof(T[]))
ThrowHelper.ThrowArrayTypeMismatchException_ArrayTypeMustBeExactMatch(typeof(T));
- int arrayLength = array.Length;
- if ((uint)start > (uint)arrayLength)
- ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start);
-
- _length = arrayLength - start;
+ _length = array.Length - start;
_pinnable = Unsafe.As>(array);
_byteOffset = SpanHelpers.PerTypeValues.ArrayAdjustment.Add(start);
}
@@ -82,12 +80,11 @@ public Span(T[] array, int start)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Span(T[] array, int start, int length)
{
- if (array == null)
- ThrowHelper.ThrowArgumentNullException(ExceptionArgument.array);
- if (default(T) == null && array.GetType() != typeof(T[]))
+ if (array == null || (uint)start > (uint)array.Length || (uint)length > (uint)(array.Length - start))
+ ThrowHelper.ThrowArgumentNullOrOutOfRangeException(array);
+ // Check will be Jitted out for ValueTypes
+ if (!SpanHelpers.IsValueType() && array.GetType() != typeof(T[]))
ThrowHelper.ThrowArrayTypeMismatchException_ArrayTypeMustBeExactMatch(typeof(T));
- if ((uint)start > (uint)array.Length || (uint)length > (uint)(array.Length - start))
- ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start);
_length = length;
_pinnable = Unsafe.As>(array);
@@ -111,10 +108,11 @@ public Span(T[] array, int start, int length)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public unsafe Span(void* pointer, int length)
{
+ // Check will be Jitted out for valid types
if (SpanHelpers.IsReferenceOrContainsReferences())
ThrowHelper.ThrowArgumentException_InvalidTypeWithPointersNotSupported(typeof(T));
if (length < 0)
- ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start);
+ ThrowHelper.ThrowArgumentOutOfRangeException_Length();
_length = length;
_pinnable = null;
@@ -138,10 +136,8 @@ public unsafe Span(void* pointer, int length)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Span DangerousCreate(object obj, ref T objectData, int length)
{
- if (obj == null)
- ThrowHelper.ThrowArgumentNullException(ExceptionArgument.obj);
- if (length < 0)
- ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.length);
+ if (obj == null || length < 0)
+ ThrowHelper.ThrowArgumentNullOrOutOfRangeException(length);
Pinnable pinnable = Unsafe.As>(obj);
IntPtr byteOffset = Unsafe.ByteOffset(ref pinnable.Data, ref objectData);
@@ -244,6 +240,7 @@ public unsafe void Clear()
var byteLength = (UIntPtr)((uint)length * Unsafe.SizeOf());
+ // Branch will be Jit eliminated
if ((Unsafe.SizeOf() & (sizeof(IntPtr) - 1)) != 0)
{
if (_pinnable == null)
@@ -261,6 +258,7 @@ public unsafe void Clear()
}
else
{
+ // Branch will be Jit eliminated
if (SpanHelpers.IsReferenceOrContainsReferences())
{
UIntPtr pointerSizedLength = (UIntPtr)((length * Unsafe.SizeOf()) / sizeof(IntPtr));
@@ -288,6 +286,7 @@ public unsafe void Fill(T value)
if (length == 0)
return;
+ // Branch will be Jit eliminated
if (Unsafe.SizeOf() == 1)
{
byte fill = Unsafe.As(ref value);
@@ -463,11 +462,10 @@ public override int GetHashCode()
public Span Slice(int start)
{
if ((uint)start > (uint)_length)
- ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start);
+ ThrowHelper.ThrowArgumentOutOfRangeException_Start();
IntPtr newOffset = _byteOffset.Add(start);
- int length = _length - start;
- return new Span(_pinnable, newOffset, length);
+ return new Span(_pinnable, newOffset, _length - start);
}
///
@@ -482,7 +480,7 @@ public Span Slice(int start)
public Span Slice(int start, int length)
{
if ((uint)start > (uint)_length || (uint)length > (uint)(_length - start))
- ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start);
+ ThrowHelper.ThrowArgumentOutOfRangeException_Start();
IntPtr newOffset = _byteOffset.Add(start);
return new Span(_pinnable, newOffset, length);
@@ -562,6 +560,7 @@ public static class Span
public static Span AsBytes(this Span source)
where T : struct
{
+ // Check will be Jitted out for valid types
if (SpanHelpers.IsReferenceOrContainsReferences())
ThrowHelper.ThrowArgumentException_InvalidTypeWithPointersNotSupported(typeof(T));
@@ -584,6 +583,7 @@ public static Span AsBytes(this Span source)
public static ReadOnlySpan AsBytes(this ReadOnlySpan source)
where T : struct
{
+ // Check will be Jitted out for valid types
if (SpanHelpers.IsReferenceOrContainsReferences())
ThrowHelper.ThrowArgumentException_InvalidTypeWithPointersNotSupported(typeof(T));
@@ -610,9 +610,10 @@ public static Span NonPortableCast(this Span source)
where TFrom : struct
where TTo : struct
{
+ // Check will be Jitted out for valid types
if (SpanHelpers.IsReferenceOrContainsReferences())
ThrowHelper.ThrowArgumentException_InvalidTypeWithPointersNotSupported(typeof(TFrom));
-
+ // Check will be Jitted out for valid types
if (SpanHelpers.IsReferenceOrContainsReferences())
ThrowHelper.ThrowArgumentException_InvalidTypeWithPointersNotSupported(typeof(TTo));
@@ -639,9 +640,10 @@ public static ReadOnlySpan NonPortableCast(this ReadOnlySpan())
ThrowHelper.ThrowArgumentException_InvalidTypeWithPointersNotSupported(typeof(TFrom));
-
+ // Check will be Jitted out for valid types
if (SpanHelpers.IsReferenceOrContainsReferences())
ThrowHelper.ThrowArgumentException_InvalidTypeWithPointersNotSupported(typeof(TTo));
@@ -658,7 +660,7 @@ public static ReadOnlySpan NonPortableCast(this ReadOnlySpan Slice(this string text)
{
if (text == null)
- ThrowHelper.ThrowArgumentNullException(ExceptionArgument.text);
+ ThrowHelper.ThrowArgumentNullException_Text();
return new ReadOnlySpan(Unsafe.As>(text), StringAdjustment, text.Length);
}
@@ -675,16 +677,13 @@ public static ReadOnlySpan Slice(this string text)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static ReadOnlySpan Slice(this string text, int start)
{
- if (text == null)
- ThrowHelper.ThrowArgumentNullException(ExceptionArgument.text);
- int textLength = text.Length;
- if ((uint)start > (uint)textLength)
- ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start);
+ if (text == null || (uint)start > (uint)text.Length)
+ ThrowHelper.ThrowArgumentNullOrOutOfRangeException(text);
unsafe
{
byte* byteOffset = ((byte*)StringAdjustment) + (uint)(start * sizeof(char));
- return new ReadOnlySpan(Unsafe.As>(text), (IntPtr)byteOffset, textLength - start);
+ return new ReadOnlySpan(Unsafe.As>(text), (IntPtr)byteOffset, text.Length - start);
}
}
@@ -701,11 +700,8 @@ public static ReadOnlySpan Slice(this string text, int start)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static ReadOnlySpan Slice(this string text, int start, int length)
{
- if (text == null)
- ThrowHelper.ThrowArgumentNullException(ExceptionArgument.text);
- int textLength = text.Length;
- if ((uint)start > (uint)textLength || (uint)length > (uint)(textLength - start))
- ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.start);
+ if (text == null || (uint)start > (uint)text.Length || (uint)length > (uint)(text.Length - start))
+ ThrowHelper.ThrowArgumentNullOrOutOfRangeException(text);
unsafe
{
diff --git a/src/System.Memory/src/System/SpanExtensions.cs b/src/System.Memory/src/System/SpanExtensions.cs
index 94b7b7deb7b3..8daa60274a81 100644
--- a/src/System.Memory/src/System/SpanExtensions.cs
+++ b/src/System.Memory/src/System/SpanExtensions.cs
@@ -2,7 +2,6 @@
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
-using System.Diagnostics;
using System.Runtime.CompilerServices;
namespace System
@@ -32,6 +31,10 @@ public static int IndexOf(this Span span, T value)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int IndexOf(this Span span, byte value)
{
+ if (!BitConverter.IsLittleEndian)
+ {
+ return SpanHelpers.IndexOfBigEndian(ref span.DangerousGetPinnableReference(), value, span.Length);
+ }
return SpanHelpers.IndexOf(ref span.DangerousGetPinnableReference(), value, span.Length);
}
@@ -131,6 +134,10 @@ public static int IndexOf(this ReadOnlySpan span, T value)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int IndexOf(this ReadOnlySpan span, byte value)
{
+ if (!BitConverter.IsLittleEndian)
+ {
+ return SpanHelpers.IndexOfBigEndian(ref span.DangerousGetPinnableReference(), value, span.Length);
+ }
return SpanHelpers.IndexOf(ref span.DangerousGetPinnableReference(), value, span.Length);
}
diff --git a/src/System.Memory/src/System/SpanHelpers.byte.cs b/src/System.Memory/src/System/SpanHelpers.byte.cs
index ba1a6ca8796e..11af5fede365 100644
--- a/src/System.Memory/src/System/SpanHelpers.byte.cs
+++ b/src/System.Memory/src/System/SpanHelpers.byte.cs
@@ -3,6 +3,7 @@
// See the LICENSE file in the project root for more information.
using System.Diagnostics;
+using System.Numerics;
using System.Runtime.CompilerServices;
namespace System
@@ -30,7 +31,16 @@ public static int IndexOf(ref byte searchSpace, int searchSpaceLength, ref byte
return -1; // The unsearched portion is now shorter than the sequence we're looking for. So it can't be there.
// Do a quick search for the first element of "value".
- int relativeIndex = IndexOf(ref Unsafe.Add(ref searchSpace, index), valueHead, remainingSearchSpaceLength);
+ int relativeIndex;
+ if (!BitConverter.IsLittleEndian)
+ {
+ relativeIndex = IndexOfBigEndian(ref Unsafe.Add(ref searchSpace, index), valueHead, remainingSearchSpaceLength);
+ }
+ else
+ {
+ relativeIndex = IndexOf(ref Unsafe.Add(ref searchSpace, index), valueHead, remainingSearchSpaceLength);
+ }
+
if (relativeIndex == -1)
return -1;
index += relativeIndex;
@@ -43,7 +53,7 @@ public static int IndexOf(ref byte searchSpace, int searchSpaceLength, ref byte
}
}
- public static int IndexOf(ref byte searchSpace, byte value, int length)
+ internal static int IndexOfBigEndian(ref byte searchSpace, byte value, int length)
{
Debug.Assert(length >= 0);
@@ -95,82 +105,218 @@ public static int IndexOf(ref byte searchSpace, byte value, int length)
return -1;
}
- public static bool SequenceEqual(ref byte first, ref byte second, int length)
+ public unsafe static bool SequenceEqual(ref byte first, ref byte second, int length)
{
Debug.Assert(length >= 0);
+ var isMatch = true;
- if (Unsafe.AreSame(ref first, ref second))
- return true;
-
- int index = 0;
- while (length >= 8)
+ if (length == 0)
{
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
-
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
-
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
-
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
-
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
-
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
-
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
-
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
-
+ goto exit;
+ }
- length -= 8;
+ fixed (byte* pFirst = &first)
+ fixed (byte* pSecond = &second)
+ {
+ var a = pFirst;
+ var b = pSecond;
+
+ if (a == b)
+ {
+ goto exitFixed;
+ }
+
+ var i = 0;
+ if (Vector.IsHardwareAccelerated)
+ {
+ while (length - Vector.Count >= i)
+ {
+ var v0 = Unsafe.Read>(a + i);
+ var v1 = Unsafe.Read>(b + i);
+ i += Vector.Count;
+
+ if (!v0.Equals(v1))
+ {
+ isMatch = false;
+ goto exitFixed;
+ }
+
+ }
+ }
+
+ while (length - sizeof(long) >= i)
+ {
+ if(*(long*)(a + i) != *(long*)(b + i))
+ {
+ isMatch = false;
+ goto exitFixed;
+ }
+
+ i += sizeof(long);
+ }
+
+ if (length - sizeof(int) >= i)
+ {
+ if (*(int*)(a + i) != *(int*)(b + i))
+ {
+ isMatch = false;
+ goto exitFixed;
+ }
+
+ i += sizeof(int);
+ }
+
+ if (length - sizeof(short) >= i)
+ {
+ if (*(short*)(a + i) != *(short*)(b + i))
+ {
+ isMatch = false;
+ goto exitFixed;
+ }
+
+ i += sizeof(short);
+ }
+
+ if (length > i && *(a + i) != *(b + i))
+ {
+ isMatch = false;
+ }
+ // Don't goto out of fixed block
+ exitFixed:;
}
+ exit:
+ return isMatch;
+ }
- while (length >= 4)
+ public unsafe static int IndexOf(ref byte searchSpace, byte value, int length)
+ {
+ Debug.Assert(length >= 0);
+ var index = -1;
+ if (length == 0)
{
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
+ goto exit;
+ }
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
+ fixed (byte* pHaystack = &searchSpace)
+ {
+ var haystack = pHaystack;
+ index = 0;
+
+ if (Vector.IsHardwareAccelerated)
+ {
+ if (length - Vector.Count >= index)
+ {
+ Vector needles = GetVector(value);
+ do
+ {
+ var flaggedMatches = Vector.Equals(Unsafe.Read>(haystack + index), needles);
+ if (flaggedMatches.Equals(Vector.Zero))
+ {
+ index += Vector.Count;
+ continue;
+ }
+
+ index += LocateFirstFoundByte(flaggedMatches);
+ goto exitFixed;
+
+ } while (length - Vector.Count >= index);
+ }
+ }
+
+ while (length - sizeof(ulong) >= index)
+ {
+ var flaggedMatches = SetLowBitsForByteMatch(*(ulong*)(haystack + index), value);
+ if (flaggedMatches == 0)
+ {
+ index += sizeof(ulong);
+ continue;
+ }
+
+ index += LocateFirstFoundByte(flaggedMatches);
+ goto exitFixed;
+ }
+
+ for (; index < length; index++)
+ {
+ if (*(haystack + index) == value)
+ {
+ goto exitFixed;
+ }
+ }
+ // No Matches
+ index = -1;
+ // Don't goto out of fixed block
+ exitFixed:;
+ }
+ exit:
+ return index;
+ }
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static int LocateFirstFoundByte(Vector match)
+ {
+ var vector64 = Vector.AsVectorUInt64(match);
+ ulong candidate = 0;
+ var i = 0;
+ // Pattern unrolled by jit https://github.com/dotnet/coreclr/pull/8001
+ for (; i < Vector.Count; i++)
+ {
+ candidate = vector64[i];
+ if (candidate == 0) continue;
+ break;
+ }
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
+ // Single LEA instruction with jitted const (using function result)
+ return i * 8 + LocateFirstFoundByte(candidate);
+ }
- length -= 4;
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static int LocateFirstFoundByte(ulong match)
+ {
+ unchecked
+ {
+ // Flag least significant power of two bit
+ var powerOfTwoFlag = match ^ (match - 1);
+ // Shift all powers of two into the high byte and extract
+ return (int)((powerOfTwoFlag * xorPowerOfTwoToHighByte) >> 57);
}
+ }
- while (length > 0)
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static ulong SetLowBitsForByteMatch(ulong potentialMatch, byte search)
+ {
+ unchecked
{
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
- length--;
+ var flaggedValue = potentialMatch ^ (byteBroadcastToUlong * search);
+ return (
+ (flaggedValue - byteBroadcastToUlong) &
+ ~(flaggedValue) &
+ filterByteHighBitsInUlong
+ ) >> 7;
}
+ }
- return true;
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static Vector GetVector(byte vectorByte)
+ {
+#if !NETCOREAPP1_2
+ // Vector .ctor doesn't become an intrinsic due to detection issue
+ // However this does cause it to become an intrinsic (with additional multiply and reg->reg copy)
+ // https://github.com/dotnet/coreclr/issues/7459#issuecomment-253965670
+ return Vector.AsVectorByte(new Vector(vectorByte * 0x01010101u));
+#else
+ return new Vector(vectorByte);
+#endif
}
+
+ private const ulong xorPowerOfTwoToHighByte = (0x07ul |
+ 0x06ul << 8 |
+ 0x05ul << 16 |
+ 0x04ul << 24 |
+ 0x03ul << 32 |
+ 0x02ul << 40 |
+ 0x01ul << 48) + 1;
+ private const ulong byteBroadcastToUlong = ~0UL / byte.MaxValue;
+ private const ulong filterByteHighBitsInUlong = (byteBroadcastToUlong >> 1) | (byteBroadcastToUlong << (sizeof(ulong) * 8 - 1));
}
}
diff --git a/src/System.Memory/src/System/SpanHelpers.char.cs b/src/System.Memory/src/System/SpanHelpers.char.cs
index 7df75aa1a05d..e33f2a849db1 100644
--- a/src/System.Memory/src/System/SpanHelpers.char.cs
+++ b/src/System.Memory/src/System/SpanHelpers.char.cs
@@ -3,6 +3,7 @@
// See the LICENSE file in the project root for more information.
using System.Diagnostics;
+using System.Numerics;
using System.Runtime.CompilerServices;
namespace System
@@ -95,82 +96,78 @@ public static int IndexOf(ref char searchSpace, char value, int length)
return -1;
}
- public static bool SequenceEqual(ref char first, ref char second, int length)
+ public unsafe static bool SequenceEqual(ref char first, ref char second, int length)
{
Debug.Assert(length >= 0);
- if (Unsafe.AreSame(ref first, ref second))
- return true;
+ var isMatch = true;
- int index = 0;
- while (length >= 8)
+ if (length == 0)
{
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
-
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
-
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
-
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
-
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
-
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
-
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
-
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
-
-
- length -= 8;
+ goto exit;
}
- while (length >= 4)
- {
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
-
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
+ length *= 2;
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
-
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
-
- length -= 4;
- }
-
- while (length > 0)
+ fixed (char* pFirst = &first)
+ fixed (char* pSecond = &second)
{
- if (Unsafe.Add(ref first, index) != Unsafe.Add(ref second, index))
- return false;
- index++;
- length--;
+ var a = (byte*)pFirst;
+ var b = (byte*)pSecond;
+
+ if (a == b)
+ {
+ goto exitFixed;
+ }
+
+ var i = 0;
+ if (Vector.IsHardwareAccelerated)
+ {
+ while (length - Vector.Count >= i)
+ {
+ var v0 = Unsafe.Read>(a + i);
+ var v1 = Unsafe.Read>(b + i);
+ i += Vector.Count;
+
+ if (!v0.Equals(v1))
+ {
+ isMatch = false;
+ goto exitFixed;
+ }
+ }
+ }
+
+ while (length - sizeof(long) >= i)
+ {
+ if (*(long*)(a + i) != *(long*)(b + i))
+ {
+ isMatch = false;
+ goto exitFixed;
+ }
+
+ i += sizeof(long);
+ }
+
+ if (length - sizeof(int) >= i)
+ {
+ if (*(int*)(a + i) != *(int*)(b + i))
+ {
+ isMatch = false;
+ goto exitFixed;
+ }
+
+ i += sizeof(int);
+ }
+
+ if (length > i && *(short*)(a + i) != *(short*)(b + i))
+ {
+ isMatch = false;
+ }
+ // Don't goto out of fixed block
+ exitFixed:;
}
-
- return true;
+ exit:
+ return isMatch;
}
}
}
diff --git a/src/System.Memory/src/System/SpanHelpers.cs b/src/System.Memory/src/System/SpanHelpers.cs
index 02a60a1a2b51..f67556732024 100644
--- a/src/System.Memory/src/System/SpanHelpers.cs
+++ b/src/System.Memory/src/System/SpanHelpers.cs
@@ -4,7 +4,6 @@
using System.Reflection;
using System.Diagnostics;
-using System.Runtime.InteropServices;
using System.Runtime.CompilerServices;
namespace System
@@ -50,6 +49,8 @@ public static IntPtr Add(this IntPtr start, int index)
///
public static bool IsReferenceOrContainsReferences() => PerTypeValues.IsReferenceOrContainsReferences;
+ public static bool IsValueType() => PerTypeValues.IsValueType;
+
private static bool IsReferenceOrContainsReferencesCore(Type type)
{
if (type.GetTypeInfo().IsPrimitive) // This is hopefully the common case. All types that return true for this are value types w/out embedded references.
@@ -84,6 +85,8 @@ public static class PerTypeValues
//
public static readonly bool IsReferenceOrContainsReferences = IsReferenceOrContainsReferencesCore(typeof(T));
+ public static readonly bool IsValueType = typeof(T).GetTypeInfo().IsValueType;
+
public static readonly T[] EmptyArray = new T[0];
public static readonly IntPtr ArrayAdjustment = MeasureArrayAdjustment();
diff --git a/src/System.Memory/src/System/ThrowHelper.cs b/src/System.Memory/src/System/ThrowHelper.cs
index d09882926ac3..0083c2212130 100644
--- a/src/System.Memory/src/System/ThrowHelper.cs
+++ b/src/System.Memory/src/System/ThrowHelper.cs
@@ -23,10 +23,46 @@ namespace System
internal static class ThrowHelper
{
- internal static void ThrowArgumentNullException(ExceptionArgument argument) { throw CreateArgumentNullException(argument); }
+ internal static void ThrowArgumentNullException_Array() { throw CreateArgumentNullException(ExceptionArgument.array); }
+ internal static void ThrowArgumentNullException_Text() { throw CreateArgumentNullException(ExceptionArgument.text); }
[MethodImpl(MethodImplOptions.NoInlining)]
private static Exception CreateArgumentNullException(ExceptionArgument argument) { return new ArgumentNullException(argument.ToString()); }
+ [MethodImpl(MethodImplOptions.NoInlining)]
+ private static Exception CreateArgumentNullOrOutofRangeException(int length) {
+ if (length < 0)
+ {
+ return CreateArgumentOutOfRangeException(ExceptionArgument.length);
+ }
+
+ return CreateArgumentNullException(ExceptionArgument.obj);
+ }
+ internal static void ThrowArgumentNullOrOutOfRangeException(int length) { throw CreateArgumentNullOrOutofRangeException(length); }
+
+ [MethodImpl(MethodImplOptions.NoInlining)]
+ private static Exception CreateArgumentNullOrOutofRangeException(Array array)
+ {
+ if (array == null)
+ {
+ return CreateArgumentNullException(ExceptionArgument.array);
+ }
+
+ return CreateArgumentOutOfRangeException(ExceptionArgument.start);
+ }
+ internal static void ThrowArgumentNullOrOutOfRangeException(Array array) { throw CreateArgumentNullOrOutofRangeException(array); }
+
+ [MethodImpl(MethodImplOptions.NoInlining)]
+ private static Exception CreateArgumentNullOrOutofRangeException(string text)
+ {
+ if (text == null)
+ {
+ return CreateArgumentNullException(ExceptionArgument.text);
+ }
+
+ return CreateArgumentOutOfRangeException(ExceptionArgument.start);
+ }
+ internal static void ThrowArgumentNullOrOutOfRangeException(string text) { throw CreateArgumentNullOrOutofRangeException(text); }
+
internal static void ThrowArrayTypeMismatchException_ArrayTypeMustBeExactMatch(Type type) { throw CreateArrayTypeMismatchException_ArrayTypeMustBeExactMatch(type); }
[MethodImpl(MethodImplOptions.NoInlining)]
private static Exception CreateArrayTypeMismatchException_ArrayTypeMustBeExactMatch(Type type) { return new ArrayTypeMismatchException(SR.Format(SR.ArrayTypeMustBeExactMatch, type)); }
@@ -43,7 +79,8 @@ internal static class ThrowHelper
[MethodImpl(MethodImplOptions.NoInlining)]
private static Exception CreateIndexOutOfRangeException() { return new IndexOutOfRangeException(); }
- internal static void ThrowArgumentOutOfRangeException(ExceptionArgument argument) { throw CreateArgumentOutOfRangeException(argument); }
+ internal static void ThrowArgumentOutOfRangeException_Start() { throw CreateArgumentOutOfRangeException(ExceptionArgument.start); }
+ internal static void ThrowArgumentOutOfRangeException_Length() { throw CreateArgumentOutOfRangeException(ExceptionArgument.length); }
[MethodImpl(MethodImplOptions.NoInlining)]
private static Exception CreateArgumentOutOfRangeException(ExceptionArgument argument) { return new ArgumentOutOfRangeException(argument.ToString()); }
}