diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMax.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMax.cs index f40f7e1e2e2ba0..d3114e7becd199 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMax.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMax.cs @@ -176,7 +176,7 @@ static Vector512 CreateVector512T(int i) => nanMask = ~Vector512.Equals(result, result); if (nanMask != Vector512.Zero) { - return IndexOfFirstMatch(nanMask); + return Vector512.IndexOfWhereAllBitsSet(nanMask); } } @@ -195,7 +195,7 @@ static Vector512 CreateVector512T(int i) => nanMask = ~Vector512.Equals(current, current); if (nanMask != Vector512.Zero) { - return i + IndexOfFirstMatch(nanMask); + return i + Vector512.IndexOfWhereAllBitsSet(nanMask); } } @@ -215,7 +215,7 @@ static Vector512 CreateVector512T(int i) => nanMask = ~Vector512.Equals(current, current); if (nanMask != Vector512.Zero) { - int indexInVectorOfFirstMatch = IndexOfFirstMatch(nanMask); + int indexInVectorOfFirstMatch = Vector512.IndexOfWhereAllBitsSet(nanMask); return typeof(T) == typeof(double) ? (int)(long)(object)currentIndex.As()[indexInVectorOfFirstMatch] : (int)(object)currentIndex.As()[indexInVectorOfFirstMatch]; @@ -260,7 +260,7 @@ static Vector256 CreateVector256T(int i) => nanMask = ~Vector256.Equals(result, result); if (nanMask != Vector256.Zero) { - return IndexOfFirstMatch(nanMask); + return Vector256.IndexOfWhereAllBitsSet(nanMask); } } @@ -279,7 +279,7 @@ static Vector256 CreateVector256T(int i) => nanMask = ~Vector256.Equals(current, current); if (nanMask != Vector256.Zero) { - return i + IndexOfFirstMatch(nanMask); + return i + Vector256.IndexOfWhereAllBitsSet(nanMask); } } @@ -299,7 +299,7 @@ static Vector256 CreateVector256T(int i) => nanMask = ~Vector256.Equals(current, current); if (nanMask != Vector256.Zero) { - int indexInVectorOfFirstMatch = IndexOfFirstMatch(nanMask); + int indexInVectorOfFirstMatch = Vector256.IndexOfWhereAllBitsSet(nanMask); return typeof(T) == typeof(double) ? (int)(long)(object)currentIndex.As()[indexInVectorOfFirstMatch] : (int)(object)currentIndex.As()[indexInVectorOfFirstMatch]; @@ -344,7 +344,7 @@ static Vector128 CreateVector128T(int i) => nanMask = ~Vector128.Equals(result, result); if (nanMask != Vector128.Zero) { - return IndexOfFirstMatch(nanMask); + return Vector128.IndexOfWhereAllBitsSet(nanMask); } } @@ -363,7 +363,7 @@ static Vector128 CreateVector128T(int i) => nanMask = ~Vector128.Equals(current, current); if (nanMask != Vector128.Zero) { - return i + IndexOfFirstMatch(nanMask); + return i + Vector128.IndexOfWhereAllBitsSet(nanMask); } } @@ -383,7 +383,7 @@ static Vector128 CreateVector128T(int i) => nanMask = ~Vector128.Equals(current, current); if (nanMask != Vector128.Zero) { - int indexInVectorOfFirstMatch = IndexOfFirstMatch(nanMask); + int indexInVectorOfFirstMatch = Vector128.IndexOfWhereAllBitsSet(nanMask); return typeof(T) == typeof(double) ? (int)(long)(object)currentIndex.As()[indexInVectorOfFirstMatch] : (int)(object)currentIndex.As()[indexInVectorOfFirstMatch]; @@ -419,15 +419,6 @@ static Vector128 CreateVector128T(int i) => return curIn; } - private static int IndexOfFirstMatch(Vector128 mask) => - BitOperations.TrailingZeroCount(mask.ExtractMostSignificantBits()); - - private static int IndexOfFirstMatch(Vector256 mask) => - BitOperations.TrailingZeroCount(mask.ExtractMostSignificantBits()); - - private static int IndexOfFirstMatch(Vector512 mask) => - BitOperations.TrailingZeroCount(mask.ExtractMostSignificantBits()); - [MethodImpl(MethodImplOptions.AggressiveInlining)] private static unsafe Vector256 IndexLessThan(Vector256 indices1, Vector256 indices2) => sizeof(T) == sizeof(long) ? Vector256.LessThan(indices1.AsInt64(), indices2.AsInt64()).As() : diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Max.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Max.cs index 8e42381c48e165..2f16e8b6b8d4f2 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Max.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Max.cs @@ -258,7 +258,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = IsNaN(result); if (nanMask != Vector512.Zero) { - return result.GetElement(IndexOfFirstMatch(nanMask)); + return result.GetElement(Vector512.IndexOfWhereAllBitsSet(nanMask)); } } @@ -277,7 +277,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = ~Vector512.Equals(current, current); if (nanMask != Vector512.Zero) { - return current.GetElement(IndexOfFirstMatch(nanMask)); + return current.GetElement(Vector512.IndexOfWhereAllBitsSet(nanMask)); } } @@ -296,7 +296,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = ~Vector512.Equals(current, current); if (nanMask != Vector512.Zero) { - return current.GetElement(IndexOfFirstMatch(nanMask)); + return current.GetElement(Vector512.IndexOfWhereAllBitsSet(nanMask)); } } @@ -323,7 +323,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = ~Vector256.Equals(result, result); if (nanMask != Vector256.Zero) { - return result.GetElement(IndexOfFirstMatch(nanMask)); + return result.GetElement(Vector256.IndexOfWhereAllBitsSet(nanMask)); } } @@ -342,7 +342,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = ~Vector256.Equals(current, current); if (nanMask != Vector256.Zero) { - return current.GetElement(IndexOfFirstMatch(nanMask)); + return current.GetElement(Vector256.IndexOfWhereAllBitsSet(nanMask)); } } @@ -362,7 +362,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = ~Vector256.Equals(current, current); if (nanMask != Vector256.Zero) { - return current.GetElement(IndexOfFirstMatch(nanMask)); + return current.GetElement(Vector256.IndexOfWhereAllBitsSet(nanMask)); } } @@ -389,7 +389,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = IsNaN(result); if (nanMask != Vector128.Zero) { - return result.GetElement(IndexOfFirstMatch(nanMask)); + return result.GetElement(Vector128.IndexOfWhereAllBitsSet(nanMask)); } } @@ -408,7 +408,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = IsNaN(current); if (nanMask != Vector128.Zero) { - return current.GetElement(IndexOfFirstMatch(nanMask)); + return current.GetElement(Vector128.IndexOfWhereAllBitsSet(nanMask)); } } @@ -427,7 +427,7 @@ private static T MinMaxCore(ReadOnlySpan x) nanMask = IsNaN(current); if (nanMask != Vector128.Zero) { - return current.GetElement(IndexOfFirstMatch(nanMask)); + return current.GetElement(Vector128.IndexOfWhereAllBitsSet(nanMask)); } } diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs index e7194f4098ed50..67cd528d198236 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs @@ -531,8 +531,8 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) Vector128 search = Vector128.Load(searchSpace + offset); // Same method as below - uint matches = Vector128.Equals(Vector128.Zero, search).ExtractMostSignificantBits(); - if (matches == 0) + Vector128 cmp = Vector128.Equals(Vector128.Zero, search); + if (cmp == Vector128.Zero) { // Zero flags set so no matches offset += (nuint)Vector128.Count; @@ -540,7 +540,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) else { // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector128.IndexOfFirstMatch(cmp)); } } @@ -553,8 +553,8 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) Vector256 search = Vector256.Load(searchSpace + offset); // Same method as below - uint matches = Vector256.Equals(Vector256.Zero, search).ExtractMostSignificantBits(); - if (matches == 0) + Vector256 cmp = Vector256.Equals(Vector256.Zero, search); + if (cmp == Vector256.Zero) { // Zero flags set so no matches offset += (nuint)Vector256.Count; @@ -562,7 +562,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) else { // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector256.IndexOfFirstMatch(cmp)); } } lengthToExamine = GetByteVector512SpanLength(offset, Length); @@ -571,10 +571,8 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) do { Vector512 search = Vector512.Load(searchSpace + offset); - ulong matches = Vector512.Equals(Vector512.Zero, search).ExtractMostSignificantBits(); - // Note that MoveMask has converted the equal vector elements into a set of bit flags, - // So the bit position in 'matches' corresponds to the element offset. - if (matches == 0) + Vector512 cmp = Vector512.Equals(Vector512.Zero, search); + if (cmp == Vector512.Zero) { // Zero flags set so no matches offset += (nuint)Vector512.Count; @@ -582,7 +580,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) } // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector512.IndexOfFirstMatch(cmp)); } while (lengthToExamine > offset); } @@ -592,8 +590,8 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) Vector256 search = Vector256.Load(searchSpace + offset); // Same method as above - uint matches = Vector256.Equals(Vector256.Zero, search).ExtractMostSignificantBits(); - if (matches == 0) + Vector256 cmp = Vector256.Equals(Vector256.Zero, search); + if (cmp == Vector256.Zero) { // Zero flags set so no matches offset += (nuint)Vector256.Count; @@ -601,7 +599,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) else { // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector256.IndexOfFirstMatch(cmp)); } } @@ -611,8 +609,8 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) Vector128 search = Vector128.Load(searchSpace + offset); // Same method as above - uint matches = Vector128.Equals(Vector128.Zero, search).ExtractMostSignificantBits(); - if (matches == 0) + Vector128 cmp = Vector128.Equals(Vector128.Zero, search); + if (cmp == Vector128.Zero) { // Zero flags set so no matches offset += (nuint)Vector128.Count; @@ -620,7 +618,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) else { // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector128.IndexOfFirstMatch(cmp)); } } @@ -644,8 +642,8 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) Vector128 search = Vector128.Load(searchSpace + offset); // Same method as below - uint matches = Vector128.Equals(Vector128.Zero, search).ExtractMostSignificantBits(); - if (matches == 0) + Vector128 cmp = Vector128.Equals(Vector128.Zero, search); + if (cmp == Vector128.Zero) { // Zero flags set so no matches offset += (nuint)Vector128.Count; @@ -653,7 +651,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) else { // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector128.IndexOfFirstMatch(cmp)); } } @@ -663,10 +661,8 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) do { Vector256 search = Vector256.Load(searchSpace + offset); - uint matches = Vector256.Equals(Vector256.Zero, search).ExtractMostSignificantBits(); - // Note that MoveMask has converted the equal vector elements into a set of bit flags, - // So the bit position in 'matches' corresponds to the element offset. - if (matches == 0) + Vector256 cmp = Vector256.Equals(Vector256.Zero, search); + if (cmp == Vector256.Zero) { // Zero flags set so no matches offset += (nuint)Vector256.Count; @@ -674,7 +670,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) } // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector256.IndexOfFirstMatch(cmp)); } while (lengthToExamine > offset); } @@ -684,8 +680,8 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) Vector128 search = Vector128.Load(searchSpace + offset); // Same method as above - uint matches = Vector128.Equals(Vector128.Zero, search).ExtractMostSignificantBits(); - if (matches == 0) + Vector128 cmp = Vector128.Equals(Vector128.Zero, search); + if (cmp == Vector128.Zero) { // Zero flags set so no matches offset += (nuint)Vector128.Count; @@ -693,7 +689,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) else { // Find bitflag offset of first match and add to current offset - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector128.IndexOfFirstMatch(cmp)); } } @@ -724,8 +720,7 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace) } // Find bitflag offset of first match and add to current offset - uint matches = compareResult.ExtractMostSignificantBits(); - return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches)); + return (int)(offset + (uint)Vector128.IndexOfFirstMatch(compareResult)); } if (offset < (nuint)(uint)Length) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs index 8e53a21801ec29..7f562e59d626aa 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs @@ -3713,49 +3713,37 @@ private static int LastIndexOfAnyValueType(ref TValue searchSp [MethodImpl(MethodImplOptions.AggressiveInlining)] private static unsafe int ComputeFirstIndex(ref T searchSpace, ref T current, Vector128 equals) where T : struct { - uint notEqualsElements = equals.ExtractMostSignificantBits(); - int index = BitOperations.TrailingZeroCount(notEqualsElements); - return index + (int)((nuint)Unsafe.ByteOffset(ref searchSpace, ref current) / (nuint)sizeof(T)); + return Vector128.IndexOfFirstMatch(equals) + (int)((nuint)Unsafe.ByteOffset(ref searchSpace, ref current) / (nuint)sizeof(T)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static unsafe int ComputeFirstIndex(ref T searchSpace, ref T current, Vector256 equals) where T : struct { - uint notEqualsElements = equals.ExtractMostSignificantBits(); - int index = BitOperations.TrailingZeroCount(notEqualsElements); - return index + (int)((nuint)Unsafe.ByteOffset(ref searchSpace, ref current) / (nuint)sizeof(T)); + return Vector256.IndexOfFirstMatch(equals) + (int)((nuint)Unsafe.ByteOffset(ref searchSpace, ref current) / (nuint)sizeof(T)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static unsafe int ComputeFirstIndex(ref T searchSpace, ref T current, Vector512 equals) where T : struct { - ulong notEqualsElements = equals.ExtractMostSignificantBits(); - int index = BitOperations.TrailingZeroCount(notEqualsElements); - return index + (int)((nuint)Unsafe.ByteOffset(ref searchSpace, ref current) / (nuint)sizeof(T)); + return Vector512.IndexOfFirstMatch(equals) + (int)((nuint)Unsafe.ByteOffset(ref searchSpace, ref current) / (nuint)sizeof(T)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int ComputeLastIndex(nint offset, Vector128 equals) where T : struct { - uint notEqualsElements = equals.ExtractMostSignificantBits(); - int index = 31 - BitOperations.LeadingZeroCount(notEqualsElements); // 31 = 32 (bits in Int32) - 1 (indexing from zero) - return (int)offset + index; + return (int)offset + Vector128.IndexOfLastMatch(equals); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int ComputeLastIndex(nint offset, Vector256 equals) where T : struct { - uint notEqualsElements = equals.ExtractMostSignificantBits(); - int index = 31 - BitOperations.LeadingZeroCount(notEqualsElements); // 31 = 32 (bits in Int32) - 1 (indexing from zero) - return (int)offset + index; + return (int)offset + Vector256.IndexOfLastMatch(equals); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int ComputeLastIndex(nint offset, Vector512 equals) where T : struct { - ulong notEqualsElements = equals.ExtractMostSignificantBits(); - int index = 63 - BitOperations.LeadingZeroCount(notEqualsElements); // 31 = 32 (bits in Int32) - 1 (indexing from zero) - return (int)offset + index; + return (int)offset + Vector512.IndexOfLastMatch(equals); } internal interface INegator where T : struct @@ -4176,7 +4164,7 @@ public static unsafe int CountValueType(ref T current, T value, int length) w ref T oneVectorAwayFromEnd = ref Unsafe.Subtract(ref end, Vector512.Count); while (Unsafe.IsAddressLessThan(ref current, ref oneVectorAwayFromEnd)) { - count += BitOperations.PopCount(Vector512.Equals(Vector512.LoadUnsafe(ref current), targetVector).ExtractMostSignificantBits()); + count += Vector512.CountMatches(Vector512.Equals(Vector512.LoadUnsafe(ref current), targetVector)); current = ref Unsafe.Add(ref current, Vector512.Count); } @@ -4191,7 +4179,7 @@ public static unsafe int CountValueType(ref T current, T value, int length) w ref T oneVectorAwayFromEnd = ref Unsafe.Subtract(ref end, Vector256.Count); while (Unsafe.IsAddressLessThan(ref current, ref oneVectorAwayFromEnd)) { - count += BitOperations.PopCount(Vector256.Equals(Vector256.LoadUnsafe(ref current), targetVector).ExtractMostSignificantBits()); + count += Vector256.CountMatches(Vector256.Equals(Vector256.LoadUnsafe(ref current), targetVector)); current = ref Unsafe.Add(ref current, Vector256.Count); } @@ -4206,7 +4194,7 @@ public static unsafe int CountValueType(ref T current, T value, int length) w ref T oneVectorAwayFromEnd = ref Unsafe.Subtract(ref end, Vector128.Count); while (Unsafe.IsAddressLessThan(ref current, ref oneVectorAwayFromEnd)) { - count += BitOperations.PopCount(Vector128.Equals(Vector128.LoadUnsafe(ref current), targetVector).ExtractMostSignificantBits()); + count += Vector128.CountMatches(Vector128.Equals(Vector128.LoadUnsafe(ref current), targetVector)); current = ref Unsafe.Add(ref current, Vector128.Count); }