diff --git a/src/libraries/Common/src/System/HexConverter.cs b/src/libraries/Common/src/System/HexConverter.cs index df2e4417b40c95..4494da4d265800 100644 --- a/src/libraries/Common/src/System/HexConverter.cs +++ b/src/libraries/Common/src/System/HexConverter.cs @@ -3,6 +3,13 @@ using System.Diagnostics; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; + +#if SYSTEM_PRIVATE_CORELIB +using Internal.Runtime.CompilerServices; +#endif namespace System { @@ -83,16 +90,85 @@ public static void ToCharsBuffer(byte value, Span buffer, int startingInde buffer[startingIndex] = (char)(packedResult >> 8); } - public static void EncodeToUtf16(ReadOnlySpan bytes, Span chars, Casing casing = Casing.Upper) + private static unsafe int ToCharsBufferAvx2(byte* bytes, int bytesCount, char* chars, Casing casing) { - Debug.Assert(chars.Length >= bytes.Length * 2); + Debug.Assert(Avx2.IsSupported); + Debug.Assert(bytesCount >= 32); + + Vector256 x00 = Vector256.Zero; + Vector256 x0F = Vector256.Create((byte)0x0F); + Vector256 hexLookupTable = casing == Casing.Lower ? + CreateVector(LowerHexLookupTable) : + CreateVector(UpperHexLookupTable); + + int bytesToRead = RoundDownToNext32(bytesCount); + byte* eof = bytes + bytesToRead; + byte* charsAsByte = (byte*)chars; + do + { + Vector256 value = Avx.LoadVector256(bytes); + bytes += 32; + + Vector256 hiShift = Avx2.ShiftRightLogical(value.AsInt16(), 4).AsByte(); + Vector256 loHalf = Avx2.And(value, x0F); + Vector256 hiHalf = Avx2.And(hiShift, x0F); + Vector256 lo02 = Avx2.UnpackLow(hiHalf, loHalf); + Vector256 hi13 = Avx2.UnpackHigh(hiHalf, loHalf); + + Vector256 resLo = Avx2.Shuffle(hexLookupTable, lo02); + Vector256 resHi = Avx2.Shuffle(hexLookupTable, hi13); + + Vector256 ae = Avx2.UnpackLow(resLo, x00); + Vector256 bf = Avx2.UnpackHigh(resLo, x00); + Vector256 cg = Avx2.UnpackLow(resHi, x00); + Vector256 dh = Avx2.UnpackHigh(resHi, x00); + + Vector256 ab = Avx2.Permute2x128(ae, bf, 0b0010_0000); + Vector256 ef = Avx2.Permute2x128(ae, bf, 0b0011_0001); + Vector256 cd = Avx2.Permute2x128(cg, dh, 0b0010_0000); + Vector256 gh = Avx2.Permute2x128(cg, dh, 0b0011_0001); + + Avx.Store(charsAsByte, ab); + charsAsByte += 32; + Avx.Store(charsAsByte, cd); + charsAsByte += 32; + Avx.Store(charsAsByte, ef); + charsAsByte += 32; + Avx.Store(charsAsByte, gh); + charsAsByte += 32; + } while (bytes != eof); + + return bytesToRead; + } - for (int pos = 0; pos < bytes.Length; ++pos) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe void EncodeToUtf16Core(byte* bytes, int bytesCount, Span chars, Casing casing) + { + int pos = 0; + if (Avx2.IsSupported && bytesCount >= 32) + { + Debug.Assert(!chars.IsEmpty); + fixed (char* charPtr = &MemoryMarshal.GetReference(chars)) + { + pos = ToCharsBufferAvx2(bytes, bytesCount, charPtr, casing); + } + Debug.Assert(pos == (bytesCount / 32) * 32); + } + for (; pos < bytesCount; ++pos) { ToCharsBuffer(bytes[pos], chars, pos * 2, casing); } } + public static unsafe void EncodeToUtf16(ReadOnlySpan bytes, Span chars, Casing casing = Casing.Upper) + { + Debug.Assert(chars.Length >= bytes.Length * 2); + fixed (byte* bytesPtr = bytes) + { + EncodeToUtf16Core(bytesPtr, bytes.Length, chars, casing); + } + } + #if ALLOW_PARTIALLY_TRUSTED_CALLERS [System.Security.SecuritySafeCriticalAttribute] #endif @@ -120,10 +196,9 @@ public static unsafe string ToString(ReadOnlySpan bytes, Casing casing = C #else fixed (byte* bytesPtr = bytes) { - return string.Create(bytes.Length * 2, (Ptr: (IntPtr)bytesPtr, bytes.Length, casing), static (chars, args) => + return string.Create(bytes.Length * 2, (Ptr: (IntPtr)bytesPtr, bytes.Length, casing), (chars, args) => { - var ros = new ReadOnlySpan((byte*)args.Ptr, args.Length); - EncodeToUtf16(ros, chars, args.casing); + EncodeToUtf16Core((byte*)args.Ptr, args.Length, chars, args.casing); }); } #endif @@ -162,13 +237,25 @@ public static bool TryDecodeFromUtf16(ReadOnlySpan chars, Span bytes return TryDecodeFromUtf16(chars, bytes, out _); } - public static bool TryDecodeFromUtf16(ReadOnlySpan chars, Span bytes, out int charsProcessed) + public static unsafe bool TryDecodeFromUtf16(ReadOnlySpan chars, Span bytes, out int charsProcessed) { Debug.Assert(chars.Length % 2 == 0, "Un-even number of characters provided"); Debug.Assert(chars.Length / 2 == bytes.Length, "Target buffer not right-sized for provided characters"); int i = 0; int j = 0; + if (Avx2.IsSupported && bytes.Length > 32) + { + fixed (char* charPtr = &MemoryMarshal.GetReference(chars)) + fixed (byte* bytePtr = &MemoryMarshal.GetReference(bytes)) + { + j = DecodeFromUtf16Avx2(charPtr, bytePtr, bytes.Length); + Debug.Assert(j % 32 == 0); + Debug.Assert(j <= bytes.Length); + i = j * 2; + } + } + int byteLo = 0; int byteHi = 0; while (j < bytes.Length) @@ -192,6 +279,92 @@ public static bool TryDecodeFromUtf16(ReadOnlySpan chars, Span bytes return (byteLo | byteHi) != 0xFF; } + private static unsafe int DecodeFromUtf16Avx2(char* chars, byte* bytes, int bytesCount) + { + Debug.Assert(Avx2.IsSupported); + + Vector256 x0F = Vector256.Create((byte) 0x0F); + Vector256 xF0 = Vector256.Create((byte) 0xF0); + Vector256 digHexSelector = CreateVector(UpperLowerDigHexSelector); + Vector256 digits = CreateVector(Digits); + Vector256 hexs = CreateVector(Hexs); + Vector256 evenBytes = CreateVector(EvenBytes); + Vector256 oddBytes = CreateVector(OddBytes); + + int bytesToWrite = RoundDownToNext32(bytesCount); + byte* eof = bytes + bytesToWrite; + byte* dest = bytes; + byte* charsAsByte = (byte*)chars; + int leftOk, rightOk; + while (dest != eof) + { + Vector256 a = Avx.LoadVector256(charsAsByte).AsInt16(); + charsAsByte += 32; + Vector256 b = Avx.LoadVector256(charsAsByte).AsInt16(); + charsAsByte += 32; + Vector256 c = Avx.LoadVector256(charsAsByte).AsInt16(); + charsAsByte += 32; + Vector256 d = Avx.LoadVector256(charsAsByte).AsInt16(); + charsAsByte += 32; + + Vector256 ab = Avx2.PackUnsignedSaturate(a, b); + Vector256 cd = Avx2.PackUnsignedSaturate(c, d); + + Vector256 inputLeft = Avx2.Permute4x64(ab.AsUInt64(), 0b11_01_10_00).AsByte(); + Vector256 inputRight = Avx2.Permute4x64(cd.AsUInt64(), 0b11_01_10_00).AsByte(); + + Vector256 loNibbleLeft = Avx2.And(inputLeft, x0F); + Vector256 loNibbleRight = Avx2.And(inputRight, x0F); + + Vector256 hiNibbleLeft = Avx2.And(inputLeft, xF0); + Vector256 hiNibbleRight = Avx2.And(inputRight, xF0); + + Vector256 leftDigits = Avx2.Shuffle(digits, loNibbleLeft); + Vector256 leftHex = Avx2.Shuffle(hexs, loNibbleLeft); + + Vector256 hiNibbleShLeft = Avx2.ShiftRightLogical(hiNibbleLeft.AsInt16(), 4).AsByte(); + Vector256 hiNibbleShRight = Avx2.ShiftRightLogical(hiNibbleRight.AsInt16(), 4).AsByte(); + + Vector256 rightDigits = Avx2.Shuffle(digits, loNibbleRight); + Vector256 rightHex = Avx2.Shuffle(hexs, loNibbleRight); + + Vector256 magicLeft = Avx2.Shuffle(digHexSelector, hiNibbleShLeft); + Vector256 magicRight = Avx2.Shuffle(digHexSelector, hiNibbleShRight); + + Vector256 valueLeft = Avx2.BlendVariable(leftDigits, leftHex, magicLeft); + Vector256 valueRight = Avx2.BlendVariable(rightDigits, rightHex, magicRight); + + Vector256 errLeft = Avx2.ShiftLeftLogical(magicLeft.AsInt16(), 7).AsByte(); + Vector256 errRight = Avx2.ShiftLeftLogical(magicRight.AsInt16(), 7).AsByte(); + + Vector256 evenBytesLeft = Avx2.Shuffle(valueLeft, evenBytes); + Vector256 oddBytesLeft = Avx2.Shuffle(valueLeft, oddBytes); + Vector256 evenBytesRight = Avx2.Shuffle(valueRight, evenBytes); + Vector256 oddBytesRight = Avx2.Shuffle(valueRight, oddBytes); + + evenBytesLeft = Avx2.ShiftLeftLogical(evenBytesLeft.AsUInt16(), 4).AsByte(); + evenBytesRight = Avx2.ShiftLeftLogical(evenBytesRight.AsUInt16(), 4).AsByte(); + + evenBytesLeft = Avx2.Or(evenBytesLeft, oddBytesLeft); + evenBytesRight = Avx2.Or(evenBytesRight, oddBytesRight); + + Vector256 result = Merge(evenBytesLeft, evenBytesRight); + + Vector256 validationResultLeft = Avx2.Or(errLeft, valueLeft); + Vector256 validationResultRight = Avx2.Or(errRight, valueRight); + + leftOk = Avx2.MoveMask(validationResultLeft); + rightOk = Avx2.MoveMask(validationResultRight); + + if ((leftOk | rightOk) != 0) break; + + Avx.Store(dest, result); + dest += 32; + } + + return bytesToWrite - (int) (eof - dest); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int FromChar(int c) { @@ -254,5 +427,63 @@ public static bool IsHexLowerChar(int c) 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 239 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF // 255 }; + + private static ReadOnlySpan LowerHexLookupTable => new byte[] + { + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66 + }; + + private static ReadOnlySpan UpperHexLookupTable => new byte[] + { + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46 + }; + + private static ReadOnlySpan UpperLowerDigHexSelector => new byte[] + { + 0x01, 0x01, 0x01, 0x00, 0x80, 0x01, 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x00, 0x80, 0x01, 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 + }; + + private static ReadOnlySpan Digits => new byte[] + { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF + }; + + private static ReadOnlySpan Hexs => new byte[] + { + 0xFF, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF + }; + + private static ReadOnlySpan EvenBytes => new byte[] + { + 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, + 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 + }; + + private static ReadOnlySpan OddBytes => new byte[] + { + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 + }; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int RoundDownToNext32(int x) + => x & 0x7FFFFFE0; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 CreateVector(ReadOnlySpan data) + => Unsafe.ReadUnaligned>(ref MemoryMarshal.GetReference(data)); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 Merge(Vector256 a, Vector256 b) + { + Vector256 a1 = Avx2.Permute4x64(a.AsUInt64(), 0b11_10_10_00); + Vector256 b1 = Avx2.Permute4x64(b.AsUInt64(), 0b11_00_01_00); + return Avx2.Blend(a1.AsUInt32(), b1.AsUInt32(), 0b1111_0000).AsByte(); + } } }