diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.BitDecrement.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.BitDecrement.cs index b97336b0a2e3a5..2504c0375592be 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.BitDecrement.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.BitDecrement.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; namespace System.Numerics.Tensors @@ -26,11 +27,162 @@ public static void BitDecrement(ReadOnlySpan x, Span destination) private readonly struct BitDecrementOperator : IUnaryOperator where T : IFloatingPointIeee754 { - public static bool Vectorizable => false; // TODO: Vectorize + public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); + public static T Invoke(T x) => T.BitDecrement(x); - public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); - public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); - public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x) + { + if (typeof(T) == typeof(float)) + { + Vector128 xFloat = x.AsSingle(); + Vector128 bits = xFloat.AsUInt32(); + + // General case: negative -> increment, positive -> decrement + Vector128 result = Vector128.ConditionalSelect( + Vector128.IsNegative(xFloat).AsUInt32(), + bits + Vector128.One, + bits - Vector128.One); + + // Handle special cases with a single conditional select + Vector128 isPositiveZero = Vector128.IsZero(xFloat).AsUInt32(); + Vector128 specialValue = Vector128.Create(BitConverter.SingleToUInt32Bits(-float.Epsilon)) & isPositiveZero; + + Vector128 isNaNOrNegInf = (Vector128.IsNaN(xFloat) | Vector128.IsNegativeInfinity(xFloat)).AsUInt32(); + specialValue |= bits & isNaNOrNegInf; + + Vector128 specialMask = isPositiveZero | isNaNOrNegInf; + return Vector128.ConditionalSelect(specialMask, specialValue, result).AsSingle().As(); + } + + if (typeof(T) == typeof(double)) + { + Vector128 xDouble = x.AsDouble(); + Vector128 bits = xDouble.AsUInt64(); + + // General case: negative -> increment, positive -> decrement + Vector128 result = Vector128.ConditionalSelect( + Vector128.IsNegative(xDouble).AsUInt64(), + bits + Vector128.One, + bits - Vector128.One); + + // Handle special cases with a single conditional select + Vector128 isPositiveZero = Vector128.IsZero(xDouble).AsUInt64(); + Vector128 specialValue = Vector128.Create(BitConverter.DoubleToUInt64Bits(-double.Epsilon)) & isPositiveZero; + + Vector128 isNaNOrNegInf = (Vector128.IsNaN(xDouble) | Vector128.IsNegativeInfinity(xDouble)).AsUInt64(); + specialValue |= bits & isNaNOrNegInf; + + Vector128 specialMask = isPositiveZero | isNaNOrNegInf; + return Vector128.ConditionalSelect(specialMask, specialValue, result).AsDouble().As(); + } + + // Fallback for unsupported types - should not be reached since Vectorizable returns false + throw new NotSupportedException(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x) + { + if (typeof(T) == typeof(float)) + { + Vector256 xFloat = x.AsSingle(); + Vector256 bits = xFloat.AsUInt32(); + + // General case: negative -> increment, positive -> decrement + Vector256 result = Vector256.ConditionalSelect( + Vector256.IsNegative(xFloat).AsUInt32(), + bits + Vector256.One, + bits - Vector256.One); + + // Handle special cases with a single conditional select + Vector256 isPositiveZero = Vector256.IsZero(xFloat).AsUInt32(); + Vector256 specialValue = Vector256.Create(BitConverter.SingleToUInt32Bits(-float.Epsilon)) & isPositiveZero; + + Vector256 isNaNOrNegInf = (Vector256.IsNaN(xFloat) | Vector256.IsNegativeInfinity(xFloat)).AsUInt32(); + specialValue |= bits & isNaNOrNegInf; + + Vector256 specialMask = isPositiveZero | isNaNOrNegInf; + return Vector256.ConditionalSelect(specialMask, specialValue, result).AsSingle().As(); + } + + if (typeof(T) == typeof(double)) + { + Vector256 xDouble = x.AsDouble(); + Vector256 bits = xDouble.AsUInt64(); + + // General case: negative -> increment, positive -> decrement + Vector256 result = Vector256.ConditionalSelect( + Vector256.IsNegative(xDouble).AsUInt64(), + bits + Vector256.One, + bits - Vector256.One); + + // Handle special cases with a single conditional select + Vector256 isPositiveZero = Vector256.IsZero(xDouble).AsUInt64(); + Vector256 specialValue = Vector256.Create(BitConverter.DoubleToUInt64Bits(-double.Epsilon)) & isPositiveZero; + + Vector256 isNaNOrNegInf = (Vector256.IsNaN(xDouble) | Vector256.IsNegativeInfinity(xDouble)).AsUInt64(); + specialValue |= bits & isNaNOrNegInf; + + Vector256 specialMask = isPositiveZero | isNaNOrNegInf; + return Vector256.ConditionalSelect(specialMask, specialValue, result).AsDouble().As(); + } + + // Fallback for unsupported types - should not be reached since Vectorizable returns false + throw new NotSupportedException(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x) + { + if (typeof(T) == typeof(float)) + { + Vector512 xFloat = x.AsSingle(); + Vector512 bits = xFloat.AsUInt32(); + + // General case: negative -> increment, positive -> decrement + Vector512 result = Vector512.ConditionalSelect( + Vector512.IsNegative(xFloat).AsUInt32(), + bits + Vector512.One, + bits - Vector512.One); + + // Handle special cases with a single conditional select + Vector512 isPositiveZero = Vector512.IsZero(xFloat).AsUInt32(); + Vector512 specialValue = Vector512.Create(BitConverter.SingleToUInt32Bits(-float.Epsilon)) & isPositiveZero; + + Vector512 isNaNOrNegInf = (Vector512.IsNaN(xFloat) | Vector512.IsNegativeInfinity(xFloat)).AsUInt32(); + specialValue |= bits & isNaNOrNegInf; + + Vector512 specialMask = isPositiveZero | isNaNOrNegInf; + return Vector512.ConditionalSelect(specialMask, specialValue, result).AsSingle().As(); + } + + if (typeof(T) == typeof(double)) + { + Vector512 xDouble = x.AsDouble(); + Vector512 bits = xDouble.AsUInt64(); + + // General case: negative -> increment, positive -> decrement + Vector512 result = Vector512.ConditionalSelect( + Vector512.IsNegative(xDouble).AsUInt64(), + bits + Vector512.One, + bits - Vector512.One); + + // Handle special cases with a single conditional select + Vector512 isPositiveZero = Vector512.IsZero(xDouble).AsUInt64(); + Vector512 specialValue = Vector512.Create(BitConverter.DoubleToUInt64Bits(-double.Epsilon)) & isPositiveZero; + + Vector512 isNaNOrNegInf = (Vector512.IsNaN(xDouble) | Vector512.IsNegativeInfinity(xDouble)).AsUInt64(); + specialValue |= bits & isNaNOrNegInf; + + Vector512 specialMask = isPositiveZero | isNaNOrNegInf; + return Vector512.ConditionalSelect(specialMask, specialValue, result).AsDouble().As(); + } + + // Fallback for unsupported types - should not be reached since Vectorizable returns false + throw new NotSupportedException(); + } } } } diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.BitIncrement.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.BitIncrement.cs index 8af973d72b41b7..361af7af06576b 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.BitIncrement.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.BitIncrement.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; namespace System.Numerics.Tensors @@ -26,11 +27,162 @@ public static void BitIncrement(ReadOnlySpan x, Span destination) private readonly struct BitIncrementOperator : IUnaryOperator where T : IFloatingPointIeee754 { - public static bool Vectorizable => false; // TODO: Vectorize + public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); + public static T Invoke(T x) => T.BitIncrement(x); - public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); - public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); - public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x) + { + if (typeof(T) == typeof(float)) + { + Vector128 xFloat = x.AsSingle(); + Vector128 bits = xFloat.AsUInt32(); + + // General case: negative -> decrement, positive -> increment + Vector128 result = Vector128.ConditionalSelect( + Vector128.IsNegative(xFloat).AsUInt32(), + bits - Vector128.One, + bits + Vector128.One); + + // Handle special cases with a single conditional select + Vector128 isNegativeZero = Vector128.Equals(bits, Vector128.Create(0x8000_0000u)); + Vector128 specialValue = Vector128.Create(0x0000_0001u) & isNegativeZero; + + Vector128 isNaNOrPosInf = (Vector128.IsNaN(xFloat) | Vector128.IsPositiveInfinity(xFloat)).AsUInt32(); + specialValue |= bits & isNaNOrPosInf; + + Vector128 specialMask = isNegativeZero | isNaNOrPosInf; + return Vector128.ConditionalSelect(specialMask, specialValue, result).AsSingle().As(); + } + + if (typeof(T) == typeof(double)) + { + Vector128 xDouble = x.AsDouble(); + Vector128 bits = xDouble.AsUInt64(); + + // General case: negative -> decrement, positive -> increment + Vector128 result = Vector128.ConditionalSelect( + Vector128.IsNegative(xDouble).AsUInt64(), + bits - Vector128.One, + bits + Vector128.One); + + // Handle special cases with a single conditional select + Vector128 isNegativeZero = Vector128.Equals(bits, Vector128.Create(0x8000_0000_0000_0000ul)); + Vector128 specialValue = Vector128.Create(0x0000_0000_0000_0001ul) & isNegativeZero; + + Vector128 isNaNOrPosInf = (Vector128.IsNaN(xDouble) | Vector128.IsPositiveInfinity(xDouble)).AsUInt64(); + specialValue |= bits & isNaNOrPosInf; + + Vector128 specialMask = isNegativeZero | isNaNOrPosInf; + return Vector128.ConditionalSelect(specialMask, specialValue, result).AsDouble().As(); + } + + // Fallback for unsupported types - should not be reached since Vectorizable returns false + throw new NotSupportedException(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x) + { + if (typeof(T) == typeof(float)) + { + Vector256 xFloat = x.AsSingle(); + Vector256 bits = xFloat.AsUInt32(); + + // General case: negative -> decrement, positive -> increment + Vector256 result = Vector256.ConditionalSelect( + Vector256.IsNegative(xFloat).AsUInt32(), + bits - Vector256.One, + bits + Vector256.One); + + // Handle special cases with a single conditional select + Vector256 isNegativeZero = Vector256.Equals(bits, Vector256.Create(0x8000_0000u)); + Vector256 specialValue = Vector256.Create(0x0000_0001u) & isNegativeZero; + + Vector256 isNaNOrPosInf = (Vector256.IsNaN(xFloat) | Vector256.IsPositiveInfinity(xFloat)).AsUInt32(); + specialValue |= bits & isNaNOrPosInf; + + Vector256 specialMask = isNegativeZero | isNaNOrPosInf; + return Vector256.ConditionalSelect(specialMask, specialValue, result).AsSingle().As(); + } + + if (typeof(T) == typeof(double)) + { + Vector256 xDouble = x.AsDouble(); + Vector256 bits = xDouble.AsUInt64(); + + // General case: negative -> decrement, positive -> increment + Vector256 result = Vector256.ConditionalSelect( + Vector256.IsNegative(xDouble).AsUInt64(), + bits - Vector256.One, + bits + Vector256.One); + + // Handle special cases with a single conditional select + Vector256 isNegativeZero = Vector256.Equals(bits, Vector256.Create(0x8000_0000_0000_0000ul)); + Vector256 specialValue = Vector256.Create(0x0000_0000_0000_0001ul) & isNegativeZero; + + Vector256 isNaNOrPosInf = (Vector256.IsNaN(xDouble) | Vector256.IsPositiveInfinity(xDouble)).AsUInt64(); + specialValue |= bits & isNaNOrPosInf; + + Vector256 specialMask = isNegativeZero | isNaNOrPosInf; + return Vector256.ConditionalSelect(specialMask, specialValue, result).AsDouble().As(); + } + + // Fallback for unsupported types - should not be reached since Vectorizable returns false + throw new NotSupportedException(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x) + { + if (typeof(T) == typeof(float)) + { + Vector512 xFloat = x.AsSingle(); + Vector512 bits = xFloat.AsUInt32(); + + // General case: negative -> decrement, positive -> increment + Vector512 result = Vector512.ConditionalSelect( + Vector512.IsNegative(xFloat).AsUInt32(), + bits - Vector512.One, + bits + Vector512.One); + + // Handle special cases with a single conditional select + Vector512 isNegativeZero = Vector512.Equals(bits, Vector512.Create(0x8000_0000u)); + Vector512 specialValue = Vector512.Create(0x0000_0001u) & isNegativeZero; + + Vector512 isNaNOrPosInf = (Vector512.IsNaN(xFloat) | Vector512.IsPositiveInfinity(xFloat)).AsUInt32(); + specialValue |= bits & isNaNOrPosInf; + + Vector512 specialMask = isNegativeZero | isNaNOrPosInf; + return Vector512.ConditionalSelect(specialMask, specialValue, result).AsSingle().As(); + } + + if (typeof(T) == typeof(double)) + { + Vector512 xDouble = x.AsDouble(); + Vector512 bits = xDouble.AsUInt64(); + + // General case: negative -> decrement, positive -> increment + Vector512 result = Vector512.ConditionalSelect( + Vector512.IsNegative(xDouble).AsUInt64(), + bits - Vector512.One, + bits + Vector512.One); + + // Handle special cases with a single conditional select + Vector512 isNegativeZero = Vector512.Equals(bits, Vector512.Create(0x8000_0000_0000_0000ul)); + Vector512 specialValue = Vector512.Create(0x0000_0000_0000_0001ul) & isNegativeZero; + + Vector512 isNaNOrPosInf = (Vector512.IsNaN(xDouble) | Vector512.IsPositiveInfinity(xDouble)).AsUInt64(); + specialValue |= bits & isNaNOrPosInf; + + Vector512 specialMask = isNegativeZero | isNaNOrPosInf; + return Vector512.ConditionalSelect(specialMask, specialValue, result).AsDouble().As(); + } + + // Fallback for unsupported types - should not be reached since Vectorizable returns false + throw new NotSupportedException(); + } } } }