Skip to content

Vectorize TensorPrimitives.BitIncrement/BitDecrement for Half using ushort operations #123754

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stephentoub merged 6 commits into from
Feb 6, 2026
+210 −2
Merged

Vectorize TensorPrimitives.BitIncrement/BitDecrement for Half using ushort operations #123754

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f19c6a3
Initial plan
Copilot Jan 29, 2026
00bc621
Optimize Half.BitIncrement and BitDecrement to use ushort operations
Copilot Jan 29, 2026
2ac7925
Add vectorized Half support to TensorPrimitives.BitIncrement/BitDecre…
Copilot Jan 29, 2026
a2d9b8e
Fix NaN handling in HalfBitIncrement/DecrementOperator
Copilot Jan 29, 2026
f5fca17
Merge branch 'main' into copilot/optimize-half-bit-increment-decrement
stephentoub Feb 5, 2026
4c2d6f9
Address Tanner's feedback: use Half.BitIncrement/Decrement, Vector.Is…
Copilot Feb 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 105 additions & 1 deletion ...tem.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.BitDecrement.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,15 @@ public static partial class TensorPrimitives
/// </para>
/// </remarks>
public static void BitDecrement<T>(ReadOnlySpan<T> x, Span<T> destination)
where T : IFloatingPointIeee754<T> =>
where T : IFloatingPointIeee754<T>
{
if (typeof(T) == typeof(Half) && TryUnaryBitwiseInvokeHalfAsInt16<T, HalfBitDecrementOperator>(x, destination))
{
return;
}

InvokeSpanIntoSpan<T, BitDecrementOperator<T>>(x, destination);
}

/// <summary>T.BitDecrement(x)</summary>
private readonly struct BitDecrementOperator<T> : IUnaryOperator<T, T>
Expand Down Expand Up @@ -184,5 +191,102 @@ public static Vector512<T> Invoke(Vector512<T> x)
throw new NotSupportedException();
}
}

/// <summary>Half.BitDecrement(x) operator for direct ushort manipulation.</summary>
private readonly struct HalfBitDecrementOperator : IUnaryOperator<short, short>
{
// Half constants
private const ushort PositiveInfinityBits = 0x7C00;
private const ushort NegativeInfinityBits = 0xFC00;
private const ushort NegativeEpsilonBits = 0x8001;

public static bool Vectorizable => true;

public static short Invoke(short x) => BitConverter.HalfToInt16Bits(Half.BitDecrement(BitConverter.Int16BitsToHalf(x)));

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<short> Invoke(Vector128<short> x)
{
Vector128<ushort> bits = x.AsUInt16();

// General case: negative -> increment, positive -> decrement
Vector128<ushort> isNegative = Vector128.IsNegative(x).AsUInt16();
Vector128<ushort> result = Vector128.ConditionalSelect(
isNegative,
bits + Vector128<ushort>.One,
bits - Vector128<ushort>.One);

// Handle special cases with a single conditional select
Vector128<ushort> isPositiveZero = Vector128.IsZero(bits);
Vector128<ushort> specialValue = Vector128.Create(NegativeEpsilonBits) & isPositiveZero;

// NaN: (bits & 0x7FFF) > 0x7C00 (both positive and negative NaN)
// -Infinity: bits == 0xFC00
Vector128<ushort> absValue = bits & Vector128.Create((ushort)0x7FFF);
Vector128<ushort> isNaN = Vector128.GreaterThan(absValue, Vector128.Create(PositiveInfinityBits));
Vector128<ushort> isNegInf = Vector128.Equals(bits, Vector128.Create(NegativeInfinityBits));
Vector128<ushort> isNaNOrNegInf = isNaN | isNegInf;
specialValue |= bits & isNaNOrNegInf;

Vector128<ushort> specialMask = isPositiveZero | isNaNOrNegInf;
return Vector128.ConditionalSelect(specialMask, specialValue, result).AsInt16();
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<short> Invoke(Vector256<short> x)
{
Vector256<ushort> bits = x.AsUInt16();

// General case: negative -> increment, positive -> decrement
Vector256<ushort> isNegative = Vector256.IsNegative(x).AsUInt16();
Vector256<ushort> result = Vector256.ConditionalSelect(
isNegative,
bits + Vector256<ushort>.One,
bits - Vector256<ushort>.One);

// Handle special cases with a single conditional select
Vector256<ushort> isPositiveZero = Vector256.IsZero(bits);
Vector256<ushort> specialValue = Vector256.Create(NegativeEpsilonBits) & isPositiveZero;

// NaN: (bits & 0x7FFF) > 0x7C00 (both positive and negative NaN)
// -Infinity: bits == 0xFC00
Vector256<ushort> absValue = bits & Vector256.Create((ushort)0x7FFF);
Vector256<ushort> isNaN = Vector256.GreaterThan(absValue, Vector256.Create(PositiveInfinityBits));
Vector256<ushort> isNegInf = Vector256.Equals(bits, Vector256.Create(NegativeInfinityBits));
Vector256<ushort> isNaNOrNegInf = isNaN | isNegInf;
specialValue |= bits & isNaNOrNegInf;

Vector256<ushort> specialMask = isPositiveZero | isNaNOrNegInf;
return Vector256.ConditionalSelect(specialMask, specialValue, result).AsInt16();
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector512<short> Invoke(Vector512<short> x)
{
Vector512<ushort> bits = x.AsUInt16();

// General case: negative -> increment, positive -> decrement
Vector512<ushort> isNegative = Vector512.IsNegative(x).AsUInt16();
Vector512<ushort> result = Vector512.ConditionalSelect(
isNegative,
bits + Vector512<ushort>.One,
bits - Vector512<ushort>.One);

// Handle special cases with a single conditional select
Vector512<ushort> isPositiveZero = Vector512.IsZero(bits);
Vector512<ushort> specialValue = Vector512.Create(NegativeEpsilonBits) & isPositiveZero;

// NaN: (bits & 0x7FFF) > 0x7C00 (both positive and negative NaN)
// -Infinity: bits == 0xFC00
Vector512<ushort> absValue = bits & Vector512.Create((ushort)0x7FFF);
Vector512<ushort> isNaN = Vector512.GreaterThan(absValue, Vector512.Create(PositiveInfinityBits));
Vector512<ushort> isNegInf = Vector512.Equals(bits, Vector512.Create(NegativeInfinityBits));
Vector512<ushort> isNaNOrNegInf = isNaN | isNegInf;
specialValue |= bits & isNaNOrNegInf;

Vector512<ushort> specialMask = isPositiveZero | isNaNOrNegInf;
return Vector512.ConditionalSelect(specialMask, specialValue, result).AsInt16();
}
}
}
}
106 changes: 105 additions & 1 deletion ...tem.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.BitIncrement.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,15 @@ public static partial class TensorPrimitives
/// </para>
/// </remarks>
public static void BitIncrement<T>(ReadOnlySpan<T> x, Span<T> destination)
where T : IFloatingPointIeee754<T> =>
where T : IFloatingPointIeee754<T>
{
if (typeof(T) == typeof(Half) && TryUnaryBitwiseInvokeHalfAsInt16<T, HalfBitIncrementOperator>(x, destination))
{
return;
}

InvokeSpanIntoSpan<T, BitIncrementOperator<T>>(x, destination);
}

/// <summary>T.BitIncrement(x)</summary>
private readonly struct BitIncrementOperator<T> : IUnaryOperator<T, T>
Expand Down Expand Up @@ -184,5 +191,102 @@ public static Vector512<T> Invoke(Vector512<T> x)
throw new NotSupportedException();
}
}

/// <summary>Half.BitIncrement(x) operator for direct ushort manipulation.</summary>
private readonly struct HalfBitIncrementOperator : IUnaryOperator<short, short>
{
// Half constants
private const ushort PositiveInfinityBits = 0x7C00;
private const ushort NegativeZeroBits = 0x8000;
private const ushort EpsilonBits = 0x0001;

public static bool Vectorizable => true;

public static short Invoke(short x) => BitConverter.HalfToInt16Bits(Half.BitIncrement(BitConverter.Int16BitsToHalf(x)));

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<short> Invoke(Vector128<short> x)
{
Vector128<ushort> bits = x.AsUInt16();

// General case: negative -> decrement, positive -> increment
Vector128<ushort> isNegative = Vector128.IsNegative(x).AsUInt16();
Vector128<ushort> result = Vector128.ConditionalSelect(
isNegative,
bits - Vector128<ushort>.One,
bits + Vector128<ushort>.One);

// Handle special cases with a single conditional select
Vector128<ushort> isNegativeZero = Vector128.Equals(bits, Vector128.Create(NegativeZeroBits));
Vector128<ushort> specialValue = Vector128.Create(EpsilonBits) & isNegativeZero;

// NaN: (bits & 0x7FFF) > 0x7C00 (both positive and negative NaN)
// +Infinity: bits == 0x7C00
Vector128<ushort> absValue = bits & Vector128.Create((ushort)0x7FFF);
Vector128<ushort> isNaN = Vector128.GreaterThan(absValue, Vector128.Create(PositiveInfinityBits));
Vector128<ushort> isPosInf = Vector128.Equals(bits, Vector128.Create(PositiveInfinityBits));
Vector128<ushort> isNaNOrPosInf = isNaN | isPosInf;
specialValue |= bits & isNaNOrPosInf;

Vector128<ushort> specialMask = isNegativeZero | isNaNOrPosInf;
return Vector128.ConditionalSelect(specialMask, specialValue, result).AsInt16();
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<short> Invoke(Vector256<short> x)
{
Vector256<ushort> bits = x.AsUInt16();

// General case: negative -> decrement, positive -> increment
Vector256<ushort> isNegative = Vector256.IsNegative(x).AsUInt16();
Vector256<ushort> result = Vector256.ConditionalSelect(
isNegative,
bits - Vector256<ushort>.One,
bits + Vector256<ushort>.One);

// Handle special cases with a single conditional select
Vector256<ushort> isNegativeZero = Vector256.Equals(bits, Vector256.Create(NegativeZeroBits));
Vector256<ushort> specialValue = Vector256.Create(EpsilonBits) & isNegativeZero;

// NaN: (bits & 0x7FFF) > 0x7C00 (both positive and negative NaN)
// +Infinity: bits == 0x7C00
Vector256<ushort> absValue = bits & Vector256.Create((ushort)0x7FFF);
Vector256<ushort> isNaN = Vector256.GreaterThan(absValue, Vector256.Create(PositiveInfinityBits));
Vector256<ushort> isPosInf = Vector256.Equals(bits, Vector256.Create(PositiveInfinityBits));
Vector256<ushort> isNaNOrPosInf = isNaN | isPosInf;
specialValue |= bits & isNaNOrPosInf;

Vector256<ushort> specialMask = isNegativeZero | isNaNOrPosInf;
return Vector256.ConditionalSelect(specialMask, specialValue, result).AsInt16();
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector512<short> Invoke(Vector512<short> x)
{
Vector512<ushort> bits = x.AsUInt16();

// General case: negative -> decrement, positive -> increment
Vector512<ushort> isNegative = Vector512.IsNegative(x).AsUInt16();
Vector512<ushort> result = Vector512.ConditionalSelect(
isNegative,
bits - Vector512<ushort>.One,
bits + Vector512<ushort>.One);

// Handle special cases with a single conditional select
Vector512<ushort> isNegativeZero = Vector512.Equals(bits, Vector512.Create(NegativeZeroBits));
Vector512<ushort> specialValue = Vector512.Create(EpsilonBits) & isNegativeZero;

// NaN: (bits & 0x7FFF) > 0x7C00 (both positive and negative NaN)
// +Infinity: bits == 0x7C00
Vector512<ushort> absValue = bits & Vector512.Create((ushort)0x7FFF);
Vector512<ushort> isNaN = Vector512.GreaterThan(absValue, Vector512.Create(PositiveInfinityBits));
Vector512<ushort> isPosInf = Vector512.Equals(bits, Vector512.Create(PositiveInfinityBits));
Vector512<ushort> isNaNOrPosInf = isNaN | isPosInf;
specialValue |= bits & isNaNOrPosInf;

Vector512<ushort> specialMask = isNegativeZero | isNaNOrPosInf;
return Vector512.ConditionalSelect(specialMask, specialValue, result).AsInt16();
}
}
}
}
Loading