Skip to content

Vectorize BitIncrement and BitDecrement for float/double in TensorPrimitives #123610

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stephentoub merged 15 commits into from
Jan 27, 2026
+312 −8
Merged

Vectorize BitIncrement and BitDecrement for float/double in TensorPrimitives #123610

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
e8ae0e1
Initial plan
Copilot Jan 26, 2026
93c545f
Implement vectorization for BitIncrement and BitDecrement operations
Copilot Jan 26, 2026
7b31447
Fix sign bit detection and NaN masking in BitIncrement and BitDecrement
Copilot Jan 26, 2026
97ccc34
Refactor: Consolidate variables and move declarations closer to usage…
Copilot Jan 26, 2026
40e6702
Add Half type support via TryUnaryInvokeHalfAsInt16
Copilot Jan 26, 2026
1d4810c
Change else if to if and add blank lines for readability
Copilot Jan 26, 2026
72a7b3c
Use Vector.IsNegative() API instead of manual sign bit checking
Copilot Jan 26, 2026
bbe89c3
Remove Half type support due to performance regression
Copilot Jan 26, 2026
0522459
Optimize special case handling with single ConditionalSelect per @tan…
Copilot Jan 26, 2026
1c6bb24
Fix IsZero to operate on float/double vectors instead of uint/ulong
Copilot Jan 26, 2026
a1f645e
Remove redundant infinity handlers - already handled by general case
Copilot Jan 26, 2026
3ec0153
Re-add Half type support via TryUnaryInvokeHalfAsInt16
Copilot Jan 26, 2026
4b3b884
Optimize IsZero check to use bitwise comparison for positive zero only
Copilot Jan 26, 2026
3df9c30
Merge branch 'main' into copilot/vectorize-bit-operations
stephentoub Jan 26, 2026
394d589
Remove Half type support due to performance regression
Copilot Jan 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
160 changes: 156 additions & 4 deletions ...tem.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.BitDecrement.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System.Runtime.CompilerServices;
using System.Runtime.Intrinsics;

namespace System.Numerics.Tensors
Expand All @@ -26,11 +27,162 @@ public static void BitDecrement<T>(ReadOnlySpan<T> x, Span<T> destination)
private readonly struct BitDecrementOperator<T> : IUnaryOperator<T, T>
where T : IFloatingPointIeee754<T>
{
public static bool Vectorizable => false; // TODO: Vectorize
public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double);

public static T Invoke(T x) => T.BitDecrement(x);
public static Vector128<T> Invoke(Vector128<T> x) => throw new NotSupportedException();
public static Vector256<T> Invoke(Vector256<T> x) => throw new NotSupportedException();
public static Vector512<T> Invoke(Vector512<T> x) => throw new NotSupportedException();

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<T> Invoke(Vector128<T> x)
{
if (typeof(T) == typeof(float))
{
Vector128<float> xFloat = x.AsSingle();
Vector128<uint> bits = xFloat.AsUInt32();

// General case: negative -> increment, positive -> decrement
Vector128<uint> result = Vector128.ConditionalSelect(
Vector128.IsNegative(xFloat).AsUInt32(),
bits + Vector128<uint>.One,
bits - Vector128<uint>.One);

// Handle special cases with a single conditional select
Vector128<uint> isPositiveZero = Vector128.IsZero(xFloat).AsUInt32();
Vector128<uint> specialValue = Vector128.Create(BitConverter.SingleToUInt32Bits(-float.Epsilon)) & isPositiveZero;

Vector128<uint> isNaNOrNegInf = (Vector128.IsNaN(xFloat) | Vector128.IsNegativeInfinity(xFloat)).AsUInt32();
specialValue |= bits & isNaNOrNegInf;

Vector128<uint> specialMask = isPositiveZero | isNaNOrNegInf;
return Vector128.ConditionalSelect(specialMask, specialValue, result).AsSingle().As<float, T>();
}

if (typeof(T) == typeof(double))
{
Vector128<double> xDouble = x.AsDouble();
Vector128<ulong> bits = xDouble.AsUInt64();

// General case: negative -> increment, positive -> decrement
Vector128<ulong> result = Vector128.ConditionalSelect(
Vector128.IsNegative(xDouble).AsUInt64(),
bits + Vector128<ulong>.One,
bits - Vector128<ulong>.One);

// Handle special cases with a single conditional select
Vector128<ulong> isPositiveZero = Vector128.IsZero(xDouble).AsUInt64();
Vector128<ulong> specialValue = Vector128.Create(BitConverter.DoubleToUInt64Bits(-double.Epsilon)) & isPositiveZero;

Vector128<ulong> isNaNOrNegInf = (Vector128.IsNaN(xDouble) | Vector128.IsNegativeInfinity(xDouble)).AsUInt64();
specialValue |= bits & isNaNOrNegInf;

Vector128<ulong> specialMask = isPositiveZero | isNaNOrNegInf;
return Vector128.ConditionalSelect(specialMask, specialValue, result).AsDouble().As<double, T>();
}

// Fallback for unsupported types - should not be reached since Vectorizable returns false
throw new NotSupportedException();
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<T> Invoke(Vector256<T> x)
{
if (typeof(T) == typeof(float))
{
Vector256<float> xFloat = x.AsSingle();
Vector256<uint> bits = xFloat.AsUInt32();

// General case: negative -> increment, positive -> decrement
Vector256<uint> result = Vector256.ConditionalSelect(
Vector256.IsNegative(xFloat).AsUInt32(),
bits + Vector256<uint>.One,
bits - Vector256<uint>.One);

// Handle special cases with a single conditional select
Vector256<uint> isPositiveZero = Vector256.IsZero(xFloat).AsUInt32();
Vector256<uint> specialValue = Vector256.Create(BitConverter.SingleToUInt32Bits(-float.Epsilon)) & isPositiveZero;

Vector256<uint> isNaNOrNegInf = (Vector256.IsNaN(xFloat) | Vector256.IsNegativeInfinity(xFloat)).AsUInt32();
specialValue |= bits & isNaNOrNegInf;

Vector256<uint> specialMask = isPositiveZero | isNaNOrNegInf;
return Vector256.ConditionalSelect(specialMask, specialValue, result).AsSingle().As<float, T>();
}

if (typeof(T) == typeof(double))
{
Vector256<double> xDouble = x.AsDouble();
Vector256<ulong> bits = xDouble.AsUInt64();

// General case: negative -> increment, positive -> decrement
Vector256<ulong> result = Vector256.ConditionalSelect(
Vector256.IsNegative(xDouble).AsUInt64(),
bits + Vector256<ulong>.One,
bits - Vector256<ulong>.One);

// Handle special cases with a single conditional select
Vector256<ulong> isPositiveZero = Vector256.IsZero(xDouble).AsUInt64();
Vector256<ulong> specialValue = Vector256.Create(BitConverter.DoubleToUInt64Bits(-double.Epsilon)) & isPositiveZero;

Vector256<ulong> isNaNOrNegInf = (Vector256.IsNaN(xDouble) | Vector256.IsNegativeInfinity(xDouble)).AsUInt64();
specialValue |= bits & isNaNOrNegInf;

Vector256<ulong> specialMask = isPositiveZero | isNaNOrNegInf;
return Vector256.ConditionalSelect(specialMask, specialValue, result).AsDouble().As<double, T>();
}

// Fallback for unsupported types - should not be reached since Vectorizable returns false
throw new NotSupportedException();
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector512<T> Invoke(Vector512<T> x)
{
if (typeof(T) == typeof(float))
{
Vector512<float> xFloat = x.AsSingle();
Vector512<uint> bits = xFloat.AsUInt32();

// General case: negative -> increment, positive -> decrement
Vector512<uint> result = Vector512.ConditionalSelect(
Vector512.IsNegative(xFloat).AsUInt32(),
bits + Vector512<uint>.One,
bits - Vector512<uint>.One);

// Handle special cases with a single conditional select
Vector512<uint> isPositiveZero = Vector512.IsZero(xFloat).AsUInt32();
Vector512<uint> specialValue = Vector512.Create(BitConverter.SingleToUInt32Bits(-float.Epsilon)) & isPositiveZero;

Vector512<uint> isNaNOrNegInf = (Vector512.IsNaN(xFloat) | Vector512.IsNegativeInfinity(xFloat)).AsUInt32();
specialValue |= bits & isNaNOrNegInf;

Vector512<uint> specialMask = isPositiveZero | isNaNOrNegInf;
return Vector512.ConditionalSelect(specialMask, specialValue, result).AsSingle().As<float, T>();
}

if (typeof(T) == typeof(double))
{
Vector512<double> xDouble = x.AsDouble();
Vector512<ulong> bits = xDouble.AsUInt64();

// General case: negative -> increment, positive -> decrement
Vector512<ulong> result = Vector512.ConditionalSelect(
Vector512.IsNegative(xDouble).AsUInt64(),
bits + Vector512<ulong>.One,
bits - Vector512<ulong>.One);

// Handle special cases with a single conditional select
Vector512<ulong> isPositiveZero = Vector512.IsZero(xDouble).AsUInt64();
Vector512<ulong> specialValue = Vector512.Create(BitConverter.DoubleToUInt64Bits(-double.Epsilon)) & isPositiveZero;

Vector512<ulong> isNaNOrNegInf = (Vector512.IsNaN(xDouble) | Vector512.IsNegativeInfinity(xDouble)).AsUInt64();
specialValue |= bits & isNaNOrNegInf;

Vector512<ulong> specialMask = isPositiveZero | isNaNOrNegInf;
return Vector512.ConditionalSelect(specialMask, specialValue, result).AsDouble().As<double, T>();
}

// Fallback for unsupported types - should not be reached since Vectorizable returns false
throw new NotSupportedException();
}
}
}
}
160 changes: 156 additions & 4 deletions ...tem.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.BitIncrement.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System.Runtime.CompilerServices;
using System.Runtime.Intrinsics;

namespace System.Numerics.Tensors
Expand All @@ -26,11 +27,162 @@ public static void BitIncrement<T>(ReadOnlySpan<T> x, Span<T> destination)
private readonly struct BitIncrementOperator<T> : IUnaryOperator<T, T>
where T : IFloatingPointIeee754<T>
{
public static bool Vectorizable => false; // TODO: Vectorize
public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double);

public static T Invoke(T x) => T.BitIncrement(x);
public static Vector128<T> Invoke(Vector128<T> x) => throw new NotSupportedException();
public static Vector256<T> Invoke(Vector256<T> x) => throw new NotSupportedException();
public static Vector512<T> Invoke(Vector512<T> x) => throw new NotSupportedException();

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<T> Invoke(Vector128<T> x)
{
if (typeof(T) == typeof(float))
{
Vector128<float> xFloat = x.AsSingle();
Vector128<uint> bits = xFloat.AsUInt32();

// General case: negative -> decrement, positive -> increment
Vector128<uint> result = Vector128.ConditionalSelect(
Vector128.IsNegative(xFloat).AsUInt32(),
bits - Vector128<uint>.One,
bits + Vector128<uint>.One);

// Handle special cases with a single conditional select
Vector128<uint> isNegativeZero = Vector128.Equals(bits, Vector128.Create(0x8000_0000u));
Vector128<uint> specialValue = Vector128.Create(0x0000_0001u) & isNegativeZero;

Vector128<uint> isNaNOrPosInf = (Vector128.IsNaN(xFloat) | Vector128.IsPositiveInfinity(xFloat)).AsUInt32();
specialValue |= bits & isNaNOrPosInf;

Vector128<uint> specialMask = isNegativeZero | isNaNOrPosInf;
return Vector128.ConditionalSelect(specialMask, specialValue, result).AsSingle().As<float, T>();
}

if (typeof(T) == typeof(double))
{
Vector128<double> xDouble = x.AsDouble();
Vector128<ulong> bits = xDouble.AsUInt64();

// General case: negative -> decrement, positive -> increment
Vector128<ulong> result = Vector128.ConditionalSelect(
Vector128.IsNegative(xDouble).AsUInt64(),
bits - Vector128<ulong>.One,
bits + Vector128<ulong>.One);

// Handle special cases with a single conditional select
Vector128<ulong> isNegativeZero = Vector128.Equals(bits, Vector128.Create(0x8000_0000_0000_0000ul));
Vector128<ulong> specialValue = Vector128.Create(0x0000_0000_0000_0001ul) & isNegativeZero;

Vector128<ulong> isNaNOrPosInf = (Vector128.IsNaN(xDouble) | Vector128.IsPositiveInfinity(xDouble)).AsUInt64();
specialValue |= bits & isNaNOrPosInf;

Vector128<ulong> specialMask = isNegativeZero | isNaNOrPosInf;
return Vector128.ConditionalSelect(specialMask, specialValue, result).AsDouble().As<double, T>();
}

// Fallback for unsupported types - should not be reached since Vectorizable returns false
throw new NotSupportedException();
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<T> Invoke(Vector256<T> x)
{
if (typeof(T) == typeof(float))
{
Vector256<float> xFloat = x.AsSingle();
Vector256<uint> bits = xFloat.AsUInt32();

// General case: negative -> decrement, positive -> increment
Vector256<uint> result = Vector256.ConditionalSelect(
Vector256.IsNegative(xFloat).AsUInt32(),
bits - Vector256<uint>.One,
bits + Vector256<uint>.One);

// Handle special cases with a single conditional select
Vector256<uint> isNegativeZero = Vector256.Equals(bits, Vector256.Create(0x8000_0000u));
Vector256<uint> specialValue = Vector256.Create(0x0000_0001u) & isNegativeZero;

Vector256<uint> isNaNOrPosInf = (Vector256.IsNaN(xFloat) | Vector256.IsPositiveInfinity(xFloat)).AsUInt32();
specialValue |= bits & isNaNOrPosInf;

Vector256<uint> specialMask = isNegativeZero | isNaNOrPosInf;
return Vector256.ConditionalSelect(specialMask, specialValue, result).AsSingle().As<float, T>();
}

if (typeof(T) == typeof(double))
{
Vector256<double> xDouble = x.AsDouble();
Vector256<ulong> bits = xDouble.AsUInt64();

// General case: negative -> decrement, positive -> increment
Vector256<ulong> result = Vector256.ConditionalSelect(
Vector256.IsNegative(xDouble).AsUInt64(),
bits - Vector256<ulong>.One,
bits + Vector256<ulong>.One);

// Handle special cases with a single conditional select
Vector256<ulong> isNegativeZero = Vector256.Equals(bits, Vector256.Create(0x8000_0000_0000_0000ul));
Vector256<ulong> specialValue = Vector256.Create(0x0000_0000_0000_0001ul) & isNegativeZero;

Vector256<ulong> isNaNOrPosInf = (Vector256.IsNaN(xDouble) | Vector256.IsPositiveInfinity(xDouble)).AsUInt64();
specialValue |= bits & isNaNOrPosInf;

Vector256<ulong> specialMask = isNegativeZero | isNaNOrPosInf;
return Vector256.ConditionalSelect(specialMask, specialValue, result).AsDouble().As<double, T>();
}

// Fallback for unsupported types - should not be reached since Vectorizable returns false
throw new NotSupportedException();
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector512<T> Invoke(Vector512<T> x)
{
if (typeof(T) == typeof(float))
{
Vector512<float> xFloat = x.AsSingle();
Vector512<uint> bits = xFloat.AsUInt32();

// General case: negative -> decrement, positive -> increment
Vector512<uint> result = Vector512.ConditionalSelect(
Vector512.IsNegative(xFloat).AsUInt32(),
bits - Vector512<uint>.One,
bits + Vector512<uint>.One);

// Handle special cases with a single conditional select
Vector512<uint> isNegativeZero = Vector512.Equals(bits, Vector512.Create(0x8000_0000u));
Vector512<uint> specialValue = Vector512.Create(0x0000_0001u) & isNegativeZero;

Vector512<uint> isNaNOrPosInf = (Vector512.IsNaN(xFloat) | Vector512.IsPositiveInfinity(xFloat)).AsUInt32();
specialValue |= bits & isNaNOrPosInf;

Vector512<uint> specialMask = isNegativeZero | isNaNOrPosInf;
return Vector512.ConditionalSelect(specialMask, specialValue, result).AsSingle().As<float, T>();
}

if (typeof(T) == typeof(double))
{
Vector512<double> xDouble = x.AsDouble();
Vector512<ulong> bits = xDouble.AsUInt64();

// General case: negative -> decrement, positive -> increment
Vector512<ulong> result = Vector512.ConditionalSelect(
Vector512.IsNegative(xDouble).AsUInt64(),
bits - Vector512<ulong>.One,
bits + Vector512<ulong>.One);

// Handle special cases with a single conditional select
Vector512<ulong> isNegativeZero = Vector512.Equals(bits, Vector512.Create(0x8000_0000_0000_0000ul));
Vector512<ulong> specialValue = Vector512.Create(0x0000_0000_0000_0001ul) & isNegativeZero;

Vector512<ulong> isNaNOrPosInf = (Vector512.IsNaN(xDouble) | Vector512.IsPositiveInfinity(xDouble)).AsUInt64();
specialValue |= bits & isNaNOrPosInf;

Vector512<ulong> specialMask = isNegativeZero | isNaNOrPosInf;
return Vector512.ConditionalSelect(specialMask, specialValue, result).AsDouble().As<double, T>();
}

// Fallback for unsupported types - should not be reached since Vectorizable returns false
throw new NotSupportedException();
}
}
}
}
Loading