Skip to content

Vectorize TensorPrimitives.Cosh/Sinh/Tanh for double #97874

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stephentoub merged 5 commits into from
Feb 3, 2024
Merged

Vectorize TensorPrimitives.Cosh/Sinh/Tanh for double #97874

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7ede670
Vectorize TensorPrimitives.Cosh/Sinh/Tanh for double
stephentoub Feb 2, 2024
edd9d57
Merge branch 'main' into vectorizehyp
stephentoub Feb 3, 2024
87037f9
Address PR feedback
stephentoub Feb 3, 2024
c1b4272
Merge branch 'main' into vectorizehyp
stephentoub Feb 3, 2024
7ed8e3e
Separate float/double constants
stephentoub Feb 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
246 changes: 176 additions & 70 deletions ...s/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.netcore.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,6 @@

#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type

// TODO:
// - Vectorize remaining trig-related functions (some aren't vectorized at all, some are only vectorized for float).
// - Vectorize integer operations when sizeof(T) == 1 or 2 (currently only vectorized in most operations for sizeof(T) == 4 or 8).

namespace System.Numerics.Tensors
{
public static unsafe partial class TensorPrimitives
Expand Down Expand Up @@ -13064,42 +13060,79 @@ public static Vector512<T> Invoke(Vector512<T> x)
//
// coshf = v/2 * exp(x - log(v)) where v = 0x1.0000e8p-1

private const float LOGV = 0.693161f;
private const float HALFV = 1.0000138f;
private const float INVV2 = 0.24999309f;
private const float SINGLE_LOGV = 0.693161f;
private const float SINGLE_HALFV = 1.0000138f;
private const float SINGLE_INVV2 = 0.24999309f;

private const double DOUBLE_LOGV = 0.6931471805599453;
private const double DOUBLE_HALFV = 1.0;
private const double DOUBLE_INVV2 = 0.25;

public static bool Vectorizable => typeof(T) == typeof(float);
public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double);

public static T Invoke(T x) => T.Cosh(x);

public static Vector128<T> Invoke(Vector128<T> t)
{
Debug.Assert(typeof(T) == typeof(float));
Vector128<float> x = t.AsSingle();
if (typeof(T) == typeof(float))
{
Vector128<float> x = t.AsSingle();

Vector128<float> y = Vector128.Abs(x);
Vector128<float> z = ExpOperator<float>.Invoke(y - Vector128.Create((float)SINGLE_LOGV));
return (Vector128.Create((float)SINGLE_HALFV) * (z + (Vector128.Create((float)SINGLE_INVV2) / z))).As<float, T>();
}
else
{
Debug.Assert(typeof(T) == typeof(double));
Vector128<double> x = t.AsDouble();

Vector128<float> y = Vector128.Abs(x);
Vector128<float> z = ExpOperator<float>.Invoke(y - Vector128.Create(LOGV));
return (Vector128.Create(HALFV) * (z + (Vector128.Create(INVV2) / z))).As<float, T>();
Vector128<double> y = Vector128.Abs(x);
Vector128<double> z = ExpOperator<double>.Invoke(y - Vector128.Create(DOUBLE_LOGV));
return (Vector128.Create(DOUBLE_HALFV) * (z + (Vector128.Create(DOUBLE_INVV2) / z))).As<double, T>();
}
}

public static Vector256<T> Invoke(Vector256<T> t)
{
Debug.Assert(typeof(T) == typeof(float));
Vector256<float> x = t.AsSingle();
if (typeof(T) == typeof(float))
{
Vector256<float> x = t.AsSingle();

Vector256<float> y = Vector256.Abs(x);
Vector256<float> z = ExpOperator<float>.Invoke(y - Vector256.Create(LOGV));
return (Vector256.Create(HALFV) * (z + (Vector256.Create(INVV2) / z))).As<float, T>();
Vector256<float> y = Vector256.Abs(x);
Vector256<float> z = ExpOperator<float>.Invoke(y - Vector256.Create((float)SINGLE_LOGV));
return (Vector256.Create((float)SINGLE_HALFV) * (z + (Vector256.Create((float)SINGLE_INVV2) / z))).As<float, T>();
}
else
{
Debug.Assert(typeof(T) == typeof(double));
Vector256<double> x = t.AsDouble();

Vector256<double> y = Vector256.Abs(x);
Vector256<double> z = ExpOperator<double>.Invoke(y - Vector256.Create(DOUBLE_LOGV));
return (Vector256.Create(DOUBLE_HALFV) * (z + (Vector256.Create(DOUBLE_INVV2) / z))).As<double, T>();
}
}

public static Vector512<T> Invoke(Vector512<T> t)
{
Debug.Assert(typeof(T) == typeof(float));
Vector512<float> x = t.AsSingle();
if (typeof(T) == typeof(float))
{
Vector512<float> x = t.AsSingle();

Vector512<float> y = Vector512.Abs(x);
Vector512<float> z = ExpOperator<float>.Invoke(y - Vector512.Create((float)SINGLE_LOGV));
return (Vector512.Create((float)SINGLE_HALFV) * (z + (Vector512.Create((float)SINGLE_INVV2) / z))).As<float, T>();
}
else
{
Debug.Assert(typeof(T) == typeof(double));
Vector512<double> x = t.AsDouble();

Vector512<float> y = Vector512.Abs(x);
Vector512<float> z = ExpOperator<float>.Invoke(y - Vector512.Create(LOGV));
return (Vector512.Create(HALFV) * (z + (Vector512.Create(INVV2) / z))).As<float, T>();
Vector512<double> y = Vector512.Abs(x);
Vector512<double> z = ExpOperator<double>.Invoke(y - Vector512.Create(DOUBLE_LOGV));
return (Vector512.Create(DOUBLE_HALFV) * (z + (Vector512.Create(DOUBLE_INVV2) / z))).As<double, T>();
}
}
}

Expand Down Expand Up @@ -13132,49 +13165,91 @@ public static Vector512<T> Invoke(Vector512<T> t)
// Same as cosh, but with `z -` rather than `z +`, and with the sign
// flipped on the result based on the sign of the input.

private const uint SIGN_MASK = 0x7FFFFFFF;
private const float LOGV = 0.693161f;
private const float HALFV = 1.0000138f;
private const float INVV2 = 0.24999309f;
private const float SINGLE_LOGV = 0.693161f;
private const float SINGLE_HALFV = 1.0000138f;
private const float SINGLE_INVV2 = 0.24999309f;

public static bool Vectorizable => typeof(T) == typeof(float);
private const double DOUBLE_LOGV = 0.6931471805599453;
private const double DOUBLE_HALFV = 1.0;
private const double DOUBLE_INVV2 = 0.25;

public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double);

public static T Invoke(T x) => T.Sinh(x);

public static Vector128<T> Invoke(Vector128<T> t)
{
Debug.Assert(typeof(T) == typeof(float));
Vector128<float> x = t.AsSingle();
if (typeof(T) == typeof(float))
{
Vector128<float> x = t.AsSingle();

Vector128<float> y = Vector128.Abs(x);
Vector128<float> z = ExpOperator<float>.Invoke(y - Vector128.Create(LOGV));
Vector128<float> result = Vector128.Create(HALFV) * (z - (Vector128.Create(INVV2) / z));
Vector128<uint> sign = x.AsUInt32() & Vector128.Create(~SIGN_MASK);
return (sign ^ result.AsUInt32()).AsSingle().As<float, T>();
Vector128<float> y = Vector128.Abs(x);
Vector128<float> z = ExpOperator<float>.Invoke(y - Vector128.Create((float)SINGLE_LOGV));
Vector128<float> result = Vector128.Create((float)SINGLE_HALFV) * (z - (Vector128.Create((float)SINGLE_INVV2) / z));
Vector128<uint> sign = x.AsUInt32() & Vector128.Create(~(uint)int.MaxValue);
return (sign ^ result.AsUInt32()).As<uint, T>();
}
else
{
Debug.Assert(typeof(T) == typeof(double));
Vector128<double> x = t.AsDouble();

Vector128<double> y = Vector128.Abs(x);
Vector128<double> z = ExpOperator<double>.Invoke(y - Vector128.Create(DOUBLE_LOGV));
Vector128<double> result = Vector128.Create(DOUBLE_HALFV) * (z - (Vector128.Create(DOUBLE_INVV2) / z));
Vector128<ulong> sign = x.AsUInt64() & Vector128.Create(~(ulong)long.MaxValue);
return (sign ^ result.AsUInt64()).As<ulong, T>();
}
}

public static Vector256<T> Invoke(Vector256<T> t)
{
Debug.Assert(typeof(T) == typeof(float));
Vector256<float> x = t.AsSingle();
if (typeof(T) == typeof(float))
{
Vector256<float> x = t.AsSingle();

Vector256<float> y = Vector256.Abs(x);
Vector256<float> z = ExpOperator<float>.Invoke(y - Vector256.Create((float)SINGLE_LOGV));
Vector256<float> result = Vector256.Create((float)SINGLE_HALFV) * (z - (Vector256.Create((float)SINGLE_INVV2) / z));
Vector256<uint> sign = x.AsUInt32() & Vector256.Create(~(uint)int.MaxValue);
return (sign ^ result.AsUInt32()).As<uint, T>();
}
else
{
Debug.Assert(typeof(T) == typeof(double));
Vector256<double> x = t.AsDouble();

Vector256<float> y = Vector256.Abs(x);
Vector256<float> z = ExpOperator<float>.Invoke(y - Vector256.Create(LOGV));
Vector256<float> result = Vector256.Create(HALFV) * (z - (Vector256.Create(INVV2) / z));
Vector256<uint> sign = x.AsUInt32() & Vector256.Create(~SIGN_MASK);
return (sign ^ result.AsUInt32()).AsSingle().As<float, T>();
Vector256<double> y = Vector256.Abs(x);
Vector256<double> z = ExpOperator<double>.Invoke(y - Vector256.Create(DOUBLE_LOGV));
Vector256<double> result = Vector256.Create(DOUBLE_HALFV) * (z - (Vector256.Create(DOUBLE_INVV2) / z));
Vector256<ulong> sign = x.AsUInt64() & Vector256.Create(~(ulong)long.MaxValue);
return (sign ^ result.AsUInt64()).As<ulong, T>();
}
}

public static Vector512<T> Invoke(Vector512<T> t)
{
Debug.Assert(typeof(T) == typeof(float));
Vector512<float> x = t.AsSingle();
if (typeof(T) == typeof(float))
{
Vector512<float> x = t.AsSingle();

Vector512<float> y = Vector512.Abs(x);
Vector512<float> z = ExpOperator<float>.Invoke(y - Vector512.Create(LOGV));
Vector512<float> result = Vector512.Create(HALFV) * (z - (Vector512.Create(INVV2) / z));
Vector512<uint> sign = x.AsUInt32() & Vector512.Create(~SIGN_MASK);
return (sign ^ result.AsUInt32()).AsSingle().As<float, T>();
Vector512<float> y = Vector512.Abs(x);
Vector512<float> z = ExpOperator<float>.Invoke(y - Vector512.Create((float)SINGLE_LOGV));
Vector512<float> result = Vector512.Create((float)SINGLE_HALFV) * (z - (Vector512.Create((float)SINGLE_INVV2) / z));
Vector512<uint> sign = x.AsUInt32() & Vector512.Create(~(uint)int.MaxValue);
return (sign ^ result.AsUInt32()).As<uint, T>();
}
else
{
Debug.Assert(typeof(T) == typeof(double));
Vector512<double> x = t.AsDouble();

Vector512<double> y = Vector512.Abs(x);
Vector512<double> z = ExpOperator<double>.Invoke(y - Vector512.Create(DOUBLE_LOGV));
Vector512<double> result = Vector512.Create(DOUBLE_HALFV) * (z - (Vector512.Create(DOUBLE_INVV2) / z));
Vector512<ulong> sign = x.AsUInt64() & Vector512.Create(~(ulong)long.MaxValue);
return (sign ^ result.AsUInt64()).As<ulong, T>();
}
}
}

Expand Down Expand Up @@ -13223,43 +13298,74 @@ public static Vector512<T> Invoke(Vector512<T> t)
// If x < 0, then we use the identity
// tanhf(-x) = -tanhf(x)

private const uint SIGN_MASK = 0x7FFFFFFF;

public static bool Vectorizable => typeof(T) == typeof(float);
public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double);

public static T Invoke(T x) => T.Tanh(x);

public static Vector128<T> Invoke(Vector128<T> t)
{
Debug.Assert(typeof(T) == typeof(float));
Vector128<float> x = t.AsSingle();
if (typeof(T) == typeof(float))
{
Vector128<float> x = t.AsSingle();

Vector128<float> y = Vector128.Abs(x);
Vector128<float> z = ExpM1Operator<float>.Invoke(Vector128.Create(-2f) * y);
Vector128<uint> sign = x.AsUInt32() & Vector128.Create(~SIGN_MASK);
return (sign ^ (-z / (z + Vector128.Create(2f))).AsUInt32()).AsSingle().As<float, T>();
Vector128<float> y = Vector128.Abs(x);
Vector128<float> z = ExpM1Operator<float>.Invoke(Vector128.Create(-2f) * y);
Vector128<uint> sign = x.AsUInt32() & Vector128.Create(~(uint)int.MaxValue);
return (sign ^ (-z / (z + Vector128.Create(2f))).AsUInt32()).As<uint, T>();
}
else
{
Vector128<double> x = t.AsDouble();

Vector128<double> y = Vector128.Abs(x);
Vector128<double> z = ExpM1Operator<double>.Invoke(Vector128.Create(-2d) * y);
Vector128<ulong> sign = x.AsUInt64() & Vector128.Create(~(ulong)long.MaxValue);
return (sign ^ (-z / (z + Vector128.Create(2d))).AsUInt64()).As<ulong, T>();
}
}

public static Vector256<T> Invoke(Vector256<T> t)
{
Debug.Assert(typeof(T) == typeof(float));
Vector256<float> x = t.AsSingle();
if (typeof(T) == typeof(float))
{
Vector256<float> x = t.AsSingle();

Vector256<float> y = Vector256.Abs(x);
Vector256<float> z = ExpM1Operator<float>.Invoke(Vector256.Create(-2f) * y);
Vector256<uint> sign = x.AsUInt32() & Vector256.Create(~SIGN_MASK);
return (sign ^ (-z / (z + Vector256.Create(2f))).AsUInt32()).AsSingle().As<float, T>();
Vector256<float> y = Vector256.Abs(x);
Vector256<float> z = ExpM1Operator<float>.Invoke(Vector256.Create(-2f) * y);
Vector256<uint> sign = x.AsUInt32() & Vector256.Create(~(uint)int.MaxValue);
return (sign ^ (-z / (z + Vector256.Create(2f))).AsUInt32()).As<uint, T>();
}
else
{
Vector256<double> x = t.AsDouble();

Vector256<double> y = Vector256.Abs(x);
Vector256<double> z = ExpM1Operator<double>.Invoke(Vector256.Create(-2d) * y);
Vector256<ulong> sign = x.AsUInt64() & Vector256.Create(~(ulong)long.MaxValue);
return (sign ^ (-z / (z + Vector256.Create(2d))).AsUInt64()).As<ulong, T>();
}
}

public static Vector512<T> Invoke(Vector512<T> t)
{
Debug.Assert(typeof(T) == typeof(float));
Vector512<float> x = t.AsSingle();
if (typeof(T) == typeof(float))
{
Vector512<float> x = t.AsSingle();

Vector512<float> y = Vector512.Abs(x);
Vector512<float> z = ExpM1Operator<float>.Invoke(Vector512.Create(-2f) * y);
Vector512<uint> sign = x.AsUInt32() & Vector512.Create(~SIGN_MASK);
return (sign ^ (-z / (z + Vector512.Create(2f))).AsUInt32()).AsSingle().As<float, T>();
Vector512<float> y = Vector512.Abs(x);
Vector512<float> z = ExpM1Operator<float>.Invoke(Vector512.Create(-2f) * y);
Vector512<uint> sign = x.AsUInt32() & Vector512.Create(~(uint)int.MaxValue);
return (sign ^ (-z / (z + Vector512.Create(2f))).AsUInt32()).As<uint, T>();
}
else
{
Vector512<double> x = t.AsDouble();

Vector512<double> y = Vector512.Abs(x);
Vector512<double> z = ExpM1Operator<double>.Invoke(Vector512.Create(-2d) * y);
Vector512<ulong> sign = x.AsUInt64() & Vector512.Create(~(ulong)long.MaxValue);
return (sign ^ (-z / (z + Vector512.Create(2d))).AsUInt64()).As<ulong, T>();
}
}
}

Expand Down