diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.netcore.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.netcore.cs index ea2b365760085a..2a903a4ca32824 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.netcore.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.netcore.cs @@ -10,10 +10,6 @@ #pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type -// TODO: -// - Vectorize remaining trig-related functions (some aren't vectorized at all, some are only vectorized for float). -// - Vectorize integer operations when sizeof(T) == 1 or 2 (currently only vectorized in most operations for sizeof(T) == 4 or 8). - namespace System.Numerics.Tensors { public static unsafe partial class TensorPrimitives @@ -13064,42 +13060,79 @@ public static Vector512 Invoke(Vector512 x) // // coshf = v/2 * exp(x - log(v)) where v = 0x1.0000e8p-1 - private const float LOGV = 0.693161f; - private const float HALFV = 1.0000138f; - private const float INVV2 = 0.24999309f; + private const float SINGLE_LOGV = 0.693161f; + private const float SINGLE_HALFV = 1.0000138f; + private const float SINGLE_INVV2 = 0.24999309f; + + private const double DOUBLE_LOGV = 0.6931471805599453; + private const double DOUBLE_HALFV = 1.0; + private const double DOUBLE_INVV2 = 0.25; - public static bool Vectorizable => typeof(T) == typeof(float); + public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); public static T Invoke(T x) => T.Cosh(x); public static Vector128 Invoke(Vector128 t) { - Debug.Assert(typeof(T) == typeof(float)); - Vector128 x = t.AsSingle(); + if (typeof(T) == typeof(float)) + { + Vector128 x = t.AsSingle(); + + Vector128 y = Vector128.Abs(x); + Vector128 z = ExpOperator.Invoke(y - Vector128.Create((float)SINGLE_LOGV)); + return (Vector128.Create((float)SINGLE_HALFV) * (z + (Vector128.Create((float)SINGLE_INVV2) / z))).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + Vector128 x = t.AsDouble(); - Vector128 y = Vector128.Abs(x); - Vector128 z = ExpOperator.Invoke(y - Vector128.Create(LOGV)); - return (Vector128.Create(HALFV) * (z + (Vector128.Create(INVV2) / z))).As(); + Vector128 y = Vector128.Abs(x); + Vector128 z = ExpOperator.Invoke(y - Vector128.Create(DOUBLE_LOGV)); + return (Vector128.Create(DOUBLE_HALFV) * (z + (Vector128.Create(DOUBLE_INVV2) / z))).As(); + } } public static Vector256 Invoke(Vector256 t) { - Debug.Assert(typeof(T) == typeof(float)); - Vector256 x = t.AsSingle(); + if (typeof(T) == typeof(float)) + { + Vector256 x = t.AsSingle(); - Vector256 y = Vector256.Abs(x); - Vector256 z = ExpOperator.Invoke(y - Vector256.Create(LOGV)); - return (Vector256.Create(HALFV) * (z + (Vector256.Create(INVV2) / z))).As(); + Vector256 y = Vector256.Abs(x); + Vector256 z = ExpOperator.Invoke(y - Vector256.Create((float)SINGLE_LOGV)); + return (Vector256.Create((float)SINGLE_HALFV) * (z + (Vector256.Create((float)SINGLE_INVV2) / z))).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + Vector256 x = t.AsDouble(); + + Vector256 y = Vector256.Abs(x); + Vector256 z = ExpOperator.Invoke(y - Vector256.Create(DOUBLE_LOGV)); + return (Vector256.Create(DOUBLE_HALFV) * (z + (Vector256.Create(DOUBLE_INVV2) / z))).As(); + } } public static Vector512 Invoke(Vector512 t) { - Debug.Assert(typeof(T) == typeof(float)); - Vector512 x = t.AsSingle(); + if (typeof(T) == typeof(float)) + { + Vector512 x = t.AsSingle(); + + Vector512 y = Vector512.Abs(x); + Vector512 z = ExpOperator.Invoke(y - Vector512.Create((float)SINGLE_LOGV)); + return (Vector512.Create((float)SINGLE_HALFV) * (z + (Vector512.Create((float)SINGLE_INVV2) / z))).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + Vector512 x = t.AsDouble(); - Vector512 y = Vector512.Abs(x); - Vector512 z = ExpOperator.Invoke(y - Vector512.Create(LOGV)); - return (Vector512.Create(HALFV) * (z + (Vector512.Create(INVV2) / z))).As(); + Vector512 y = Vector512.Abs(x); + Vector512 z = ExpOperator.Invoke(y - Vector512.Create(DOUBLE_LOGV)); + return (Vector512.Create(DOUBLE_HALFV) * (z + (Vector512.Create(DOUBLE_INVV2) / z))).As(); + } } } @@ -13132,49 +13165,91 @@ public static Vector512 Invoke(Vector512 t) // Same as cosh, but with `z -` rather than `z +`, and with the sign // flipped on the result based on the sign of the input. - private const uint SIGN_MASK = 0x7FFFFFFF; - private const float LOGV = 0.693161f; - private const float HALFV = 1.0000138f; - private const float INVV2 = 0.24999309f; + private const float SINGLE_LOGV = 0.693161f; + private const float SINGLE_HALFV = 1.0000138f; + private const float SINGLE_INVV2 = 0.24999309f; - public static bool Vectorizable => typeof(T) == typeof(float); + private const double DOUBLE_LOGV = 0.6931471805599453; + private const double DOUBLE_HALFV = 1.0; + private const double DOUBLE_INVV2 = 0.25; + + public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); public static T Invoke(T x) => T.Sinh(x); public static Vector128 Invoke(Vector128 t) { - Debug.Assert(typeof(T) == typeof(float)); - Vector128 x = t.AsSingle(); + if (typeof(T) == typeof(float)) + { + Vector128 x = t.AsSingle(); - Vector128 y = Vector128.Abs(x); - Vector128 z = ExpOperator.Invoke(y - Vector128.Create(LOGV)); - Vector128 result = Vector128.Create(HALFV) * (z - (Vector128.Create(INVV2) / z)); - Vector128 sign = x.AsUInt32() & Vector128.Create(~SIGN_MASK); - return (sign ^ result.AsUInt32()).AsSingle().As(); + Vector128 y = Vector128.Abs(x); + Vector128 z = ExpOperator.Invoke(y - Vector128.Create((float)SINGLE_LOGV)); + Vector128 result = Vector128.Create((float)SINGLE_HALFV) * (z - (Vector128.Create((float)SINGLE_INVV2) / z)); + Vector128 sign = x.AsUInt32() & Vector128.Create(~(uint)int.MaxValue); + return (sign ^ result.AsUInt32()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + Vector128 x = t.AsDouble(); + + Vector128 y = Vector128.Abs(x); + Vector128 z = ExpOperator.Invoke(y - Vector128.Create(DOUBLE_LOGV)); + Vector128 result = Vector128.Create(DOUBLE_HALFV) * (z - (Vector128.Create(DOUBLE_INVV2) / z)); + Vector128 sign = x.AsUInt64() & Vector128.Create(~(ulong)long.MaxValue); + return (sign ^ result.AsUInt64()).As(); + } } public static Vector256 Invoke(Vector256 t) { - Debug.Assert(typeof(T) == typeof(float)); - Vector256 x = t.AsSingle(); + if (typeof(T) == typeof(float)) + { + Vector256 x = t.AsSingle(); + + Vector256 y = Vector256.Abs(x); + Vector256 z = ExpOperator.Invoke(y - Vector256.Create((float)SINGLE_LOGV)); + Vector256 result = Vector256.Create((float)SINGLE_HALFV) * (z - (Vector256.Create((float)SINGLE_INVV2) / z)); + Vector256 sign = x.AsUInt32() & Vector256.Create(~(uint)int.MaxValue); + return (sign ^ result.AsUInt32()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + Vector256 x = t.AsDouble(); - Vector256 y = Vector256.Abs(x); - Vector256 z = ExpOperator.Invoke(y - Vector256.Create(LOGV)); - Vector256 result = Vector256.Create(HALFV) * (z - (Vector256.Create(INVV2) / z)); - Vector256 sign = x.AsUInt32() & Vector256.Create(~SIGN_MASK); - return (sign ^ result.AsUInt32()).AsSingle().As(); + Vector256 y = Vector256.Abs(x); + Vector256 z = ExpOperator.Invoke(y - Vector256.Create(DOUBLE_LOGV)); + Vector256 result = Vector256.Create(DOUBLE_HALFV) * (z - (Vector256.Create(DOUBLE_INVV2) / z)); + Vector256 sign = x.AsUInt64() & Vector256.Create(~(ulong)long.MaxValue); + return (sign ^ result.AsUInt64()).As(); + } } public static Vector512 Invoke(Vector512 t) { - Debug.Assert(typeof(T) == typeof(float)); - Vector512 x = t.AsSingle(); + if (typeof(T) == typeof(float)) + { + Vector512 x = t.AsSingle(); - Vector512 y = Vector512.Abs(x); - Vector512 z = ExpOperator.Invoke(y - Vector512.Create(LOGV)); - Vector512 result = Vector512.Create(HALFV) * (z - (Vector512.Create(INVV2) / z)); - Vector512 sign = x.AsUInt32() & Vector512.Create(~SIGN_MASK); - return (sign ^ result.AsUInt32()).AsSingle().As(); + Vector512 y = Vector512.Abs(x); + Vector512 z = ExpOperator.Invoke(y - Vector512.Create((float)SINGLE_LOGV)); + Vector512 result = Vector512.Create((float)SINGLE_HALFV) * (z - (Vector512.Create((float)SINGLE_INVV2) / z)); + Vector512 sign = x.AsUInt32() & Vector512.Create(~(uint)int.MaxValue); + return (sign ^ result.AsUInt32()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + Vector512 x = t.AsDouble(); + + Vector512 y = Vector512.Abs(x); + Vector512 z = ExpOperator.Invoke(y - Vector512.Create(DOUBLE_LOGV)); + Vector512 result = Vector512.Create(DOUBLE_HALFV) * (z - (Vector512.Create(DOUBLE_INVV2) / z)); + Vector512 sign = x.AsUInt64() & Vector512.Create(~(ulong)long.MaxValue); + return (sign ^ result.AsUInt64()).As(); + } } } @@ -13223,43 +13298,74 @@ public static Vector512 Invoke(Vector512 t) // If x < 0, then we use the identity // tanhf(-x) = -tanhf(x) - private const uint SIGN_MASK = 0x7FFFFFFF; - - public static bool Vectorizable => typeof(T) == typeof(float); + public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); public static T Invoke(T x) => T.Tanh(x); public static Vector128 Invoke(Vector128 t) { - Debug.Assert(typeof(T) == typeof(float)); - Vector128 x = t.AsSingle(); + if (typeof(T) == typeof(float)) + { + Vector128 x = t.AsSingle(); - Vector128 y = Vector128.Abs(x); - Vector128 z = ExpM1Operator.Invoke(Vector128.Create(-2f) * y); - Vector128 sign = x.AsUInt32() & Vector128.Create(~SIGN_MASK); - return (sign ^ (-z / (z + Vector128.Create(2f))).AsUInt32()).AsSingle().As(); + Vector128 y = Vector128.Abs(x); + Vector128 z = ExpM1Operator.Invoke(Vector128.Create(-2f) * y); + Vector128 sign = x.AsUInt32() & Vector128.Create(~(uint)int.MaxValue); + return (sign ^ (-z / (z + Vector128.Create(2f))).AsUInt32()).As(); + } + else + { + Vector128 x = t.AsDouble(); + + Vector128 y = Vector128.Abs(x); + Vector128 z = ExpM1Operator.Invoke(Vector128.Create(-2d) * y); + Vector128 sign = x.AsUInt64() & Vector128.Create(~(ulong)long.MaxValue); + return (sign ^ (-z / (z + Vector128.Create(2d))).AsUInt64()).As(); + } } public static Vector256 Invoke(Vector256 t) { - Debug.Assert(typeof(T) == typeof(float)); - Vector256 x = t.AsSingle(); + if (typeof(T) == typeof(float)) + { + Vector256 x = t.AsSingle(); - Vector256 y = Vector256.Abs(x); - Vector256 z = ExpM1Operator.Invoke(Vector256.Create(-2f) * y); - Vector256 sign = x.AsUInt32() & Vector256.Create(~SIGN_MASK); - return (sign ^ (-z / (z + Vector256.Create(2f))).AsUInt32()).AsSingle().As(); + Vector256 y = Vector256.Abs(x); + Vector256 z = ExpM1Operator.Invoke(Vector256.Create(-2f) * y); + Vector256 sign = x.AsUInt32() & Vector256.Create(~(uint)int.MaxValue); + return (sign ^ (-z / (z + Vector256.Create(2f))).AsUInt32()).As(); + } + else + { + Vector256 x = t.AsDouble(); + + Vector256 y = Vector256.Abs(x); + Vector256 z = ExpM1Operator.Invoke(Vector256.Create(-2d) * y); + Vector256 sign = x.AsUInt64() & Vector256.Create(~(ulong)long.MaxValue); + return (sign ^ (-z / (z + Vector256.Create(2d))).AsUInt64()).As(); + } } public static Vector512 Invoke(Vector512 t) { - Debug.Assert(typeof(T) == typeof(float)); - Vector512 x = t.AsSingle(); + if (typeof(T) == typeof(float)) + { + Vector512 x = t.AsSingle(); - Vector512 y = Vector512.Abs(x); - Vector512 z = ExpM1Operator.Invoke(Vector512.Create(-2f) * y); - Vector512 sign = x.AsUInt32() & Vector512.Create(~SIGN_MASK); - return (sign ^ (-z / (z + Vector512.Create(2f))).AsUInt32()).AsSingle().As(); + Vector512 y = Vector512.Abs(x); + Vector512 z = ExpM1Operator.Invoke(Vector512.Create(-2f) * y); + Vector512 sign = x.AsUInt32() & Vector512.Create(~(uint)int.MaxValue); + return (sign ^ (-z / (z + Vector512.Create(2f))).AsUInt32()).As(); + } + else + { + Vector512 x = t.AsDouble(); + + Vector512 y = Vector512.Abs(x); + Vector512 z = ExpM1Operator.Invoke(Vector512.Create(-2d) * y); + Vector512 sign = x.AsUInt64() & Vector512.Create(~(ulong)long.MaxValue); + return (sign ^ (-z / (z + Vector512.Create(2d))).AsUInt64()).As(); + } } }