From 8b0b476fccc95cb13d0413d312e32960aa6ff986 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Wed, 26 Jun 2024 09:25:55 -0700 Subject: [PATCH] Update Mono to handle various other vector bitcast APIs --- src/mono/mono/mini/interp/interp-internals.h | 2 + .../mono/mini/interp/interp-simd-intrins.def | 8 + src/mono/mono/mini/interp/interp-simd.c | 82 ++++ src/mono/mono/mini/interp/simd-methods.def | 21 + src/mono/mono/mini/interp/transform-simd.c | 404 ++++++++++++------ src/mono/mono/mini/simd-intrinsics.c | 92 +++- src/mono/mono/mini/simd-methods.h | 6 + 7 files changed, 480 insertions(+), 135 deletions(-) diff --git a/src/mono/mono/mini/interp/interp-internals.h b/src/mono/mono/mini/interp/interp-internals.h index 5fe04bf314c8c7..ef230fb3417284 100644 --- a/src/mono/mono/mini/interp/interp-internals.h +++ b/src/mono/mono/mini/interp/interp-internals.h @@ -31,6 +31,8 @@ #define MINT_STACK_ALIGNMENT (2 * MINT_STACK_SLOT_SIZE) #define MINT_SIMD_ALIGNMENT (MINT_STACK_ALIGNMENT) #define SIZEOF_V128 16 +#define SIZEOF_V2 8 +#define SIZEOF_V3 12 #define INTERP_STACK_SIZE (1024*1024) #define INTERP_REDZONE_SIZE (8*1024) diff --git a/src/mono/mono/mini/interp/interp-simd-intrins.def b/src/mono/mono/mini/interp/interp-simd-intrins.def index d88e543af23471..b0aaa646830a62 100644 --- a/src/mono/mono/mini/interp/interp-simd-intrins.def +++ b/src/mono/mono/mini/interp/interp-simd-intrins.def @@ -70,6 +70,14 @@ INTERP_SIMD_INTRINSIC_P_PP (INTERP_SIMD_INTRINSIC_V128_R4_MULTIPLY, interp_v128_ INTERP_SIMD_INTRINSIC_P_PP (INTERP_SIMD_INTRINSIC_V128_R4_DIVISION, interp_v128_r4_op_division, 231) +INTERP_SIMD_INTRINSIC_P_P (INTERP_SIMD_INTRINSIC_V128_AS_V2, interp_v128_as_v2, -1) +INTERP_SIMD_INTRINSIC_P_P (INTERP_SIMD_INTRINSIC_V128_AS_V3, interp_v128_as_v3, -1) +INTERP_SIMD_INTRINSIC_P_P (INTERP_SIMD_INTRINSIC_V128_BITCAST, interp_v128_bitcast, -1) +INTERP_SIMD_INTRINSIC_P_P (INTERP_SIMD_INTRINSIC_V128_FROM_V2, interp_v128_from_v2, -1) +INTERP_SIMD_INTRINSIC_P_P (INTERP_SIMD_INTRINSIC_V128_FROM_V2_UNSAFE, interp_v128_from_v2_unsafe, -1) +INTERP_SIMD_INTRINSIC_P_P (INTERP_SIMD_INTRINSIC_V128_FROM_V3, interp_v128_from_v3, -1) +INTERP_SIMD_INTRINSIC_P_P (INTERP_SIMD_INTRINSIC_V128_FROM_V3_UNSAFE, interp_v128_from_v3_unsafe, -1) + INTERP_SIMD_INTRINSIC_P_P (INTERP_SIMD_INTRINSIC_V128_I1_NEGATION, interp_v128_i1_op_negation, 97) INTERP_SIMD_INTRINSIC_P_P (INTERP_SIMD_INTRINSIC_V128_I2_NEGATION, interp_v128_i2_op_negation, 129) INTERP_SIMD_INTRINSIC_P_P (INTERP_SIMD_INTRINSIC_V128_I4_NEGATION, interp_v128_i4_op_negation, 161) diff --git a/src/mono/mono/mini/interp/interp-simd.c b/src/mono/mono/mini/interp/interp-simd.c index cdc6a951c5f92a..eb60fea05b1d85 100644 --- a/src/mono/mono/mini/interp/interp-simd.c +++ b/src/mono/mono/mini/interp/interp-simd.c @@ -28,6 +28,88 @@ interp_v128_i4_all_bits_set (gpointer res) memset (res, 0xff, SIZEOF_V128); } +// Vector2 AsVector2(Vector128 v1) +static void +interp_v128_as_v2 (gpointer res, gpointer v1) +{ + float *res_typed = (float*)res; + float *v1_typed = (float*)v1; + + res_typed [0] = v1_typed [0]; + res_typed [1] = v1_typed [1]; +} + +// Vector3 AsVector3(Vector128 v1) +static void +interp_v128_as_v3 (gpointer res, gpointer v1) +{ + float *res_typed = (float*)res; + float *v1_typed = (float*)v1; + + res_typed [0] = v1_typed [0]; + res_typed [1] = v1_typed [1]; + res_typed [2] = v1_typed [2]; +} + +// Vector128 As(Vector128 v1) +static void +interp_v128_bitcast (gpointer res, gpointer v1) +{ + *(v128_i1*)res = *(v128_i1*)v1; +} + +// Vector128 AsVector128(Vector2 v1) +static void +interp_v128_from_v2 (gpointer res, gpointer v1) +{ + float *res_typed = (float*)res; + float *v1_typed = (float*)v1; + + res_typed [0] = v1_typed [0]; + res_typed [1] = v1_typed [1]; + res_typed [2] = 0; + res_typed [3] = 0; +} + +// Vector128 AsVector128Unsafe(Vector2 v1) +static void +interp_v128_from_v2_unsafe (gpointer res, gpointer v1) +{ + float *res_typed = (float*)res; + float *v1_typed = (float*)v1; + + res_typed [0] = v1_typed [0]; + res_typed [1] = v1_typed [1]; + res_typed [2] = 0; + res_typed [3] = 0; +} + +// Vector128 AsVector128(Vector3 v1) +static void +interp_v128_from_v3 (gpointer res, gpointer v1) +{ + float *res_typed = (float*)res; + float *v1_typed = (float*)v1; + + res_typed [0] = v1_typed [0]; + res_typed [1] = v1_typed [1]; + res_typed [2] = v1_typed [2]; + res_typed [3] = 0; +} + +// Vector128 AsVector128Unsafe(Vector3 v1) +static void +interp_v128_from_v3_unsafe (gpointer res, gpointer v1) +{ + float *res_typed = (float*)res; + float *v1_typed = (float*)v1; + + res_typed [0] = v1_typed [0]; + res_typed [1] = v1_typed [1]; + res_typed [2] = v1_typed [2]; + res_typed [3] = 0; +} + // op_Addition static void interp_v128_i1_op_addition (gpointer res, gpointer v1, gpointer v2) diff --git a/src/mono/mono/mini/interp/simd-methods.def b/src/mono/mono/mini/interp/simd-methods.def index 2806dc9c12c4fd..8b262a4b8c0f89 100644 --- a/src/mono/mono/mini/interp/simd-methods.def +++ b/src/mono/mono/mini/interp/simd-methods.def @@ -23,6 +23,27 @@ SIMD_METHOD(op_UnaryNegation) SIMD_METHOD(op_UnsignedRightShift) SIMD_METHOD(AndNot) +SIMD_METHOD(As) +SIMD_METHOD(AsByte) +SIMD_METHOD(AsDouble) +SIMD_METHOD(AsInt16) +SIMD_METHOD(AsInt32) +SIMD_METHOD(AsInt64) +SIMD_METHOD(AsNInt) +SIMD_METHOD(AsNUInt) +SIMD_METHOD(AsPlane) +SIMD_METHOD(AsQuaternion) +SIMD_METHOD(AsSByte) +SIMD_METHOD(AsSingle) +SIMD_METHOD(AsUInt16) +SIMD_METHOD(AsUInt32) +SIMD_METHOD(AsUInt64) +SIMD_METHOD(AsVector) +SIMD_METHOD(AsVector2) +SIMD_METHOD(AsVector3) +SIMD_METHOD(AsVector4) +SIMD_METHOD(AsVector128) +SIMD_METHOD(AsVector128Unsafe) SIMD_METHOD(ConditionalSelect) SIMD_METHOD(Create) SIMD_METHOD(CreateScalar) diff --git a/src/mono/mono/mini/interp/transform-simd.c b/src/mono/mono/mini/interp/transform-simd.c index ee0501db4658e2..9d34d94157d5cd 100644 --- a/src/mono/mono/mini/interp/transform-simd.c +++ b/src/mono/mono/mini/interp/transform-simd.c @@ -56,6 +56,27 @@ lookup_intrins (guint16 *intrinsics, int size, MonoMethod *cmethod) // i.e. all 'get_' and 'op_' need to come after regular title-case names static guint16 sri_vector128_methods [] = { SN_AndNot, + SN_As, + SN_AsByte, + SN_AsDouble, + SN_AsInt16, + SN_AsInt32, + SN_AsInt64, + SN_AsNInt, + SN_AsNUInt, + SN_AsPlane, + SN_AsQuaternion, + SN_AsSByte, + SN_AsSingle, + SN_AsUInt16, + SN_AsUInt32, + SN_AsUInt64, + SN_AsVector, + SN_AsVector128, + SN_AsVector128Unsafe, + SN_AsVector2, + SN_AsVector3, + SN_AsVector4, SN_ConditionalSelect, SN_Create, SN_CreateScalar, @@ -283,35 +304,37 @@ emit_common_simd_operations (TransformData *td, int id, int atype, int vector_si } static gboolean -get_common_simd_info (MonoClass *vector_klass, MonoMethodSignature *csignature, MonoTypeEnum *atype, int *vector_size, int *arg_size, int *scalar_arg) +get_common_simd_info (MonoClass *klass, MonoTypeEnum *atype, int *klass_size, int *arg_size) { - if (!m_class_is_simd_type (vector_klass) && csignature->param_count) - vector_klass = mono_class_from_mono_type_internal (csignature->params [0]); - if (!m_class_is_simd_type (vector_klass)) + *klass_size = mono_class_value_size (klass, NULL); + if (!m_class_is_simd_type (klass)) return FALSE; - - MonoType *arg_type = mono_class_get_context (vector_klass)->class_inst->type_argv [0]; - if (!mono_type_is_primitive (arg_type)) - return FALSE; - *atype = arg_type->type; - if (*atype == MONO_TYPE_BOOLEAN) - return FALSE; - *vector_size = mono_class_value_size (vector_klass, NULL); - g_assert (*vector_size == SIZEOF_V128); - if (arg_size) + if (mono_class_is_ginst (klass)) { + MonoType *arg_type = mono_class_get_context (klass)->class_inst->type_argv [0]; + if (!mono_type_is_primitive (arg_type)) + return FALSE; + *atype = arg_type->type; *arg_size = mono_class_value_size (mono_class_from_mono_type_internal (arg_type), NULL); + if (*atype == MONO_TYPE_BOOLEAN) + return FALSE; + } else { + *atype = MONO_TYPE_R4; + *arg_size = sizeof (float); + } + return TRUE; +} - *scalar_arg = -1; +static int get_common_simd_scalar_arg (MonoMethodSignature *csignature) +{ for (int i = 0; i < csignature->param_count; i++) { - if (csignature->params [i]->type != MONO_TYPE_GENERICINST) - *scalar_arg = i; + if (MONO_TYPE_IS_PRIMITIVE(csignature->params [i])) + return i; } - - return TRUE; + return -1; } static void -emit_common_simd_epilogue (TransformData *td, MonoClass *vector_klass, MonoMethodSignature *csignature, int vector_size, gboolean allow_void) +emit_common_simd_epilogue (TransformData *td, MonoMethodSignature *csignature) { td->sp -= csignature->param_count; for (int i = 0; i < csignature->param_count; i++) @@ -319,11 +342,11 @@ emit_common_simd_epilogue (TransformData *td, MonoClass *vector_klass, MonoMetho int ret_mt = mono_mint_type (csignature->ret); if (csignature->ret->type == MONO_TYPE_VOID) { - g_assert (allow_void); interp_ins_set_dummy_dreg (td->last_ins, td); } else if (ret_mt == MINT_TYPE_VT) { - // For these intrinsics, if we return a VT then it is a V128 - push_type_vt (td, vector_klass, vector_size); + MonoClass *ret_klass = mono_class_from_mono_type_internal (csignature->ret); + int ret_size = mono_class_value_size (ret_klass, NULL); + push_type_vt (td, ret_klass, ret_size); interp_ins_set_dreg (td->last_ins, td->sp [-1].var); } else { push_simple_type (td, stack_type [ret_mt]); @@ -364,9 +387,6 @@ emit_sri_vector128 (TransformData *td, MonoMethod *cmethod, MonoMethodSignature if (id == -1) return FALSE; - MonoClass *vector_klass = NULL; - int vector_size = 0; - if (id == SN_get_IsHardwareAccelerated) { interp_add_ins (td, MINT_LDC_I4_1); goto opcode_added; @@ -375,124 +395,241 @@ emit_sri_vector128 (TransformData *td, MonoMethod *cmethod, MonoMethodSignature gint16 simd_opcode = -1; gint16 simd_intrins = -1; - vector_klass = mono_class_from_mono_type_internal (csignature->ret); + MonoTypeEnum ret_atype; + int ret_size, ret_arg_size; + MonoClass *ret_klass = mono_class_from_mono_type_internal (csignature->ret); + bool ret_is_simd = get_common_simd_info (ret_klass, &ret_atype, &ret_size, &ret_arg_size); + + MonoTypeEnum param0_atype; + int param0_size, param0_arg_size; + MonoClass *param0_klass = NULL; + bool param0_is_simd = false; + if (csignature->param_count) { + param0_klass = mono_class_from_mono_type_internal (csignature->params [0]); + param0_is_simd = get_common_simd_info (param0_klass, ¶m0_atype, ¶m0_size, ¶m0_arg_size); + } - MonoTypeEnum atype; - int arg_size, scalar_arg; - if (!get_common_simd_info (vector_klass, csignature, &atype, &vector_size, &arg_size, &scalar_arg)) + if (!ret_is_simd && !param0_is_simd) { return FALSE; + } switch (id) { - case SN_AndNot: + case SN_AndNot: { + g_assert ((ret_size == SIZEOF_V128) && (param0_size == SIZEOF_V128)); simd_opcode = MINT_SIMD_INTRINS_P_PP; simd_intrins = INTERP_SIMD_INTRINSIC_V128_AND_NOT; break; - case SN_ConditionalSelect: + } + case SN_As: + case SN_AsByte: + case SN_AsDouble: + case SN_AsInt16: + case SN_AsInt32: + case SN_AsInt64: + case SN_AsNInt: + case SN_AsNUInt: + case SN_AsPlane: + case SN_AsQuaternion: + case SN_AsSByte: + case SN_AsSingle: + case SN_AsUInt16: + case SN_AsUInt32: + case SN_AsUInt64: + case SN_AsVector: + case SN_AsVector4: { + if (!ret_is_simd || !param0_is_simd) { + return FALSE; + } + g_assert ((ret_size == SIZEOF_V128) && (param0_size == SIZEOF_V128)); + simd_opcode = MINT_SIMD_INTRINS_P_P; + simd_intrins = INTERP_SIMD_INTRINSIC_V128_BITCAST; + break; + } + case SN_AsVector128: { + g_assert (ret_size == SIZEOF_V128); + + if (param0_size == SIZEOF_V2) { + simd_opcode = MINT_SIMD_INTRINS_P_P; + simd_intrins = INTERP_SIMD_INTRINSIC_V128_FROM_V2; + break; + } else if (param0_size == SIZEOF_V3) { + simd_opcode = MINT_SIMD_INTRINS_P_P; + simd_intrins = INTERP_SIMD_INTRINSIC_V128_FROM_V3; + break; + } + + g_assert (param0_size == SIZEOF_V128); + simd_opcode = MINT_SIMD_INTRINS_P_P; + simd_intrins = INTERP_SIMD_INTRINSIC_V128_BITCAST; + break; + } + case SN_AsVector128Unsafe: { + g_assert (ret_size == SIZEOF_V128); + + if (param0_size == SIZEOF_V2) { + simd_opcode = MINT_SIMD_INTRINS_P_P; + simd_intrins = INTERP_SIMD_INTRINSIC_V128_FROM_V2_UNSAFE; + break; + } + + g_assert (param0_size == SIZEOF_V3); + simd_opcode = MINT_SIMD_INTRINS_P_P; + simd_intrins = INTERP_SIMD_INTRINSIC_V128_FROM_V3_UNSAFE; + break; + } + case SN_AsVector2: { + g_assert ((ret_size == SIZEOF_V2) && (param0_size == SIZEOF_V128)); + simd_opcode = MINT_SIMD_INTRINS_P_P; + simd_intrins = INTERP_SIMD_INTRINSIC_V128_AS_V2; + break; + } + case SN_AsVector3: { + g_assert ((ret_size == SIZEOF_V3) && (param0_size == SIZEOF_V128)); + simd_opcode = MINT_SIMD_INTRINS_P_P; + simd_intrins = INTERP_SIMD_INTRINSIC_V128_AS_V3; + break; + } + case SN_ConditionalSelect: { + g_assert ((ret_size == SIZEOF_V128) && (param0_size == SIZEOF_V128)); simd_opcode = MINT_SIMD_INTRINS_P_PPP; simd_intrins = INTERP_SIMD_INTRINSIC_V128_CONDITIONAL_SELECT; break; - case SN_Create: - if (csignature->param_count == 1 && atype == csignature->params [0]->type) { + } + case SN_Create: { + g_assert (ret_size == SIZEOF_V128); + if (ret_atype != csignature->params [0]->type) { + return FALSE; + } + if (csignature->param_count == 1) { simd_opcode = MINT_SIMD_INTRINS_P_P; - if (arg_size == 1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I1_CREATE; - else if (arg_size == 2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I2_CREATE; - else if (arg_size == 4) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I4_CREATE; - else if (arg_size == 8) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I8_CREATE; - } else if (csignature->param_count == vector_size / arg_size && atype == csignature->params [0]->type) { - emit_vector_create (td, csignature, vector_klass, vector_size); + if (ret_arg_size == 1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I1_CREATE; + else if (ret_arg_size == 2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I2_CREATE; + else if (ret_arg_size == 4) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I4_CREATE; + else if (ret_arg_size == 8) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I8_CREATE; + } else if (csignature->param_count == ret_size / ret_arg_size) { + emit_vector_create (td, csignature, ret_klass, ret_size); td->ip += 5; return TRUE; } break; + } case SN_CreateScalar: - case SN_CreateScalarUnsafe: + case SN_CreateScalarUnsafe: { + g_assert ((ret_size == SIZEOF_V128) && (ret_atype == csignature->params [0]->type)); simd_opcode = MINT_SIMD_INTRINS_P_P; - if (arg_size == 1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I1_CREATE_SCALAR; - else if (arg_size == 2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I2_CREATE_SCALAR; - else if (arg_size == 4) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I4_CREATE_SCALAR; - else if (arg_size == 8) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I8_CREATE_SCALAR; + if (ret_arg_size == 1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I1_CREATE_SCALAR; + else if (ret_arg_size == 2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I2_CREATE_SCALAR; + else if (ret_arg_size == 4) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I4_CREATE_SCALAR; + else if (ret_arg_size == 8) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I8_CREATE_SCALAR; break; - case SN_Equals: + } + case SN_Equals: { + g_assert ((ret_size == SIZEOF_V128) && (param0_size == SIZEOF_V128)); simd_opcode = MINT_SIMD_INTRINS_P_PP; - if (atype == MONO_TYPE_I1 || atype == MONO_TYPE_U1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I1_EQUALS; - else if (atype == MONO_TYPE_I2 || atype == MONO_TYPE_U2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I2_EQUALS; - else if (atype == MONO_TYPE_I4 || atype == MONO_TYPE_U4) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I4_EQUALS; - else if (atype == MONO_TYPE_I8 || atype == MONO_TYPE_U8) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I8_EQUALS; - else if (atype == MONO_TYPE_R4) simd_intrins = INTERP_SIMD_INTRINSIC_V128_R4_EQUALS; + if (ret_atype == MONO_TYPE_I1 || ret_atype == MONO_TYPE_U1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I1_EQUALS; + else if (ret_atype == MONO_TYPE_I2 || ret_atype == MONO_TYPE_U2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I2_EQUALS; + else if (ret_atype == MONO_TYPE_I4 || ret_atype == MONO_TYPE_U4) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I4_EQUALS; + else if (ret_atype == MONO_TYPE_I8 || ret_atype == MONO_TYPE_U8) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I8_EQUALS; + else if (ret_atype == MONO_TYPE_R4) simd_intrins = INTERP_SIMD_INTRINSIC_V128_R4_EQUALS; break; - case SN_EqualsAny: + } + case SN_EqualsAny: { + g_assert (param0_size == SIZEOF_V128); simd_opcode = MINT_SIMD_INTRINS_P_PP; - if (atype == MONO_TYPE_I1 || atype == MONO_TYPE_U1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I1_EQUALS_ANY; - else if (atype == MONO_TYPE_I2 || atype == MONO_TYPE_U2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I2_EQUALS_ANY; - else if (atype == MONO_TYPE_I4 || atype == MONO_TYPE_U4) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I4_EQUALS_ANY; - else if (atype == MONO_TYPE_I8 || atype == MONO_TYPE_U8) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I8_EQUALS_ANY; + if (param0_atype == MONO_TYPE_I1 || param0_atype == MONO_TYPE_U1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I1_EQUALS_ANY; + else if (param0_atype == MONO_TYPE_I2 || param0_atype == MONO_TYPE_U2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I2_EQUALS_ANY; + else if (param0_atype == MONO_TYPE_I4 || param0_atype == MONO_TYPE_U4) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I4_EQUALS_ANY; + else if (param0_atype == MONO_TYPE_I8 || param0_atype == MONO_TYPE_U8) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I8_EQUALS_ANY; break; - case SN_ExtractMostSignificantBits: + } + case SN_ExtractMostSignificantBits: { + g_assert (param0_size == SIZEOF_V128); simd_opcode = MINT_SIMD_INTRINS_P_P; - if (arg_size == 1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I1_EXTRACT_MSB; - else if (arg_size == 2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I2_EXTRACT_MSB; - else if (arg_size == 4) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I4_EXTRACT_MSB; - else if (arg_size == 8) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I8_EXTRACT_MSB; + if (param0_arg_size == 1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I1_EXTRACT_MSB; + else if (param0_arg_size == 2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I2_EXTRACT_MSB; + else if (param0_arg_size == 4) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I4_EXTRACT_MSB; + else if (param0_arg_size == 8) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I8_EXTRACT_MSB; break; - case SN_GreaterThan: + } + case SN_GreaterThan: { + g_assert ((ret_size == SIZEOF_V128) && (param0_size == SIZEOF_V128)); simd_opcode = MINT_SIMD_INTRINS_P_PP; - if (atype == MONO_TYPE_U1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_U1_GREATER_THAN; + if (ret_atype == MONO_TYPE_U1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_U1_GREATER_THAN; break; - case SN_LessThan: + } + case SN_LessThan: { + g_assert ((ret_size == SIZEOF_V128) && (param0_size == SIZEOF_V128)); simd_opcode = MINT_SIMD_INTRINS_P_PP; - if (atype == MONO_TYPE_I1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I1_LESS_THAN; - else if (atype == MONO_TYPE_U1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_U1_LESS_THAN; - else if (atype == MONO_TYPE_I2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I2_LESS_THAN; + if (ret_atype == MONO_TYPE_I1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I1_LESS_THAN; + else if (ret_atype == MONO_TYPE_U1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_U1_LESS_THAN; + else if (ret_atype == MONO_TYPE_I2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I2_LESS_THAN; break; - case SN_LessThanOrEqual: + } + case SN_LessThanOrEqual: { + g_assert ((ret_size == SIZEOF_V128) && (param0_size == SIZEOF_V128)); simd_opcode = MINT_SIMD_INTRINS_P_PP; - if (atype == MONO_TYPE_U2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_U2_LESS_THAN_EQUAL; + if (ret_atype == MONO_TYPE_U2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_U2_LESS_THAN_EQUAL; break; - case SN_Narrow: + } + case SN_Narrow: { + g_assert ((ret_size == SIZEOF_V128) && (param0_size == SIZEOF_V128)); simd_opcode = MINT_SIMD_INTRINS_P_PP; - if (atype == MONO_TYPE_U1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_U1_NARROW; + if (ret_atype == MONO_TYPE_U1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_U1_NARROW; break; - case SN_ShiftLeft: - g_assert (scalar_arg == 1); + } + case SN_ShiftLeft: { + g_assert ((ret_size == SIZEOF_V128) && (param0_size == SIZEOF_V128)); + g_assert (get_common_simd_scalar_arg(csignature) == 1); simd_opcode = MINT_SIMD_INTRINS_P_PP; - if (arg_size == 1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I1_LEFT_SHIFT; - else if (arg_size == 2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I2_LEFT_SHIFT; - else if (arg_size == 4) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I4_LEFT_SHIFT; - else if (arg_size == 8) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I8_LEFT_SHIFT; + if (ret_arg_size == 1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I1_LEFT_SHIFT; + else if (ret_arg_size == 2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I2_LEFT_SHIFT; + else if (ret_arg_size == 4) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I4_LEFT_SHIFT; + else if (ret_arg_size == 8) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I8_LEFT_SHIFT; break; - case SN_ShiftRightLogical: - g_assert (scalar_arg == 1); + } + case SN_ShiftRightLogical: { + g_assert ((ret_size == SIZEOF_V128) && (param0_size == SIZEOF_V128)); + g_assert (get_common_simd_scalar_arg(csignature) == 1); simd_opcode = MINT_SIMD_INTRINS_P_PP; - if (arg_size == 1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I1_URIGHT_SHIFT; - else if (arg_size == 2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I2_URIGHT_SHIFT; - else if (arg_size == 4) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I4_URIGHT_SHIFT; - else if (arg_size == 8) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I8_URIGHT_SHIFT; + if (ret_arg_size == 1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I1_URIGHT_SHIFT; + else if (ret_arg_size == 2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I2_URIGHT_SHIFT; + else if (ret_arg_size == 4) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I4_URIGHT_SHIFT; + else if (ret_arg_size == 8) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I8_URIGHT_SHIFT; break; - case SN_ShiftRightArithmetic: - g_assert (scalar_arg == 1); + } + case SN_ShiftRightArithmetic: { + g_assert ((ret_size == SIZEOF_V128) && (param0_size == SIZEOF_V128)); + g_assert (get_common_simd_scalar_arg(csignature) == 1); simd_opcode = MINT_SIMD_INTRINS_P_PP; - if (atype == MONO_TYPE_I1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I1_RIGHT_SHIFT; - else if (atype == MONO_TYPE_I2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I2_RIGHT_SHIFT; - else if (atype == MONO_TYPE_I4) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I4_RIGHT_SHIFT; - else if (atype == MONO_TYPE_U1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I1_URIGHT_SHIFT; - else if (atype == MONO_TYPE_U2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I2_URIGHT_SHIFT; - else if (atype == MONO_TYPE_U4) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I4_URIGHT_SHIFT; - break; - case SN_Shuffle: + if (ret_atype == MONO_TYPE_I1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I1_RIGHT_SHIFT; + else if (ret_atype == MONO_TYPE_I2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I2_RIGHT_SHIFT; + else if (ret_atype == MONO_TYPE_I4) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I4_RIGHT_SHIFT; + else if (ret_atype == MONO_TYPE_U1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I1_URIGHT_SHIFT; + else if (ret_atype == MONO_TYPE_U2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I2_URIGHT_SHIFT; + else if (ret_atype == MONO_TYPE_U4) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I4_URIGHT_SHIFT; + break; + } + case SN_Shuffle: { + g_assert ((ret_size == SIZEOF_V128) && (param0_size == SIZEOF_V128)); simd_opcode = MINT_SIMD_INTRINS_P_PP; - if (arg_size == 1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I1_SHUFFLE; - else if (arg_size == 2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I2_SHUFFLE; - else if (arg_size == 4) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I4_SHUFFLE; - else if (arg_size == 8) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I8_SHUFFLE; + if (ret_arg_size == 1) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I1_SHUFFLE; + else if (ret_arg_size == 2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I2_SHUFFLE; + else if (ret_arg_size == 4) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I4_SHUFFLE; + else if (ret_arg_size == 8) simd_intrins = INTERP_SIMD_INTRINSIC_V128_I8_SHUFFLE; break; - case SN_WidenLower: + } + case SN_WidenLower: { + g_assert ((ret_size == SIZEOF_V128) && (param0_size == SIZEOF_V128)); simd_opcode = MINT_SIMD_INTRINS_P_P; - if (atype == MONO_TYPE_U2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_U2_WIDEN_LOWER; + if (ret_atype == MONO_TYPE_U2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_U2_WIDEN_LOWER; break; - case SN_WidenUpper: + } + case SN_WidenUpper: { + g_assert ((ret_size == SIZEOF_V128) && (param0_size == SIZEOF_V128)); simd_opcode = MINT_SIMD_INTRINS_P_P; - if (atype == MONO_TYPE_U2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_U2_WIDEN_UPPER; + if (ret_atype == MONO_TYPE_U2) simd_intrins = INTERP_SIMD_INTRINSIC_V128_U2_WIDEN_UPPER; break; + } default: return FALSE; } @@ -505,7 +642,7 @@ emit_sri_vector128 (TransformData *td, MonoMethod *cmethod, MonoMethodSignature td->last_ins->data [0] = simd_intrins; opcode_added: - emit_common_simd_epilogue (td, vector_klass, csignature, vector_size, FALSE); + emit_common_simd_epilogue (td, csignature); return TRUE; } @@ -519,13 +656,12 @@ emit_sri_vector128_t (TransformData *td, MonoMethod *cmethod, MonoMethodSignatur gint16 simd_opcode = -1; gint16 simd_intrins = -1; - // First argument is always vector - MonoClass *vector_klass = cmethod->klass; - MonoTypeEnum atype; - int vector_size, arg_size, scalar_arg; - if (!get_common_simd_info (vector_klass, csignature, &atype, &vector_size, &arg_size, &scalar_arg)) + int vector_size, arg_size; + MonoClass *vector_klass = cmethod->klass; + if (!get_common_simd_info (vector_klass, &atype, &vector_size, &arg_size)) return FALSE; + int scalar_arg = get_common_simd_scalar_arg (csignature); if (emit_common_simd_operations (td, id, atype, vector_size, arg_size, scalar_arg, &simd_opcode, &simd_intrins)) goto opcode_added; @@ -537,7 +673,7 @@ emit_sri_vector128_t (TransformData *td, MonoMethod *cmethod, MonoMethodSignatur td->last_ins->data [0] = simd_intrins; opcode_added: - emit_common_simd_epilogue (td, vector_klass, csignature, vector_size, FALSE); + emit_common_simd_epilogue (td, csignature); return TRUE; } @@ -551,15 +687,12 @@ emit_sn_vector_t (TransformData *td, MonoMethod *cmethod, MonoMethodSignature *c gint16 simd_opcode = -1; gint16 simd_intrins = -1; - // First argument is always vector - MonoClass *vector_klass = cmethod->klass; - if (!m_class_is_simd_type (vector_klass)) - return FALSE; - MonoTypeEnum atype; - int vector_size, arg_size, scalar_arg; - if (!get_common_simd_info (vector_klass, csignature, &atype, &vector_size, &arg_size, &scalar_arg)) + int vector_size, arg_size; + MonoClass *vector_klass = cmethod->klass; + if (!get_common_simd_info (vector_klass, &atype, &vector_size, &arg_size)) return FALSE; + int scalar_arg = get_common_simd_scalar_arg (csignature); if (emit_common_simd_operations (td, id, atype, vector_size, arg_size, scalar_arg, &simd_opcode, &simd_intrins)) { goto opcode_added; @@ -582,7 +715,7 @@ emit_sn_vector_t (TransformData *td, MonoMethod *cmethod, MonoMethodSignature *c td->last_ins->data [0] = simd_intrins; opcode_added: - emit_common_simd_epilogue (td, vector_klass, csignature, vector_size, FALSE); + emit_common_simd_epilogue (td, csignature); return TRUE; } @@ -596,17 +729,12 @@ emit_sn_vector4 (TransformData *td, MonoMethod *cmethod, MonoMethodSignature *cs gint16 simd_opcode = -1; gint16 simd_intrins = -1; - // First argument is always vector + MonoTypeEnum atype; + int vector_size, arg_size; MonoClass *vector_klass = cmethod->klass; - - MonoTypeEnum atype = MONO_TYPE_R4; - int vector_size = SIZEOF_V128; - int arg_size = sizeof (float); - int scalar_arg = -1; - for (int i = 0; i < csignature->param_count; i++) { - if (csignature->params [i]->type != MONO_TYPE_GENERICINST) - scalar_arg = i; - } + if (!get_common_simd_info (vector_klass, &atype, &vector_size, &arg_size)) + return FALSE; + int scalar_arg = get_common_simd_scalar_arg (csignature); if (emit_common_simd_operations (td, id, atype, vector_size, arg_size, scalar_arg, &simd_opcode, &simd_intrins)) { goto opcode_added; @@ -629,7 +757,7 @@ emit_sn_vector4 (TransformData *td, MonoMethod *cmethod, MonoMethodSignature *cs td->last_ins->data [0] = simd_intrins; opcode_added: - emit_common_simd_epilogue (td, vector_klass, csignature, vector_size, FALSE); + emit_common_simd_epilogue (td, csignature); return TRUE; } @@ -844,10 +972,6 @@ emit_sri_packedsimd (TransformData *td, MonoMethod *cmethod, MonoMethodSignature int id = lookup_intrins (sri_packedsimd_methods, sizeof (sri_packedsimd_methods), cmethod); // We don't early-out for an unrecognized method, we will generate an NIY later - MonoClass *vector_klass = mono_class_from_mono_type_internal (csignature->ret); - MonoTypeEnum atype; - int vector_size = -1, arg_size, scalar_arg; - // NOTE: Linker substitutions (used in AOT) will prevent this from running. if ((id == SN_get_IsSupported) || (id == SN_get_IsHardwareAccelerated)) { #if HOST_BROWSER @@ -858,12 +982,24 @@ emit_sri_packedsimd (TransformData *td, MonoMethod *cmethod, MonoMethodSignature goto opcode_added; } - get_common_simd_info (vector_klass, csignature, &atype, &vector_size, &arg_size, &scalar_arg); - #if HOST_BROWSER gint16 simd_opcode = -1; gint16 simd_intrins = -1; + MonoTypeEnum ret_atype; + int ret_size, ret_arg_size; + MonoClass *ret_klass = mono_class_from_mono_type_internal (csignature->ret); + bool ret_is_simd = get_common_simd_info (ret_klass, &ret_atype, &ret_size, &ret_arg_size); + + g_assert (csignature->param_count); + + MonoTypeEnum param0_atype; + int param0_size, param0_arg_size; + MonoClass *param0_klass = mono_class_from_mono_type_internal (csignature->params [0]); + bool param0_is_simd = get_common_simd_info (param0_klass, ¶m0_atype, ¶m0_size, ¶m0_arg_size); + + g_assert (ret_is_simd || param0_is_simd); + PackedSimdIntrinsicInfo *info = lookup_packedsimd_intrinsic (cmethod->name, csignature->params[0]); if (info && info->interp_opcode && info->simd_intrins) { @@ -895,7 +1031,7 @@ emit_sri_packedsimd (TransformData *td, MonoMethod *cmethod, MonoMethodSignature #endif // HOST_BROWSER opcode_added: - emit_common_simd_epilogue (td, vector_klass, csignature, vector_size, TRUE); + emit_common_simd_epilogue (td, csignature); return TRUE; } diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 2344c04c37ae73..e0093639e71af2 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1185,11 +1185,21 @@ static guint16 sri_vector_methods [] = { SN_AsInt16, SN_AsInt32, SN_AsInt64, + SN_AsNInt, + SN_AsNUInt, + SN_AsPlane, + SN_AsQuaternion, SN_AsSByte, SN_AsSingle, SN_AsUInt16, SN_AsUInt32, SN_AsUInt64, + SN_AsVector, + SN_AsVector128, + SN_AsVector128Unsafe, + SN_AsVector2, + SN_AsVector3, + SN_AsVector4, SN_BitwiseAnd, SN_BitwiseOr, SN_Ceiling, @@ -1618,15 +1628,95 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi case SN_AsInt16: case SN_AsInt32: case SN_AsInt64: + case SN_AsNInt: + case SN_AsNUInt: + case SN_AsPlane: + case SN_AsQuaternion: case SN_AsSByte: case SN_AsSingle: case SN_AsUInt16: case SN_AsUInt32: - case SN_AsUInt64: { + case SN_AsUInt64: + case SN_AsVector: { if (!is_element_type_primitive (fsig->ret) || !is_element_type_primitive (fsig->params [0])) return NULL; return emit_simd_ins (cfg, klass, OP_XCAST, args [0]->dreg, -1); } + case SN_AsVector2: { + if (!mini_class_is_simd (cfg, mono_class_from_mono_type_internal (fsig->ret))) { + // FIXME: Support Vector2 and Vector3 for WASM and AMD64 + return NULL; + } + return emit_simd_ins_for_sig (cfg, klass, OP_XLOWER, 0, arg0_type, fsig, args); + } + case SN_AsVector3: { + if (!mini_class_is_simd (cfg, mono_class_from_mono_type_internal (fsig->ret))) { + // FIXME: Support Vector2 and Vector3 for WASM and AMD64 + return NULL; + } + return emit_simd_ins (cfg, klass, OP_XCAST, args [0]->dreg, -1); + } + case SN_AsVector4: { + if (!COMPILE_LLVM (cfg)) { + // FIXME: This can assert for load opcodes needing OP_LOADV_MEMBASE + return NULL; + } + return emit_simd_ins (cfg, klass, OP_XCAST, args [0]->dreg, -1); + } + case SN_AsVector128: { + if (!is_element_type_primitive (fsig->ret) || !is_element_type_primitive (fsig->params [0])) + return NULL; + + MonoClass *input_klass = mono_class_from_mono_type_internal (fsig->params [0]); + + if (!mini_class_is_simd (cfg, input_klass)) { + // FIXME: Support Vector2 and Vector3 for WASM and AMD64 + return NULL; + } + + int input_size = mono_class_value_size (input_klass, NULL); + + if ((input_size == 8) && COMPILE_LLVM (cfg)) + return emit_simd_ins (cfg, klass, OP_XWIDEN, args [0]->dreg, -1); + + if (input_size == 16) + return emit_simd_ins (cfg, klass, OP_XCAST, args [0]->dreg, -1); + + g_assert ((input_size == 8) || (input_size == 12)); + + static float r4_0 = 0; + MonoInst *zero; + int zero_dreg = alloc_freg (cfg); + MONO_INST_NEW (cfg, zero, OP_R4CONST); + zero->inst_p0 = (void*)&r4_0; + zero->dreg = zero_dreg; + MONO_ADD_INS (cfg->cbb, zero); + + MonoInst *ins = args [0]; + if (input_size == 8) { + ins = emit_vector_insert_element (cfg, klass, ins, MONO_TYPE_R4, zero, 2, FALSE); + } + return emit_vector_insert_element (cfg, klass, ins, MONO_TYPE_R4, zero, 3, FALSE); + } + case SN_AsVector128Unsafe: { + if (!is_element_type_primitive (fsig->ret) || !is_element_type_primitive (fsig->params [0])) + return NULL; + + MonoClass *input_klass = mono_class_from_mono_type_internal (fsig->params [0]); + + if (!mini_class_is_simd (cfg, input_klass)) { + // FIXME: Support Vector2 and Vector3 for WASM and AMD64 + return NULL; + } + + int input_size = mono_class_value_size (input_klass, NULL); + + if ((input_size == 8) && COMPILE_LLVM (cfg)) + return emit_simd_ins (cfg, klass, OP_XWIDEN_UNSAFE, args [0]->dreg, -1); + + g_assert (input_size == 12); + return emit_simd_ins (cfg, klass, OP_XCAST, args [0]->dreg, -1); + } case SN_Ceiling: case SN_Floor: { if (!type_enum_is_float (arg0_type)) diff --git a/src/mono/mono/mini/simd-methods.h b/src/mono/mono/mini/simd-methods.h index 6c0c47d26288c7..548067ff1d5bf3 100644 --- a/src/mono/mono/mini/simd-methods.h +++ b/src/mono/mono/mini/simd-methods.h @@ -71,12 +71,18 @@ METHOD(AsDouble) METHOD(AsInt16) METHOD(AsInt32) METHOD(AsInt64) +METHOD(AsNInt) +METHOD(AsNUInt) +METHOD(AsPlane) +METHOD(AsQuaternion) METHOD(AsSByte) METHOD(AsSingle) METHOD(AsUInt16) METHOD(AsUInt32) METHOD(AsUInt64) +METHOD(AsVector) METHOD(AsVector128) +METHOD(AsVector128Unsafe) METHOD(AsVector2) METHOD(AsVector256) METHOD(AsVector3)