From a0449e190036e8328a13855ffab9e8ee53d4cfc1 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 5 Mar 2020 22:07:48 +0300 Subject: [PATCH 1/3] Intrinsify Interlocked.And and Interlocked.Or --- src/mono/mono/mini/intrinsics.c | 38 ++++++++++++++++++++++------ src/mono/mono/mini/mini-llvm-cpp.cpp | 6 +++++ src/mono/mono/mini/mini-llvm-cpp.h | 2 ++ src/mono/mono/mini/mini-llvm.c | 22 +++++++++++++--- src/mono/mono/mini/mini-ops.h | 4 +++ 5 files changed, 60 insertions(+), 12 deletions(-) diff --git a/src/mono/mono/mini/intrinsics.c b/src/mono/mono/mini/intrinsics.c index d9c7a389d3a763..01559f1647ff55 100644 --- a/src/mono/mono/mini/intrinsics.c +++ b/src/mono/mono/mini/intrinsics.c @@ -1186,26 +1186,48 @@ mini_emit_inst_for_method (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSign ins->type = (opcode == OP_ATOMIC_ADD_I4) ? STACK_I4 : STACK_I8; MONO_ADD_INS (cfg->cbb, ins); } - } else if (strcmp (cmethod->name, "Add") == 0 && fsig->param_count == 2) { + } else if (fsig->param_count == 2 && + ((strcmp (cmethod->name, "Add") == 0) || + (strcmp (cmethod->name, "And") == 0) || + (strcmp (cmethod->name, "Or") == 0))) { + guint32 opcode = 0; + guint32 opcode_i4 = 0; + guint32 opcode_i8 = 0; + gboolean llvm_only = FALSE; + + if (strcmp(cmethod->name, "Add") == 0) { + opcode_i4 = OP_ATOMIC_ADD_I4; + opcode_i8 = OP_ATOMIC_ADD_I8; + } else if (strcmp(cmethod->name, "And") == 0) { + opcode_i4 = OP_ATOMIC_AND_I4; + opcode_i8 = OP_ATOMIC_AND_I8; + llvm_only = TRUE; + } else if (strcmp(cmethod->name, "And") == 0) { + opcode_i4 = OP_ATOMIC_OR_I4; + opcode_i8 = OP_ATOMIC_OR_I8; + llvm_only = TRUE; + } else { + g_assert_not_reached (); + } if (fsig->params [0]->type == MONO_TYPE_I4) { - opcode = OP_ATOMIC_ADD_I4; + opcode = opcode_i4; cfg->has_atomic_add_i4 = TRUE; } -#if SIZEOF_REGISTER == 8 - else if (fsig->params [0]->type == MONO_TYPE_I8) - opcode = OP_ATOMIC_ADD_I8; -#endif + else if (fsig->params [0]->type == MONO_TYPE_I8 && SIZEOF_REGISTER == 8) { + opcode = opcode_i8; + } + if (opcode) { - if (!mono_arch_opcode_supported (opcode)) + if (!mono_arch_opcode_supported (opcode) && !llvm_only) return NULL; MONO_INST_NEW (cfg, ins, opcode); ins->dreg = mono_alloc_ireg (cfg); ins->inst_basereg = args [0]->dreg; ins->inst_offset = 0; ins->sreg2 = args [1]->dreg; - ins->type = (opcode == OP_ATOMIC_ADD_I4) ? STACK_I4 : STACK_I8; + ins->type = (opcode == opcode_i4) ? STACK_I4 : STACK_I8; MONO_ADD_INS (cfg->cbb, ins); } } diff --git a/src/mono/mono/mini/mini-llvm-cpp.cpp b/src/mono/mono/mini/mini-llvm-cpp.cpp index 86fca1c8eef686..fef9352bf85a24 100644 --- a/src/mono/mono/mini/mini-llvm-cpp.cpp +++ b/src/mono/mono/mini/mini-llvm-cpp.cpp @@ -174,6 +174,12 @@ mono_llvm_build_atomic_rmw (LLVMBuilderRef builder, AtomicRMWOp op, LLVMValueRef case LLVM_ATOMICRMW_OP_ADD: aop = AtomicRMWInst::Add; break; + case LLVM_ATOMICRMW_OP_AND: + aop = AtomicRMWInst::And; + break; + case LLVM_ATOMICRMW_OP_OR: + aop = AtomicRMWInst::Or; + break; default: g_assert_not_reached (); break; diff --git a/src/mono/mono/mini/mini-llvm-cpp.h b/src/mono/mono/mini/mini-llvm-cpp.h index 9c2e6d060efccc..f1b90637aa52b6 100644 --- a/src/mono/mono/mini/mini-llvm-cpp.h +++ b/src/mono/mono/mini/mini-llvm-cpp.h @@ -35,6 +35,8 @@ typedef enum { typedef enum { LLVM_ATOMICRMW_OP_XCHG = 0, LLVM_ATOMICRMW_OP_ADD = 1, + LLVM_ATOMICRMW_OP_AND = 2, + LLVM_ATOMICRMW_OP_OR = 3, } AtomicRMWOp; typedef enum { diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index f7dccafe83118d..14aa65a4d0104a 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -6434,11 +6434,15 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) break; } case OP_ATOMIC_ADD_I4: - case OP_ATOMIC_ADD_I8: { + case OP_ATOMIC_ADD_I8: + case OP_ATOMIC_AND_I4: + case OP_ATOMIC_AND_I8: + case OP_ATOMIC_OR_I4: + case OP_ATOMIC_OR_I8: { LLVMValueRef args [2]; LLVMTypeRef t; - - if (ins->opcode == OP_ATOMIC_ADD_I4) + + if (ins->type == STACK_I4) t = LLVMInt32Type (); else t = LLVMInt64Type (); @@ -6448,7 +6452,17 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) args [0] = convert (ctx, lhs, LLVMPointerType (t, 0)); args [1] = convert (ctx, rhs, t); ARM64_ATOMIC_FENCE_FIX; - values [ins->dreg] = LLVMBuildAdd (builder, mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_ADD, args [0], args [1]), args [1], dname); + if (ins->opcode == OP_ATOMIC_ADD_I4 || ins->opcode == OP_ATOMIC_ADD_I8) + values [ins->dreg] = LLVMBuildAdd (builder, + mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_ADD, args [0], args [1]), args [1], dname); + else if (ins->opcode == OP_ATOMIC_AND_I4 || ins->opcode == OP_ATOMIC_AND_I8) + values [ins->dreg] = LLVMBuildAnd (builder, + mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_AND, args [0], args [1]), args [1], dname); + else if (ins->opcode == OP_ATOMIC_OR_I4 || ins->opcode == OP_ATOMIC_OR_I8) + values [ins->dreg] = LLVMBuildOr (builder, + mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_OR, args [0], args [1]), args [1], dname); + else + g_assert_not_reached (); ARM64_ATOMIC_FENCE_FIX; break; } diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index 325e0de2faff51..9a1d2ae40850c2 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1119,6 +1119,10 @@ MINI_OP(OP_ATOMIC_STORE_R8, "atomic_store_r8", IREG, FREG, NONE) MINI_OP(OP_ATOMIC_ADD_I4, "atomic_add_i4", IREG, IREG, IREG) MINI_OP(OP_ATOMIC_ADD_I8, "atomic_add_i8", IREG, IREG, IREG) +MINI_OP(OP_ATOMIC_AND_I4, "atomic_and_i4", IREG, IREG, IREG) +MINI_OP(OP_ATOMIC_AND_I8, "atomic_and_i8", IREG, IREG, IREG) +MINI_OP(OP_ATOMIC_OR_I4, "atomic_or_i4", IREG, IREG, IREG) +MINI_OP(OP_ATOMIC_OR_I8, "atomic_or_i8", IREG, IREG, IREG) MINI_OP(OP_ATOMIC_EXCHANGE_I4, "atomic_exchange_i4", IREG, IREG, IREG) MINI_OP(OP_ATOMIC_EXCHANGE_I8, "atomic_exchange_i8", IREG, IREG, IREG) From f89d665626fa721a76f9a6ed56144f8dca5bd013 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 5 Mar 2020 22:42:13 +0300 Subject: [PATCH 2/3] turns out Interlocked.And and Interlocked.Or return old value --- src/mono/mono/mini/intrinsics.c | 14 +++++--------- src/mono/mono/mini/mini-llvm.c | 11 +++++------ 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/mono/mono/mini/intrinsics.c b/src/mono/mono/mini/intrinsics.c index 01559f1647ff55..9015eab5caa049 100644 --- a/src/mono/mono/mini/intrinsics.c +++ b/src/mono/mono/mini/intrinsics.c @@ -1190,20 +1190,19 @@ mini_emit_inst_for_method (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSign ((strcmp (cmethod->name, "Add") == 0) || (strcmp (cmethod->name, "And") == 0) || (strcmp (cmethod->name, "Or") == 0))) { - guint32 opcode = 0; guint32 opcode_i4 = 0; guint32 opcode_i8 = 0; gboolean llvm_only = FALSE; - if (strcmp(cmethod->name, "Add") == 0) { + if (strcmp (cmethod->name, "Add") == 0) { opcode_i4 = OP_ATOMIC_ADD_I4; opcode_i8 = OP_ATOMIC_ADD_I8; - } else if (strcmp(cmethod->name, "And") == 0) { + } else if (strcmp (cmethod->name, "And") == 0) { opcode_i4 = OP_ATOMIC_AND_I4; opcode_i8 = OP_ATOMIC_AND_I8; llvm_only = TRUE; - } else if (strcmp(cmethod->name, "And") == 0) { + } else if (strcmp (cmethod->name, "Or") == 0) { opcode_i4 = OP_ATOMIC_OR_I4; opcode_i8 = OP_ATOMIC_OR_I8; llvm_only = TRUE; @@ -1214,14 +1213,11 @@ mini_emit_inst_for_method (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSign if (fsig->params [0]->type == MONO_TYPE_I4) { opcode = opcode_i4; cfg->has_atomic_add_i4 = TRUE; - } - else if (fsig->params [0]->type == MONO_TYPE_I8 && SIZEOF_REGISTER == 8) { + } else if (fsig->params [0]->type == MONO_TYPE_I8 && SIZEOF_REGISTER == 8) { opcode = opcode_i8; } - if (opcode) { - if (!mono_arch_opcode_supported (opcode) && !llvm_only) - return NULL; + if (opcode && (mono_arch_opcode_supported (opcode) || llvm_only)) { MONO_INST_NEW (cfg, ins, opcode); ins->dreg = mono_alloc_ireg (cfg); ins->inst_basereg = args [0]->dreg; diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 14aa65a4d0104a..87772fbefb1a9f 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -6453,14 +6453,13 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) args [1] = convert (ctx, rhs, t); ARM64_ATOMIC_FENCE_FIX; if (ins->opcode == OP_ATOMIC_ADD_I4 || ins->opcode == OP_ATOMIC_ADD_I8) - values [ins->dreg] = LLVMBuildAdd (builder, - mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_ADD, args [0], args [1]), args [1], dname); + // Interlocked.Add returns new value (that's why we emit additional Add here) + // see https://github.com/dotnet/runtime/pull/33102 + values [ins->dreg] = LLVMBuildAdd (builder, mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_ADD, args [0], args [1]), args [1], dname); else if (ins->opcode == OP_ATOMIC_AND_I4 || ins->opcode == OP_ATOMIC_AND_I8) - values [ins->dreg] = LLVMBuildAnd (builder, - mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_AND, args [0], args [1]), args [1], dname); + values [ins->dreg] = mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_AND, args [0], args [1]); else if (ins->opcode == OP_ATOMIC_OR_I4 || ins->opcode == OP_ATOMIC_OR_I8) - values [ins->dreg] = LLVMBuildOr (builder, - mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_OR, args [0], args [1]), args [1], dname); + values [ins->dreg] = mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_OR, args [0], args [1]); else g_assert_not_reached (); ARM64_ATOMIC_FENCE_FIX; From 1466b2fae49d1965049011f82619e6ebf838831b Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 5 Mar 2020 23:51:28 +0300 Subject: [PATCH 3/3] Remove llvm_only check --- src/mono/mono/mini/intrinsics.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/mono/mono/mini/intrinsics.c b/src/mono/mono/mini/intrinsics.c index 9015eab5caa049..aa593298d3ac4b 100644 --- a/src/mono/mono/mini/intrinsics.c +++ b/src/mono/mono/mini/intrinsics.c @@ -1193,7 +1193,6 @@ mini_emit_inst_for_method (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSign guint32 opcode = 0; guint32 opcode_i4 = 0; guint32 opcode_i8 = 0; - gboolean llvm_only = FALSE; if (strcmp (cmethod->name, "Add") == 0) { opcode_i4 = OP_ATOMIC_ADD_I4; @@ -1201,11 +1200,9 @@ mini_emit_inst_for_method (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSign } else if (strcmp (cmethod->name, "And") == 0) { opcode_i4 = OP_ATOMIC_AND_I4; opcode_i8 = OP_ATOMIC_AND_I8; - llvm_only = TRUE; } else if (strcmp (cmethod->name, "Or") == 0) { opcode_i4 = OP_ATOMIC_OR_I4; opcode_i8 = OP_ATOMIC_OR_I8; - llvm_only = TRUE; } else { g_assert_not_reached (); } @@ -1217,7 +1214,8 @@ mini_emit_inst_for_method (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSign opcode = opcode_i8; } - if (opcode && (mono_arch_opcode_supported (opcode) || llvm_only)) { + // For now, only Add is supported in non-LLVM back-ends + if (opcode && (COMPILE_LLVM (cfg) || mono_arch_opcode_supported (opcode))) { MONO_INST_NEW (cfg, ins, opcode); ins->dreg = mono_alloc_ireg (cfg); ins->inst_basereg = args [0]->dreg;