diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 36f0fdcc45e55c..5469904c4ce554 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -10285,9 +10285,19 @@ void Compiler::impImportBlockCode(BasicBlock* block) case CEE_CPBLK: { GenTreeFlags indirFlags = impPrefixFlagsToIndirFlags(prefixFlags); - op3 = impPopStack().val; // Size - op2 = impPopStack().val; // Value / Src addr - op1 = impPopStack().val; // Dst addr + const bool isVolatile = (indirFlags & GTF_IND_VOLATILE) != 0; +#ifndef TARGET_X86 + if (isVolatile && !impStackTop(0).val->IsCnsIntOrI()) + { + // We're going to emit a helper call surrounded by memory barriers, so we need to spill any side + // effects. + impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("spilling side-effects")); + } +#endif + + op3 = impPopStack().val; // Size + op2 = impPopStack().val; // Value / Src addr + op1 = impPopStack().val; // Dst addr if (op3->IsCnsIntOrI()) { @@ -10324,6 +10334,29 @@ void Compiler::impImportBlockCode(BasicBlock* block) } else { + if (TARGET_POINTER_SIZE == 8) + { + // Cast size to TYP_LONG on 64-bit targets + op3 = gtNewCastNode(TYP_LONG, op3, /* fromUnsigned */ true, TYP_LONG); + } + +// TODO: enable for X86 as well, it currently doesn't support memset/memcpy helpers +// Then, get rid of GT_STORE_DYN_BLK entirely. +#ifndef TARGET_X86 + const unsigned helper = opcode == CEE_INITBLK ? CORINFO_HELP_MEMSET : CORINFO_HELP_MEMCPY; + if (isVolatile) + { + // Wrap with memory barriers: full-barrier + call + load-barrier + impAppendTree(gtNewMemoryBarrier(), CHECK_SPILL_ALL, impCurStmtDI); + impAppendTree(gtNewHelperCallNode(helper, TYP_VOID, op1, op2, op3), CHECK_SPILL_ALL, + impCurStmtDI); + op1 = gtNewMemoryBarrier(true); + } + else + { + op1 = gtNewHelperCallNode(helper, TYP_VOID, op1, op2, op3); + } +#else if (opcode == CEE_INITBLK) { if (!op2->IsIntegralConst(0)) @@ -10335,13 +10368,8 @@ void Compiler::impImportBlockCode(BasicBlock* block) { op2 = gtNewIndir(TYP_STRUCT, op2); } - -#ifdef TARGET_64BIT - // STORE_DYN_BLK takes a native uint size as it turns into call to memcpy. - op3 = gtNewCastNode(TYP_I_IMPL, op3, /* fromUnsigned */ true, TYP_I_IMPL); -#endif - op1 = gtNewStoreDynBlkNode(op1, op2, op3, indirFlags); +#endif } goto SPILL_APPEND; } diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 5acb6ff95112e1..18d7f388e908a7 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -1844,6 +1844,7 @@ GenTree* Lowering::AddrGen(void* addr) //------------------------------------------------------------------------ // LowerCallMemmove: Replace Buffer.Memmove(DST, SRC, CNS_SIZE) with a GT_STORE_BLK: +// Do the same for CORINFO_HELP_MEMCPY(DST, SRC, CNS_SIZE) // // * STORE_BLK struct (copy) (Unroll) // +--* LCL_VAR byref dst @@ -1860,7 +1861,8 @@ GenTree* Lowering::AddrGen(void* addr) bool Lowering::LowerCallMemmove(GenTreeCall* call, GenTree** next) { JITDUMP("Considering Memmove [%06d] for unrolling.. ", comp->dspTreeID(call)) - assert(comp->lookupNamedIntrinsic(call->gtCallMethHnd) == NI_System_Buffer_Memmove); + assert(call->IsHelperCall(comp, CORINFO_HELP_MEMCPY) || + (comp->lookupNamedIntrinsic(call->gtCallMethHnd) == NI_System_Buffer_Memmove)); assert(call->gtArgs.CountUserArgs() == 3); @@ -1894,7 +1896,8 @@ bool Lowering::LowerCallMemmove(GenTreeCall* call, GenTree** next) // TODO-CQ: Use GenTreeBlk::BlkOpKindUnroll here if srcAddr and dstAddr don't overlap, thus, we can // unroll this memmove as memcpy - it doesn't require lots of temp registers - storeBlk->gtBlkOpKind = GenTreeBlk::BlkOpKindUnrollMemmove; + storeBlk->gtBlkOpKind = call->IsHelperCall(comp, CORINFO_HELP_MEMCPY) ? GenTreeBlk::BlkOpKindUnroll + : GenTreeBlk::BlkOpKindUnrollMemmove; BlockRange().InsertBefore(call, srcBlk); BlockRange().InsertBefore(call, storeBlk); @@ -2215,16 +2218,22 @@ GenTree* Lowering::LowerCall(GenTree* node) } #if defined(TARGET_AMD64) || defined(TARGET_ARM64) + GenTree* nextNode = nullptr; if (call->gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC) { - GenTree* nextNode = nullptr; - NamedIntrinsic ni = comp->lookupNamedIntrinsic(call->gtCallMethHnd); + NamedIntrinsic ni = comp->lookupNamedIntrinsic(call->gtCallMethHnd); if (((ni == NI_System_Buffer_Memmove) && LowerCallMemmove(call, &nextNode)) || ((ni == NI_System_SpanHelpers_SequenceEqual) && LowerCallMemcmp(call, &nextNode))) { return nextNode; } } + + // Try to lower CORINFO_HELP_MEMCPY to unrollable STORE_BLK + if (call->IsHelperCall(comp, CORINFO_HELP_MEMCPY) && LowerCallMemmove(call, &nextNode)) + { + return nextNode; + } #endif call->ClearOtherRegs();