diff --git a/src/coreclr/inc/patchpointinfo.h b/src/coreclr/inc/patchpointinfo.h index 6e8d32938b27ab..735b476a2c3e10 100644 --- a/src/coreclr/inc/patchpointinfo.h +++ b/src/coreclr/inc/patchpointinfo.h @@ -47,6 +47,8 @@ struct PatchpointInfo m_keptAliveThisOffset = -1; m_securityCookieOffset = -1; m_monitorAcquiredOffset = -1; + m_asyncExecutionContextOffset = -1; + m_asyncSynchronizationContextOffset = -1; } // Copy @@ -58,6 +60,8 @@ struct PatchpointInfo m_keptAliveThisOffset = original->m_keptAliveThisOffset; m_securityCookieOffset = original->m_securityCookieOffset; m_monitorAcquiredOffset = original->m_monitorAcquiredOffset; + m_asyncExecutionContextOffset = original->m_asyncExecutionContextOffset; + m_asyncSynchronizationContextOffset = original->m_asyncSynchronizationContextOffset; for (uint32_t i = 0; i < original->m_numberOfLocals; i++) { diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index 8e7fb1f6b29f5e..8707b4a940eb74 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -348,11 +348,7 @@ class CodeGen final : public CodeGenInterface unsigned lclNum, unsigned offset, unsigned paramLclNum, const ABIPassingSegment& seg, class RegGraph* graph); void genSpillOrAddNonStandardRegisterParam(unsigned lclNum, regNumber sourceReg, class RegGraph* graph); void genEnregisterIncomingStackArgs(); -#if defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) void genEnregisterOSRArgsAndLocals(regNumber initReg, bool* pInitRegZeroed); -#else - void genEnregisterOSRArgsAndLocals(); -#endif void genHomeStackSegment(unsigned lclNum, const ABIPassingSegment& seg, regNumber initReg, bool* pInitRegZeroed); void genHomeSwiftStructStackParameters(); @@ -363,6 +359,7 @@ class CodeGen final : public CodeGenInterface void genClearStackVec3ArgUpperBits(); #endif // UNIX_AMD64_ABI && FEATURE_SIMD + void genPushCalleeSavedRegisters(regNumber initReg, bool* pInitRegZeroed); #if defined(TARGET_ARM64) bool genInstrWithConstant(instruction ins, emitAttr attr, @@ -384,15 +381,23 @@ class CodeGen final : public CodeGenInterface void genPrologSaveReg(regNumber reg1, int spOffset, int spDelta, regNumber tmpReg, bool* pTmpRegIsZero); - void genEpilogRestoreRegPair(regNumber reg1, - regNumber reg2, - int spOffset, - int spDelta, - bool useSaveNextPair, - regNumber tmpReg, - bool* pTmpRegIsZero); - - void genEpilogRestoreReg(regNumber reg1, int spOffset, int spDelta, regNumber tmpReg, bool* pTmpRegIsZero); + void genRestoreRegPair(regNumber reg1, + regNumber reg2, + regNumber baseReg, + int spOffset, + int spDelta, + bool useSaveNextPair, + regNumber tmpReg, + bool* pTmpRegIsZero, + bool reportUnwindData); + + void genRestoreReg(regNumber reg1, + regNumber baseReg, + int spOffset, + int spDelta, + regNumber tmpReg, + bool* pTmpRegIsZero, + bool reportUnwindData); // A simple struct to keep register pairs for prolog and epilog. struct RegPair @@ -423,13 +428,12 @@ class CodeGen final : public CodeGenInterface static int genGetSlotSizeForRegsInMask(regMaskTP regsMask); void genSaveCalleeSavedRegisterGroup(regMaskTP regsMask, int spDelta, int spOffset); - void genRestoreCalleeSavedRegisterGroup(regMaskTP regsMask, int spDelta, int spOffset); + void genRestoreCalleeSavedRegisterGroup( + regMaskTP regsMask, regNumber baseReg, int spDelta, int spOffset, bool reportUnwindData); void genSaveCalleeSavedRegistersHelp(regMaskTP regsToSaveMask, int lowestCalleeSavedOffset, int spDelta); void genRestoreCalleeSavedRegistersHelp(regMaskTP regsToRestoreMask, int lowestCalleeSavedOffset, int spDelta); - void genPushCalleeSavedRegisters(regNumber initReg, bool* pInitRegZeroed); - #elif defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) bool genInstrWithConstant(instruction ins, emitAttr attr, @@ -442,17 +446,14 @@ class CodeGen final : public CodeGenInterface void genStackPointerAdjustment(ssize_t spAdjustment, regNumber tmpReg, bool* pTmpRegIsZero, bool reportUnwindData); void genSaveCalleeSavedRegistersHelp(regMaskTP regsToSaveMask, int lowestCalleeSavedOffset); - void genRestoreCalleeSavedRegistersHelp(regMaskTP regsToRestoreMask, int lowestCalleeSavedOffset); - void genPushCalleeSavedRegisters(regNumber initReg, bool* pInitRegZeroed); - -#else - void genPushCalleeSavedRegisters(); + void genRestoreCalleeSavedRegistersHelp(regMaskTP regsToRestoreMask, + regNumber baseReg, + int lowestCalleeSavedOffset, + bool reportUnwindData); #endif -#if defined(TARGET_AMD64) - void genOSRRecordTier0CalleeSavedRegistersAndFrame(); + void genOSRHandleTier0CalleeSavedRegistersAndFrame(); void genOSRSaveRemainingCalleeSavedRegisters(); -#endif // TARGET_AMD64 void genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pInitRegZeroed, regMaskTP maskArgRegsLiveIn); diff --git a/src/coreclr/jit/codegenarm.cpp b/src/coreclr/jit/codegenarm.cpp index 2d2ab0026bd9b0..e28571af1629e2 100644 --- a/src/coreclr/jit/codegenarm.cpp +++ b/src/coreclr/jit/codegenarm.cpp @@ -1853,6 +1853,15 @@ void CodeGen::genProfilingLeaveCallback(unsigned helper) #endif // PROFILING_SUPPORTED +//------------------------------------------------------------------------ +// genOSRHandleTier0CalleeSavedRegistersAndFrame: +// Not called for arm without OSR support. +// +void CodeGen::genOSRHandleTier0CalleeSavedRegistersAndFrame() +{ + unreached(); +} + //------------------------------------------------------------------------ // genEstablishFramePointer: Set up the frame pointer by adding an offset to the stack pointer. // diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 15ee1ef3768a29..4f7eb08e69c1cf 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -132,7 +132,8 @@ void CodeGen::genPopCalleeSavedRegistersAndFreeLclFrame(bool jmpEpilog) // add sp,sp,#remainingFrameSz JITDUMP(" alignmentAdjustment2=%d\n", alignmentAdjustment2); - genEpilogRestoreRegPair(REG_FP, REG_LR, alignmentAdjustment2, spAdjustment2, false, REG_IP1, nullptr); + genRestoreRegPair(REG_FP, REG_LR, REG_SPBASE, alignmentAdjustment2, spAdjustment2, false, REG_IP1, + nullptr, /* reportUnwindData */ true); } else { @@ -153,8 +154,8 @@ void CodeGen::genPopCalleeSavedRegistersAndFreeLclFrame(bool jmpEpilog) JITDUMP(" remainingFrameSz=%d\n", remainingFrameSz); - genEpilogRestoreRegPair(REG_FP, REG_LR, m_compiler->lvaOutgoingArgSpaceSize, remainingFrameSz, false, - REG_IP1, nullptr); + genRestoreRegPair(REG_FP, REG_LR, REG_SPBASE, m_compiler->lvaOutgoingArgSpaceSize, remainingFrameSz, + false, REG_IP1, nullptr, /* reportUnwindData */ true); } // Unlike frameType=1 or frameType=2 that restore SP at the end, @@ -483,6 +484,7 @@ void CodeGen::genPrologSaveRegPair(regNumber reg1, if (spDelta != 0) { assert(!useSaveNextPair); + if ((spOffset == 0) && (spDelta >= -512)) { // We can use pre-indexed addressing. @@ -584,7 +586,7 @@ void CodeGen::genPrologSaveReg(regNumber reg1, int spOffset, int spDelta, regNum } //------------------------------------------------------------------------ -// genEpilogRestoreRegPair: This is the opposite of genPrologSaveRegPair(), run in the epilog instead of the prolog. +// genRestoreRegPair: This is the opposite of genPrologSaveRegPair(), run in the epilog instead of the prolog. // The stack pointer adjustment, if requested, is done after the register restore, using post-index addressing. // The caller must ensure that we can use the LDP instruction, and that spOffset will be in the legal range for that // instruction. @@ -592,7 +594,8 @@ void CodeGen::genPrologSaveReg(regNumber reg1, int spOffset, int spDelta, regNum // Arguments: // reg1 - First register of pair to restore. // reg2 - Second register of pair to restore. -// spOffset - The offset from SP to load reg1 (must be positive or zero). +// baseReg - Base register to load values from +// spOffset - The offset from the base register to load reg1 // spDelta - If non-zero, the amount to add to SP after the register restores (must be positive or // zero). // useSaveNextPair - True if the last prolog instruction was to save the previous register pair. This @@ -604,15 +607,17 @@ void CodeGen::genPrologSaveReg(regNumber reg1, int spOffset, int spDelta, regNum // Return Value: // None. -void CodeGen::genEpilogRestoreRegPair(regNumber reg1, - regNumber reg2, - int spOffset, - int spDelta, - bool useSaveNextPair, - regNumber tmpReg, - bool* pTmpRegIsZero) +void CodeGen::genRestoreRegPair(regNumber reg1, + regNumber reg2, + regNumber baseReg, + int spOffset, + int spDelta, + bool useSaveNextPair, + regNumber tmpReg, + bool* pTmpRegIsZero, + bool reportUnwindData) { - assert(spOffset >= 0); + assert((spOffset >= -512) && (spOffset <= 504)); assert(spDelta >= 0); assert((spDelta % 16) == 0); // SP changes must be 16-byte aligned assert(genIsValidFloatReg(reg1) == genIsValidFloatReg(reg2)); // registers must be both general-purpose, or both @@ -625,43 +630,53 @@ void CodeGen::genEpilogRestoreRegPair(regNumber reg1, { // Fold the SP change into this instruction. // ldp reg1, reg2, [SP], #spDelta - GetEmitter()->emitIns_R_R_R_I(INS_ldp, EA_PTRSIZE, reg1, reg2, REG_SPBASE, spDelta, INS_OPTS_POST_INDEX); - m_compiler->unwindSaveRegPairPreindexed(reg1, reg2, -spDelta); + GetEmitter()->emitIns_R_R_R_I(INS_ldp, EA_PTRSIZE, reg1, reg2, baseReg, spDelta, INS_OPTS_POST_INDEX); + + if (reportUnwindData) + { + m_compiler->unwindSaveRegPairPreindexed(reg1, reg2, -spDelta); + } } else // (spOffset != 0) || (spDelta > 504) { // Can't fold in the SP change; need to use a separate ADD instruction. // ldp reg1, reg2, [SP, #offset] - GetEmitter()->emitIns_R_R_R_I(INS_ldp, EA_PTRSIZE, reg1, reg2, REG_SPBASE, spOffset); - m_compiler->unwindSaveRegPair(reg1, reg2, spOffset); + GetEmitter()->emitIns_R_R_R_I(INS_ldp, EA_PTRSIZE, reg1, reg2, baseReg, spOffset); + if (reportUnwindData) + { + m_compiler->unwindSaveRegPair(reg1, reg2, spOffset); + } // generate add SP,SP,imm - genStackPointerAdjustment(spDelta, tmpReg, pTmpRegIsZero, /* reportUnwindData */ true); + genStackPointerAdjustment(spDelta, tmpReg, pTmpRegIsZero, /* reportUnwindData */ reportUnwindData); } } else { - GetEmitter()->emitIns_R_R_R_I(INS_ldp, EA_PTRSIZE, reg1, reg2, REG_SPBASE, spOffset); + GetEmitter()->emitIns_R_R_R_I(INS_ldp, EA_PTRSIZE, reg1, reg2, baseReg, spOffset); - if (TargetOS::IsUnix && m_compiler->generateCFIUnwindCodes()) + if (reportUnwindData) { - useSaveNextPair = false; - } + if (TargetOS::IsUnix && m_compiler->generateCFIUnwindCodes()) + { + useSaveNextPair = false; + } - if (useSaveNextPair) - { - m_compiler->unwindSaveNext(); - } - else - { - m_compiler->unwindSaveRegPair(reg1, reg2, spOffset); + if (useSaveNextPair) + { + m_compiler->unwindSaveNext(); + } + else + { + m_compiler->unwindSaveRegPair(reg1, reg2, spOffset); + } } } } //------------------------------------------------------------------------ -// genEpilogRestoreReg: The opposite of genPrologSaveReg(), run in the epilog instead of the prolog. +// genRestoreReg: The opposite of genPrologSaveReg(), run in the epilog instead of the prolog. // // Arguments: // reg1 - Register to restore. @@ -675,9 +690,14 @@ void CodeGen::genEpilogRestoreRegPair(regNumber reg1, // Return Value: // None. -void CodeGen::genEpilogRestoreReg(regNumber reg1, int spOffset, int spDelta, regNumber tmpReg, bool* pTmpRegIsZero) +void CodeGen::genRestoreReg(regNumber reg1, + regNumber baseReg, + int spOffset, + int spDelta, + regNumber tmpReg, + bool* pTmpRegIsZero, + bool reportUnwindData) { - assert(spOffset >= 0); assert(spDelta >= 0); assert((spDelta % 16) == 0); // SP changes must be 16-byte aligned @@ -687,24 +707,36 @@ void CodeGen::genEpilogRestoreReg(regNumber reg1, int spOffset, int spDelta, reg { // We can use post-index addressing. // ldr REG, [SP], #spDelta - GetEmitter()->emitIns_R_R_I(INS_ldr, EA_PTRSIZE, reg1, REG_SPBASE, spDelta, INS_OPTS_POST_INDEX); - m_compiler->unwindSaveRegPreindexed(reg1, -spDelta); + GetEmitter()->emitIns_R_R_I(INS_ldr, EA_PTRSIZE, reg1, baseReg, spDelta, INS_OPTS_POST_INDEX); + + if (reportUnwindData) + { + m_compiler->unwindSaveRegPreindexed(reg1, -spDelta); + } } else // (spOffset != 0) || (spDelta > 255) { // ldr reg1, [SP, #offset] - GetEmitter()->emitIns_R_R_I(INS_ldr, EA_PTRSIZE, reg1, REG_SPBASE, spOffset); - m_compiler->unwindSaveReg(reg1, spOffset); + GetEmitter()->emitIns_R_R_I(INS_ldr, EA_PTRSIZE, reg1, baseReg, spOffset); + + if (reportUnwindData) + { + m_compiler->unwindSaveReg(reg1, spOffset); + } // generate add SP,SP,imm - genStackPointerAdjustment(spDelta, tmpReg, pTmpRegIsZero, /* reportUnwindData */ true); + genStackPointerAdjustment(spDelta, tmpReg, pTmpRegIsZero, reportUnwindData); } } else { // ldr reg1, [SP, #offset] - GetEmitter()->emitIns_R_R_I(INS_ldr, EA_PTRSIZE, reg1, REG_SPBASE, spOffset); - m_compiler->unwindSaveReg(reg1, spOffset); + GetEmitter()->emitIns_R_R_I(INS_ldr, EA_PTRSIZE, reg1, baseReg, spOffset); + + if (reportUnwindData) + { + m_compiler->unwindSaveReg(reg1, spOffset); + } } } @@ -968,9 +1000,10 @@ void CodeGen::genSaveCalleeSavedRegistersHelp(regMaskTP regsToSaveMask, int lowe // Arguments: // regsMask - a mask of registers for epilog generation; // spDelta - if non-zero, the amount to add to SP after the last register restore (or together with it); -// spOffset - the offset from SP that is the beginning of the callee-saved register area; +// spOffset - the offset from SP that is the top of the callee-saved register area; // -void CodeGen::genRestoreCalleeSavedRegisterGroup(regMaskTP regsMask, int spDelta, int spOffset) +void CodeGen::genRestoreCalleeSavedRegisterGroup( + regMaskTP regsMask, regNumber baseReg, int spDelta, int spOffset, bool reportUnwindData) { const int slotSize = genGetSlotSizeForRegsInMask(regsMask); @@ -996,18 +1029,19 @@ void CodeGen::genRestoreCalleeSavedRegisterGroup(regMaskTP regsMask, int spDelta if (genReverseAndPairCalleeSavedRegisters) { - genEpilogRestoreRegPair(regPair.reg2, regPair.reg1, spOffset, stackDelta, false, REG_IP1, nullptr); + genRestoreRegPair(regPair.reg2, regPair.reg1, baseReg, spOffset, stackDelta, false, REG_IP1, nullptr, + reportUnwindData); } else { - genEpilogRestoreRegPair(regPair.reg1, regPair.reg2, spOffset, stackDelta, regPair.useSaveNextPair, - REG_IP1, nullptr); + genRestoreRegPair(regPair.reg1, regPair.reg2, baseReg, spOffset, stackDelta, regPair.useSaveNextPair, + REG_IP1, nullptr, reportUnwindData); } } else { spOffset -= slotSize; - genEpilogRestoreReg(regPair.reg1, spOffset, stackDelta, REG_IP1, nullptr); + genRestoreReg(regPair.reg1, baseReg, spOffset, stackDelta, REG_IP1, nullptr, reportUnwindData); } } } @@ -1079,20 +1113,23 @@ void CodeGen::genRestoreCalleeSavedRegistersHelp(regMaskTP regsToRestoreMask, in { int spFrameDelta = (maskRestoreRegsFloat != RBM_NONE || maskRestoreRegsInt != RBM_NONE) ? 0 : spDelta; spOffset -= 2 * REGSIZE_BYTES; - genEpilogRestoreRegPair(REG_FP, REG_LR, spOffset, spFrameDelta, false, REG_IP1, nullptr); + genRestoreRegPair(REG_FP, REG_LR, REG_SPBASE, spOffset, spFrameDelta, false, REG_IP1, nullptr, + /* reportUnwindData */ true); } if (maskRestoreRegsInt != RBM_NONE) { int spIntDelta = (maskRestoreRegsFloat != RBM_NONE) ? 0 : spDelta; // should we delay the SP adjustment? - genRestoreCalleeSavedRegisterGroup(maskRestoreRegsInt, spIntDelta, spOffset); + genRestoreCalleeSavedRegisterGroup(maskRestoreRegsInt, REG_SPBASE, spIntDelta, spOffset, + /* reportUnwindData */ true); spOffset -= genCountBits(maskRestoreRegsInt) * REGSIZE_BYTES; } if (maskRestoreRegsFloat != RBM_NONE) { // If there is any spDelta, it must be used here. - genRestoreCalleeSavedRegisterGroup(maskRestoreRegsFloat, spDelta, spOffset); + genRestoreCalleeSavedRegisterGroup(maskRestoreRegsFloat, REG_SPBASE, spDelta, spOffset, + /* reportUnwindData */ true); // No need to update spOffset since it's not used after this. } } @@ -5532,6 +5569,90 @@ void CodeGen::genStoreLclTypeSimd12(GenTreeLclVarCommon* treeNode) #endif // FEATURE_SIMD +//----------------------------------------------------------------------------- +// genOSRHandleTier0CalleeSavedRegistersAndFrame: +// Handle the tier0 callee saves by restoring them from the original tier0 frame. +// Also report phantom unwind data for the allocated stack by the tier0 frame. +// +void CodeGen::genOSRHandleTier0CalleeSavedRegistersAndFrame() +{ + assert(m_compiler->compGeneratingProlog); + assert(m_compiler->opts.IsOSR()); + assert(m_compiler->funCurrentFunc()->funKind == FuncKind::FUNC_ROOT); + + PatchpointInfo* const patchpointInfo = m_compiler->info.compPatchpointInfo; + regMaskTP const tier0CalleeSaves((regMaskSmall)patchpointInfo->CalleeSaveRegisters()); + + JITDUMP("--OSR--- tier0 has already saved "); + JITDUMPEXEC(dspRegMask(tier0CalleeSaves)); + JITDUMP("\nEmitting restores\n"); + + // Note: the restore of LR relies on the tier0 method having been unhijacked when the OSR method prolog runs. + // This happens in the transition helper. If transition helper is not used (e.g. because we directly jump into OSR) + // then hijacking tier0 is not supported -- this is similar to tailcalls so the situation can be recorded via + // SetHasTailCalls. + + regMaskTP restoreRegsFrame = tier0CalleeSaves & (RBM_FP | RBM_LR); + regMaskTP restoreRegsFloat = tier0CalleeSaves & RBM_ALLFLOAT; + regMaskTP restoreRegsInt = tier0CalleeSaves & ~restoreRegsFrame & ~restoreRegsFloat; + + regNumber baseReg; + int topOfCalleeSaves; + if (restoreRegsFrame != RBM_NONE) + { + // FP/LR was saved with the callee saves. It is always at the top. + // Restore rest of callee saves with the offset from FP. + baseReg = REG_FP; + topOfCalleeSaves = 0; + } + else + { + // FP/LR was not saved with the callee saves. Here we do not actually + // know the offset from FP to the callee saves, but we do know the + // offset from SP. + baseReg = REG_SP; + topOfCalleeSaves = patchpointInfo->TotalFrameSize(); + if (m_compiler->info.compIsVarArgs) + { + topOfCalleeSaves -= MAX_REG_ARG * REGSIZE_BYTES; + } + + if ((topOfCalleeSaves > 504) && ((restoreRegsInt != RBM_NONE) || (restoreRegsFloat != RBM_NONE))) + { + // Too far to encode ldp with sp directly. Compute top into another register. + // Note: not reporting unwind nops for this as we will pad below anyway. + genInstrWithConstant(INS_add, EA_PTRSIZE, REG_IP0, REG_SP, topOfCalleeSaves, REG_IP0, + /* inUnwindRegion */ false); + baseReg = REG_IP0; + topOfCalleeSaves = 0; + } + } + + if (restoreRegsInt != RBM_NONE) + { + genRestoreCalleeSavedRegisterGroup(restoreRegsInt, baseReg, 0, topOfCalleeSaves, /* reportUnwindData */ false); + topOfCalleeSaves -= genCountBits(restoreRegsInt) * REGSIZE_BYTES; + } + + if (restoreRegsFloat != RBM_NONE) + { + genRestoreCalleeSavedRegisterGroup(restoreRegsFloat, baseReg, 0, topOfCalleeSaves, + /* reportUnwindData */ false); + topOfCalleeSaves -= genCountBits(restoreRegsFloat) * REGSIZE_BYTES; + } + + // Regardless of frame type fp always points to the saved fp/lr for frame + // pointer chaining purposes, so restoring them is trivial. + genRestoreRegPair(REG_FP, REG_LR, REG_FP, 0, 0, false, REG_IP1, nullptr, + /* reportUnwindData */ false); + + // Emit phantom unwind data for the tier0 frame. + m_compiler->unwindAllocStack(patchpointInfo->TotalFrameSize()); + // Emit nops to make the prolog 1:1 in unwind codes to instructions. This + // is needed for win-arm64. + m_compiler->unwindPadding(); +} + #ifdef PROFILING_SUPPORTED //----------------------------------------------------------------------------------- diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index ff1f2ccd7131ee..60abe7fa996839 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -4285,16 +4285,12 @@ void CodeGen::genSIMDSplitReturn(GenTree* src, const ReturnTypeDesc* retTypeDesc //------------------------------------------------------------------------ // genPushCalleeSavedRegisters: Push any callee-saved registers we have used. // -// Arguments (arm64): +// Arguments: // initReg - A scratch register (that gets set to zero on some platforms). // pInitRegZeroed - OUT parameter. *pInitRegZeroed is set to 'true' if this method sets initReg register to zero, // 'false' if initReg was set to a non-zero value, and left unchanged if initReg was not touched. // -#if defined(TARGET_ARM64) void CodeGen::genPushCalleeSavedRegisters(regNumber initReg, bool* pInitRegZeroed) -#else -void CodeGen::genPushCalleeSavedRegisters() -#endif { assert(m_compiler->compGeneratingProlog); @@ -4778,6 +4774,7 @@ void CodeGen::genPushCalleeSavedRegisters() JITDUMP(" spAdjustment2=%d\n", spAdjustment2); genPrologSaveRegPair(REG_FP, REG_LR, alignmentAdjustment2, -spAdjustment2, false, initReg, pInitRegZeroed); + offset += spAdjustment2; // Now subtract off the #outsz (or the rest of the #outsz if it was unaligned, and the above "sub" @@ -4804,6 +4801,7 @@ void CodeGen::genPushCalleeSavedRegisters() { genPrologSaveRegPair(REG_FP, REG_LR, m_compiler->lvaOutgoingArgSpaceSize, -remainingFrameSz, false, initReg, pInitRegZeroed); + offset += remainingFrameSz; offsetSpToSavedFp = m_compiler->lvaOutgoingArgSpaceSize; diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 8cb24d54903b7f..9d7b4cb19ede5d 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -4135,11 +4135,7 @@ void CodeGen::genZeroInitFrame(int untrLclHi, int untrLclLo, regNumber initReg, // initReg -- scratch register to use if needed // pInitRegZeroed -- [IN,OUT] if init reg is zero (on entry/exit) // -#if defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) void CodeGen::genEnregisterOSRArgsAndLocals(regNumber initReg, bool* pInitRegZeroed) -#else -void CodeGen::genEnregisterOSRArgsAndLocals() -#endif { assert(m_compiler->opts.IsOSR()); PatchpointInfo* const patchpointInfo = m_compiler->info.compPatchpointInfo; @@ -5053,23 +5049,6 @@ void CodeGen::genFnProlog() genBeginFnProlog(); -#if defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) - // For arm64 OSR, emit a "phantom prolog" to account for the actions taken - // in the tier0 frame that impact FP and SP on entry to the OSR method. - // - // x64 handles this differently; the phantom prolog unwind is emitted in - // genOSRRecordTier0CalleeSavedRegistersAndFrame. - // - if (m_compiler->opts.IsOSR()) - { - PatchpointInfo* patchpointInfo = m_compiler->info.compPatchpointInfo; - const int tier0FrameSize = patchpointInfo->TotalFrameSize(); - - // SP is tier0 method's SP. - m_compiler->unwindAllocStack(tier0FrameSize); - } -#endif // defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) - #ifdef DEBUG if (m_compiler->compJitHaltMethod()) @@ -5320,11 +5299,7 @@ void CodeGen::genFnProlog() const bool isRoot = (m_compiler->funCurrentFunc()->funKind == FuncKind::FUNC_ROOT); -#ifdef TARGET_AMD64 - const bool isOSRx64Root = isRoot && m_compiler->opts.IsOSR(); -#else - const bool isOSRx64Root = false; -#endif // TARGET_AMD64 + const bool inheritsCalleeSaves = isRoot && m_compiler->opts.IsOSR(); regMaskTP tempMask = initRegs & RBM_ALLINT & ~excludeMask & ~regSet.rsMaskResvd; @@ -5348,37 +5323,16 @@ void CodeGen::genFnProlog() } } -#if defined(TARGET_AMD64) - // For x64 OSR root frames, we can't use any as of yet unsaved + // For OSR root frames, we can't use any as of yet unsaved // callee save as initReg, as we defer saving these until later in // the prolog, and we don't have normal arg regs. - if (isOSRx64Root) - { - initReg = REG_SCRATCH; // REG_EAX - } -#elif defined(TARGET_ARM64) - // For arm64 OSR root frames, we may need a scratch register for large - // offset addresses. Use a register that won't be allocated. - // - if (isRoot && m_compiler->opts.IsOSR()) - { - initReg = REG_IP1; - } -#elif defined(TARGET_LOONGARCH64) - // For LoongArch64 OSR root frames, we may need a scratch register for large - // offset addresses. Use a register that won't be allocated. - if (isRoot && m_compiler->opts.IsOSR()) + if (inheritsCalleeSaves) { initReg = REG_SCRATCH; - } -#elif defined(TARGET_RISCV64) - // For RISC-V64 OSR root frames, we may need a scratch register for large - // offset addresses. Use a register that won't be allocated. - if (isRoot && m_compiler->opts.IsOSR()) - { - initReg = REG_SCRATCH; // REG_T0 - } +#if defined(TARGET_ARM64) + initReg = REG_IP1; #endif + } #if defined(TARGET_AMD64) // If we are a varargs call, in order to set up the arguments correctly this @@ -5412,30 +5366,29 @@ void CodeGen::genFnProlog() } #endif // TARGET_ARM #else // TARGET_WASM - regNumber initReg = REG_NA; - bool initRegZeroed = false; - bool isOSRx64Root = false; + regNumber initReg = REG_NA; + bool initRegZeroed = false; + bool inheritsCalleeSaves = false; #endif // TARGET_WASM unsigned extraFrameSize = 0; -#ifdef TARGET_XARCH - -#ifdef TARGET_AMD64 - if (isOSRx64Root) + if (inheritsCalleeSaves) { // Account for the Tier0 callee saves // - genOSRRecordTier0CalleeSavedRegistersAndFrame(); + genOSRHandleTier0CalleeSavedRegistersAndFrame(); +#ifdef TARGET_AMD64 // We don't actually push any callee saves on the OSR frame, // but we still reserve space, so account for this when // allocating the local frame. // extraFrameSize = m_compiler->compCalleeRegsPushed * REGSIZE_BYTES; +#endif } -#endif // TARGET_AMD64 +#ifdef TARGET_XARCH if (doubleAlignOrFramePointerUsed()) { // OSR methods handle "saving" FP specially. @@ -5444,7 +5397,7 @@ void CodeGen::genFnProlog() // Tier0 method. The save we do here is just to set up a // proper RBP-based frame chain link. // - if (isOSRx64Root && isFramePointerUsed()) + if (inheritsCalleeSaves && isFramePointerUsed()) { GetEmitter()->emitIns_R_AR(INS_mov, EA_8BYTE, initReg, REG_FPBASE, 0); inst_RV(INS_push, initReg, TYP_REF); @@ -5460,9 +5413,10 @@ void CodeGen::genFnProlog() inst_RV(INS_push, REG_FPBASE, TYP_REF); m_compiler->unwindPush(REG_FPBASE); } -#ifndef TARGET_AMD64 // On AMD64, establish the frame pointer after the "sub rsp" +#ifdef TARGET_X86 + // On x86 establish frame pointer now. For x64 we establish it after the "sub rsp". genEstablishFramePointer(0, /*reportUnwindData*/ true); -#endif // !TARGET_AMD64 +#endif // TARGET_X86 #if DOUBLE_ALIGN if (m_compiler->genDoubleAlign()) @@ -5476,16 +5430,20 @@ void CodeGen::genFnProlog() } #endif // TARGET_XARCH -#if defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) - genPushCalleeSavedRegisters(initReg, &initRegZeroed); - -#else // !TARGET_ARM64 && !TARGET_LOONGARCH64 && !TARGET_RISCV64 + bool pushesCalleeSaves = true; +#ifdef TARGET_AMD64 + // For OSR x64 we need canonical epilogs (sequence of pops). Hence we do + // not push any register in the prolog, we rather store them in the area + // allocated by the tier0 method. For OSR on other platforms we have no + // such requirement, instead we restore tier0 saved callee saves from its + // area on entry and then run the prolog as normal. + pushesCalleeSaves = !inheritsCalleeSaves; +#endif - if (!isOSRx64Root) + if (pushesCalleeSaves) { - genPushCalleeSavedRegisters(); + genPushCalleeSavedRegisters(initReg, &initRegZeroed); } -#endif // !TARGET_ARM64 && !TARGET_LOONGARCH64 && !TARGET_RISCV64 #ifdef TARGET_ARM bool needToEstablishFP = false; @@ -5530,14 +5488,14 @@ void CodeGen::genFnProlog() } #endif // !TARGET_ARM64 && !TARGET_LOONGARCH64 && !TARGET_RISCV64 -#ifdef TARGET_AMD64 - // For x64 OSR we have to finish saving int callee saves. + // For x64 OSR we have to finish saving callee saves. // - if (isOSRx64Root) +#ifdef TARGET_AMD64 + if (inheritsCalleeSaves) { genOSRSaveRemainingCalleeSavedRegisters(); } -#endif // TARGET_AMD64 +#endif //------------------------------------------------------------------------- @@ -5667,12 +5625,7 @@ void CodeGen::genFnProlog() // we've set the live-in regs with values from the Tier0 frame. // // Otherwise we'll do some of these fetches twice. - -#if defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) genEnregisterOSRArgsAndLocals(initReg, &initRegZeroed); -#else - genEnregisterOSRArgsAndLocals(); -#endif // OSR functions take no parameters in registers. Ensure no mappings // are present. assert((m_compiler->m_paramRegLocalMappings == nullptr) || m_compiler->m_paramRegLocalMappings->Empty()); diff --git a/src/coreclr/jit/codegenloongarch64.cpp b/src/coreclr/jit/codegenloongarch64.cpp index 2b3cc0821a8791..f075858ec24c21 100644 --- a/src/coreclr/jit/codegenloongarch64.cpp +++ b/src/coreclr/jit/codegenloongarch64.cpp @@ -257,7 +257,9 @@ void CodeGen::genSaveCalleeSavedRegistersHelp(regMaskTP regsToSaveMask, int lowe // // Arguments: // regsToRestoreMask - The mask of callee-saved registers to restore. If empty, this function does nothing. +// baseReg - Base register to use when loading values // lowestCalleeSavedOffset - The offset from SP that is the beginning of the callee-saved register area. +// reportUnwindData - If true, report the change in unwind data. Otherwise, do not report it. // // Here's an example restore sequence: // ld.d s8,sp,#xxx @@ -273,7 +275,10 @@ void CodeGen::genSaveCalleeSavedRegistersHelp(regMaskTP regsToSaveMask, int lowe // Return Value: // None. -void CodeGen::genRestoreCalleeSavedRegistersHelp(regMaskTP regsToRestoreMask, int lowestCalleeSavedOffset) +void CodeGen::genRestoreCalleeSavedRegistersHelp(regMaskTP regsToRestoreMask, + regNumber baseReg, + int lowestCalleeSavedOffset, + bool reportUnwindData) { // The FP and RA are not in RBM_CALLEE_SAVED. assert(!(regsToRestoreMask & (~RBM_CALLEE_SAVED))); @@ -294,8 +299,12 @@ void CodeGen::genRestoreCalleeSavedRegistersHelp(regMaskTP regsToRestoreMask, in if (maskSaveRegs < 0) { highestCalleeSavedOffset -= REGSIZE_BYTES; - emit->emitIns_R_R_I(INS_fld_d, EA_8BYTE, (regNumber)regNum, REG_SP, highestCalleeSavedOffset); - m_compiler->unwindSaveReg((regNumber)regNum, highestCalleeSavedOffset); + emit->emitIns_R_R_I(INS_fld_d, EA_8BYTE, (regNumber)regNum, baseReg, highestCalleeSavedOffset); + + if (reportUnwindData) + { + m_compiler->unwindSaveReg((regNumber)regNum, highestCalleeSavedOffset); + } } maskSaveRegs <<= 1; regNum -= 1; @@ -309,8 +318,12 @@ void CodeGen::genRestoreCalleeSavedRegistersHelp(regMaskTP regsToRestoreMask, in if (maskSaveRegs < 0) { highestCalleeSavedOffset -= REGSIZE_BYTES; - emit->emitIns_R_R_I(INS_ld_d, EA_8BYTE, (regNumber)regNum, REG_SP, highestCalleeSavedOffset); - m_compiler->unwindSaveReg((regNumber)regNum, highestCalleeSavedOffset); + emit->emitIns_R_R_I(INS_ld_d, EA_8BYTE, (regNumber)regNum, baseReg, highestCalleeSavedOffset); + + if (reportUnwindData) + { + m_compiler->unwindSaveReg((regNumber)regNum, highestCalleeSavedOffset); + } } maskSaveRegs <<= 1; regNum -= 1; @@ -319,6 +332,31 @@ void CodeGen::genRestoreCalleeSavedRegistersHelp(regMaskTP regsToRestoreMask, in assert(highestCalleeSavedOffset >= 16); // the callee-saved regs always above ra/fp. } +//----------------------------------------------------------------------------- +// genOSRHandleTier0CalleeSavedRegistersAndFrame: +// Handle the tier0 callee saves by restoring them from the original tier0 frame. +// Also report phantom unwind data for the allocated stack by the tier0 frame. +// +void CodeGen::genOSRHandleTier0CalleeSavedRegistersAndFrame() +{ + assert(m_compiler->compGeneratingProlog); + assert(m_compiler->opts.IsOSR()); + assert(m_compiler->funCurrentFunc()->funKind == FuncKind::FUNC_ROOT); + + PatchpointInfo* const patchpointInfo = m_compiler->info.compPatchpointInfo; + regMaskTP tier0CalleeSaves(patchpointInfo->CalleeSaveRegisters()); + + JITDUMP("--OSR--- tier0 has already saved "); + JITDUMPEXEC(dspRegMask(tier0CalleeSaves)); + JITDUMP("\nEmitting restores\n"); + + genRestoreCalleeSavedRegistersHelp(tier0CalleeSaves & ~(RBM_FP | RBM_RA), REG_FP, 16, /* reportUnwindData */ false); + GetEmitter()->emitIns_R_R_I(INS_ld_d, EA_PTRSIZE, REG_RA, REG_FP, 8); + GetEmitter()->emitIns_R_R_I(INS_ld_d, EA_PTRSIZE, REG_FP, REG_FP, 0); + + m_compiler->unwindAllocStack(patchpointInfo->TotalFrameSize()); +} + // clang-format off /***************************************************************************** * @@ -504,7 +542,7 @@ void CodeGen::genFuncletEpilog(BasicBlock* /* block */) FP_offset = FP_offset & 0xf; } - genRestoreCalleeSavedRegistersHelp(maskSaveRegs, FP_offset + 16); + genRestoreCalleeSavedRegistersHelp(maskSaveRegs, REG_SPBASE, FP_offset + 16, /* reportUnwindData */ true); GetEmitter()->emitIns_R_R_I(INS_ld_d, EA_PTRSIZE, REG_RA, REG_SPBASE, FP_offset + 8); m_compiler->unwindSaveReg(REG_RA, FP_offset + 8); @@ -6846,7 +6884,7 @@ void CodeGen::genPopCalleeSavedRegisters(bool jmpEpilog) } JITDUMP(" calleeSaveSPOffset=%d\n", FP_offset + 16); - genRestoreCalleeSavedRegistersHelp(regsToRestoreMask, FP_offset + 16); + genRestoreCalleeSavedRegistersHelp(regsToRestoreMask, REG_SPBASE, FP_offset + 16, /* reportUnwindData */ true); emit->emitIns_R_R_I(INS_ld_d, EA_PTRSIZE, REG_RA, REG_SPBASE, FP_offset + 8); m_compiler->unwindSaveReg(REG_RA, FP_offset + 8); diff --git a/src/coreclr/jit/codegenriscv64.cpp b/src/coreclr/jit/codegenriscv64.cpp index e88bee5a01c991..03ea018a131f4a 100644 --- a/src/coreclr/jit/codegenriscv64.cpp +++ b/src/coreclr/jit/codegenriscv64.cpp @@ -245,7 +245,9 @@ void CodeGen::genSaveCalleeSavedRegistersHelp(regMaskTP regsToSaveMask, int lowe // // Arguments: // regsToRestoreMask - The mask of callee-saved registers to restore. If empty, this function does nothing. +// baseReg - Base register to use when loading values // lowestCalleeSavedOffset - The offset from SP that is the beginning of the callee-saved register area. +// reportUnwindData - If true, report the change in unwind data. Otherwise, do not report it. // // Here's an example restore sequence: // ld s11, #xxx(sp) @@ -263,7 +265,10 @@ void CodeGen::genSaveCalleeSavedRegistersHelp(regMaskTP regsToSaveMask, int lowe // Return Value: // None. -void CodeGen::genRestoreCalleeSavedRegistersHelp(regMaskTP regsToRestoreMask, int lowestCalleeSavedOffset) +void CodeGen::genRestoreCalleeSavedRegistersHelp(regMaskTP regsToRestoreMask, + regNumber baseReg, + int lowestCalleeSavedOffset, + bool reportUnwindData) { // The FP and RA are not in RBM_CALLEE_SAVED. assert(!(regsToRestoreMask & (~RBM_CALLEE_SAVED))); @@ -284,8 +289,12 @@ void CodeGen::genRestoreCalleeSavedRegistersHelp(regMaskTP regsToRestoreMask, in if (maskSaveRegs < 0) { highestCalleeSavedOffset -= REGSIZE_BYTES; - emit->emitIns_R_R_I(INS_fld, EA_8BYTE, (regNumber)regNum, REG_SP, highestCalleeSavedOffset); - m_compiler->unwindSaveReg((regNumber)regNum, highestCalleeSavedOffset); + emit->emitIns_R_R_I(INS_fld, EA_8BYTE, (regNumber)regNum, baseReg, highestCalleeSavedOffset); + + if (reportUnwindData) + { + m_compiler->unwindSaveReg((regNumber)regNum, highestCalleeSavedOffset); + } } maskSaveRegs <<= 1; regNum -= 1; @@ -299,8 +308,12 @@ void CodeGen::genRestoreCalleeSavedRegistersHelp(regMaskTP regsToRestoreMask, in if (maskSaveRegs < 0) { highestCalleeSavedOffset -= REGSIZE_BYTES; - emit->emitIns_R_R_I(INS_ld, EA_8BYTE, (regNumber)regNum, REG_SP, highestCalleeSavedOffset); - m_compiler->unwindSaveReg((regNumber)regNum, highestCalleeSavedOffset); + emit->emitIns_R_R_I(INS_ld, EA_8BYTE, (regNumber)regNum, baseReg, highestCalleeSavedOffset); + + if (reportUnwindData) + { + m_compiler->unwindSaveReg((regNumber)regNum, highestCalleeSavedOffset); + } } maskSaveRegs <<= 1; regNum -= 1; @@ -309,6 +322,31 @@ void CodeGen::genRestoreCalleeSavedRegistersHelp(regMaskTP regsToRestoreMask, in assert(highestCalleeSavedOffset >= 16); // the callee-saved regs always above ra/fp. } +//----------------------------------------------------------------------------- +// genOSRHandleTier0CalleeSavedRegistersAndFrame: +// Handle the tier0 callee saves by restoring them from the original tier0 frame. +// Also report phantom unwind data for the allocated stack by the tier0 frame. +// +void CodeGen::genOSRHandleTier0CalleeSavedRegistersAndFrame() +{ + assert(m_compiler->compGeneratingProlog); + assert(m_compiler->opts.IsOSR()); + assert(m_compiler->funCurrentFunc()->funKind == FuncKind::FUNC_ROOT); + + PatchpointInfo* const patchpointInfo = m_compiler->info.compPatchpointInfo; + regMaskTP const tier0CalleeSaves(patchpointInfo->CalleeSaveRegisters()); + + JITDUMP("--OSR--- tier0 has already saved "); + JITDUMPEXEC(dspRegMask(tier0CalleeSaves)); + JITDUMP("\nEmitting restores\n"); + + genRestoreCalleeSavedRegistersHelp(tier0CalleeSaves & ~(RBM_FP | RBM_RA), REG_FP, 16, /* reportUnwindData */ false); + GetEmitter()->emitIns_R_R_I(INS_ld, EA_PTRSIZE, REG_RA, REG_FP, 8); + GetEmitter()->emitIns_R_R_I(INS_ld, EA_PTRSIZE, REG_FP, REG_FP, 0); + + m_compiler->unwindAllocStack(patchpointInfo->TotalFrameSize()); +} + // clang-format off /***************************************************************************** * @@ -492,7 +530,7 @@ void CodeGen::genFuncletEpilog(BasicBlock* /* block */) FP_offset = FP_offset & 0xf; } - genRestoreCalleeSavedRegistersHelp(maskSaveRegs, FP_offset + 16); + genRestoreCalleeSavedRegistersHelp(maskSaveRegs, REG_SPBASE, FP_offset + 16, /* reportUnwindData */ true); GetEmitter()->emitIns_R_R_I(INS_ld, EA_PTRSIZE, REG_RA, REG_SPBASE, FP_offset + 8); m_compiler->unwindSaveReg(REG_RA, FP_offset + 8); @@ -6534,7 +6572,7 @@ void CodeGen::genPopCalleeSavedRegisters(bool jmpEpilog) } JITDUMP(" calleeSaveSPOffset=%d\n", FP_offset + 16); - genRestoreCalleeSavedRegistersHelp(regsToRestoreMask, FP_offset + 16); + genRestoreCalleeSavedRegistersHelp(regsToRestoreMask, REG_SPBASE, FP_offset + 16, /* reportUnwindData */ true); emit->emitIns_R_R_I(INS_ld, EA_PTRSIZE, REG_RA, REG_SPBASE, FP_offset + 8); m_compiler->unwindSaveReg(REG_RA, FP_offset + 8); diff --git a/src/coreclr/jit/codegenwasm.cpp b/src/coreclr/jit/codegenwasm.cpp index 0ce8d19640e56c..a8c072e8cacafe 100644 --- a/src/coreclr/jit/codegenwasm.cpp +++ b/src/coreclr/jit/codegenwasm.cpp @@ -85,7 +85,7 @@ void CodeGen::genBeginFnProlog() //------------------------------------------------------------------------ // genPushCalleeSavedRegisters: no-op since we don't need to save anything. // -void CodeGen::genPushCalleeSavedRegisters() +void CodeGen::genPushCalleeSavedRegisters(regNumber initReg, bool* pInitRegZeroed) { } @@ -137,11 +137,20 @@ void CodeGen::genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pIni //------------------------------------------------------------------------ // genEnregisterOSRArgsAndLocals: enregister OSR args and locals. // -void CodeGen::genEnregisterOSRArgsAndLocals() +void CodeGen::genEnregisterOSRArgsAndLocals(regNumber initReg, bool* pInitRegZeroed) { unreached(); // OSR not supported on WASM. } +//------------------------------------------------------------------------ +// genOSRHandleTier0CalleeSavedRegistersAndFrame: +// Not called for WASM without OSR support. +// +void CodeGen::genOSRHandleTier0CalleeSavedRegistersAndFrame() +{ + unreached(); +} + //------------------------------------------------------------------------ // genHomeRegisterParams: place register arguments into their RA-assigned locations. // diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 19c789e444bc43..f781a6438cbb84 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -9860,11 +9860,11 @@ void CodeGen::genProfilingLeaveCallback(unsigned helper) #ifdef TARGET_AMD64 //------------------------------------------------------------------------ -// genOSRRecordTier0CalleeSavedRegistersAndFrame: for OSR methods, record the +// genOSRHandleTier0CalleeSavedRegistersAndFrame: for OSR methods, record the // subset of callee saves already saved by the Tier0 method, and the frame // created by Tier0. // -void CodeGen::genOSRRecordTier0CalleeSavedRegistersAndFrame() +void CodeGen::genOSRHandleTier0CalleeSavedRegistersAndFrame() { assert(m_compiler->compGeneratingProlog); assert(m_compiler->opts.IsOSR()); @@ -10006,18 +10006,29 @@ void CodeGen::genOSRSaveRemainingCalleeSavedRegisters() osrAdditionalIntCalleeSaves &= ~regBit; } } +#else + +//------------------------------------------------------------------------ +// genOSRHandleTier0CalleeSavedRegistersAndFrame: +// Not called for x86 without OSR support. +// +void CodeGen::genOSRHandleTier0CalleeSavedRegistersAndFrame() +{ + unreached(); +} + #endif // TARGET_AMD64 //------------------------------------------------------------------------ // genPushCalleeSavedRegisters: Push any callee-saved registers we have used. // -void CodeGen::genPushCalleeSavedRegisters() +void CodeGen::genPushCalleeSavedRegisters(regNumber initReg, bool* pInitRegZeroed) { assert(m_compiler->compGeneratingProlog); #if DEBUG // OSR root frames must handle this differently. See - // genOSRRecordTier0CalleeSavedRegisters() + // genOSRHandleTier0CalleeSavedRegistersAndFrame() // genOSRSaveRemainingCalleeSavedRegisters() // if (m_compiler->opts.IsOSR()) diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index f16065d09c60cf..724a9a476d9372 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -5754,17 +5754,30 @@ void Compiler::generatePatchpointInfo() patchpointInfo->AsyncSynchronizationContextOffset()); } -#if defined(TARGET_AMD64) // Record callee save registers. - // Currently only needed for x64. // regMaskTP rsPushRegs = codeGen->regSet.rsGetModifiedCalleeSavedRegsMask(); rsPushRegs |= RBM_FPBASE; - patchpointInfo->SetCalleeSaveRegisters((uint64_t)rsPushRegs); +#if defined(TARGET_ARM64) + rsPushRegs |= RBM_LR; +#elif defined(TARGET_LOONGARCH64) + rsPushRegs |= RBM_RA; +#elif defined(TARGET_RISCV64) + rsPushRegs |= RBM_RA; +#endif + +#ifdef TARGET_ARM64 + // For arm64 we communicate whether fp/lr are stored with the callee saves in this mask. + if (!codeGen->IsSaveFpLrWithAllCalleeSavedRegisters()) + { + rsPushRegs &= ~(RBM_FP | RBM_LR); + } +#endif + + patchpointInfo->SetCalleeSaveRegisters((uint64_t)rsPushRegs.getLow()); JITDUMP("--OSR-- Tier0 callee saves: "); - JITDUMPEXEC(dspRegMask((regMaskTP)patchpointInfo->CalleeSaveRegisters())); + JITDUMPEXEC(dspRegMask(regMaskTP((regMaskSmall)patchpointInfo->CalleeSaveRegisters()))); JITDUMP("\n"); -#endif // Register this with the runtime. info.compCompHnd->setPatchpointInfo(patchpointInfo); diff --git a/src/coreclr/vm/jithelpers.cpp b/src/coreclr/vm/jithelpers.cpp index 952824661b77fa..a6e468ebb134be 100644 --- a/src/coreclr/vm/jithelpers.cpp +++ b/src/coreclr/vm/jithelpers.cpp @@ -1721,11 +1721,10 @@ extern "C" void JIT_PatchpointWorkerWorkerWithPolicy(TransitionBlock * pTransiti SetSSP(pFrameContext, ssp - 8); } #endif // TARGET_WINDOWS - - pFrameContext->Rbp = currentFP; -#endif // TARGET_AMD64 +#endif // TARGET_AMD64 SetSP(pFrameContext, currentSP); + SetFP(pFrameContext, currentFP); // Note we can get here w/o triggering, if there is an existing OSR method and // we hit the patchpoint. diff --git a/src/coreclr/vm/jitinterface.cpp b/src/coreclr/vm/jitinterface.cpp index 9dd542b94c03c5..c1210091d75330 100644 --- a/src/coreclr/vm/jitinterface.cpp +++ b/src/coreclr/vm/jitinterface.cpp @@ -11400,6 +11400,26 @@ void CEEJitInfo::setPatchpointInfo(PatchpointInfo* patchpointInfo) // We receive ownership of the array _ASSERTE(m_pPatchpointInfoFromJit == NULL); m_pPatchpointInfoFromJit = patchpointInfo; + +#if defined(_DEBUG) && defined(ALLOW_SXS_JIT) + if (m_jitFlags.IsSet(CORJIT_FLAGS::CORJIT_FLAG_ALT_JIT)) + { + uint32_t ppiSize = patchpointInfo->PatchpointInfoSize(); + + AllocMemTracker am; + void* mem = am.Track(m_pMethodBeingCompiled->GetLoaderAllocator()->GetLowFrequencyHeap()->AllocMem(S_SIZE_T(ppiSize))); + PatchpointInfo *newPpi = new (mem) PatchpointInfo; + newPpi->Initialize(patchpointInfo->NumberOfLocals(), patchpointInfo->TotalFrameSize()); + newPpi->Copy(patchpointInfo); + + HRESULT hr = m_pMethodBeingCompiled->SetMethodDescAltJitPatchpointInfo(newPpi); + if (SUCCEEDED(hr)) + { + am.SuppressRelease(); + } + } +#endif + #else UNREACHABLE(); #endif @@ -11423,6 +11443,18 @@ PatchpointInfo* CEEJitInfo::getOSRInfo(unsigned* ilOffset) #ifdef FEATURE_ON_STACK_REPLACEMENT result = m_pPatchpointInfoFromRuntime; *ilOffset = m_ilOffset; + +#if defined(_DEBUG) && defined(ALLOW_SXS_JIT) + if (m_jitFlags.IsSet(CORJIT_FLAGS::CORJIT_FLAG_ALT_JIT)) + { + PatchpointInfo* ppi = m_pMethodBeingCompiled->GetMethodDescAltJitPatchpointInfo(); + if (ppi != NULL) + { + result = ppi; + } + } +#endif + #endif EE_TO_JIT_TRANSITION(); diff --git a/src/coreclr/vm/method.cpp b/src/coreclr/vm/method.cpp index 6e4772c539db5c..c079fba250253f 100644 --- a/src/coreclr/vm/method.cpp +++ b/src/coreclr/vm/method.cpp @@ -274,6 +274,29 @@ void MethodDesc::SetMethodDescOptimizationTier(NativeCodeVersion::OptimizationTi _ASSERTE(m_codeData != NULL); VolatileStoreWithoutBarrier(&m_codeData->OptimizationTier, tier); } + +#if defined(_DEBUG) && defined(ALLOW_SXS_JIT) +HRESULT MethodDesc::SetMethodDescAltJitPatchpointInfo(PatchpointInfo* pInfo) +{ + WRAPPER_NO_CONTRACT; + + HRESULT hr; + IfFailRet(EnsureCodeDataExists(NULL)); + + _ASSERTE(m_codeData != NULL); + VolatileStoreWithoutBarrier(&m_codeData->AltJitPatchpointInfo, pInfo); + return S_OK; +} + +PatchpointInfo* MethodDesc::GetMethodDescAltJitPatchpointInfo() +{ + WRAPPER_NO_CONTRACT; + if (m_codeData == NULL) + return nullptr; + return VolatileLoadWithoutBarrier(&m_codeData->AltJitPatchpointInfo); +} +#endif // _DEBUG && ALLOW_SXS_JIT + #endif // FEATURE_CODE_VERSIONING #ifdef FEATURE_INTERPRETER diff --git a/src/coreclr/vm/method.hpp b/src/coreclr/vm/method.hpp index 9188d081cd5ee4..aee47a6a8a876d 100644 --- a/src/coreclr/vm/method.hpp +++ b/src/coreclr/vm/method.hpp @@ -262,6 +262,9 @@ struct MethodDescCodeData final #ifdef FEATURE_INTERPRETER CallStubHeader *CallStub; #endif // FEATURE_INTERPRETER +#if defined(_DEBUG) && defined(ALLOW_SXS_JIT) + PatchpointInfo *AltJitPatchpointInfo; +#endif // _DEBUG && ALLOW_SXS_JIT }; using PTR_MethodDescCodeData = DPTR(MethodDescCodeData); @@ -1989,6 +1992,11 @@ class MethodDesc #ifndef DACCESS_COMPILE HRESULT SetMethodDescVersionState(PTR_MethodDescVersioningState state); void SetMethodDescOptimizationTier(NativeCodeVersion::OptimizationTier tier); + +#if defined(_DEBUG) && defined(ALLOW_SXS_JIT) + HRESULT SetMethodDescAltJitPatchpointInfo(PatchpointInfo* pInfo); + PatchpointInfo* GetMethodDescAltJitPatchpointInfo(); +#endif #endif // !DACCESS_COMPILE PTR_MethodDescVersioningState GetMethodDescVersionState(); NativeCodeVersion::OptimizationTier GetMethodDescOptimizationTier();