diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index e15225736ab41f..1e2be5c6d7c167 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -1201,16 +1201,15 @@ void AsyncTransformation::BuildContinuation(BasicBlock* block, JITDUMP(" Call has return; continuation will have return value\n"); } - // For OSR, we store the IL offset that inspired the OSR method at the - // beginning of the data (and store -1 in the tier0 version). This must be - // at the beginning because the tier0 and OSR versions need to agree on - // this. + // For OSR, we store the address of the OSR function at the beginning of + // the data (and store 0 in the tier0 version). This must be at the + // beginning because the tier0 and OSR versions need to agree on this. if (m_compiler->doesMethodHavePatchpoints() || m_compiler->opts.IsOSR()) { - JITDUMP(" Method %s; keeping IL offset that inspired OSR method at the beginning of non-GC data\n", + JITDUMP(" Method %s; keeping OSR address at the beginning of non-GC data\n", m_compiler->doesMethodHavePatchpoints() ? "has patchpoints" : "is an OSR method"); // Must be pointer sized for compatibility with Continuation methods that access fields - layoutBuilder->SetNeedsOSRILOffset(); + layoutBuilder->SetNeedsOSRAddress(); } if (HasNonContextRestoreExceptionalFlow(block)) @@ -1252,9 +1251,9 @@ void AsyncTransformation::BuildContinuation(BasicBlock* block, void ContinuationLayout::Dump(int indent) { printf("%*sContinuation layout (%u bytes):\n", indent, "", Size); - if (OSRILOffset != UINT_MAX) + if (OSRAddressOffset != UINT_MAX) { - printf("%*s +%03u OSR IL offset\n", indent, "", OSRILOffset); + printf("%*s +%03u OSR address\n", indent, "", OSRAddressOffset); } if (ExceptionOffset != UINT_MAX) @@ -1415,10 +1414,10 @@ ContinuationLayout* ContinuationLayoutBuilder::Create() return offset; }; - if (m_needsOSRILOffset) + if (m_needsOSRAddress) { // Must be pointer sized for compatibility with Continuation methods that access fields - layout->OSRILOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); + layout->OSRAddressOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); } if (m_needsException) @@ -1876,7 +1875,7 @@ GenTreeCall* AsyncTransformation::CreateAllocContinuationCall(bool //------------------------------------------------------------------------ // AsyncTransformation::FillInDataOnSuspension: // Create IR that fills the data array of the continuation object with -// live local values, OSR IL offset, continuation context, and execution +// live local values, OSR address, continuation context, and execution // context. // // Parameters: @@ -1894,19 +1893,23 @@ void AsyncTransformation::FillInDataOnSuspension(GenTreeCall* { if ((saveSet != SaveSet::MutatedLocals) && (m_compiler->doesMethodHavePatchpoints() || m_compiler->opts.IsOSR())) { - GenTree* ilOffsetToStore; + GenTree* osrAddressToStore; if (m_compiler->doesMethodHavePatchpoints()) - ilOffsetToStore = m_compiler->gtNewIconNode(-1); + { + osrAddressToStore = m_compiler->gtNewIconNode(0, TYP_I_IMPL); + } else - ilOffsetToStore = m_compiler->gtNewIconNode((int)m_compiler->info.compILEntry); + { + osrAddressToStore = new (m_compiler, GT_FTN_ENTRY) GenTree(GT_FTN_ENTRY, TYP_I_IMPL); + } - // OSR IL offset needs to be at offset 0 because OSR and tier0 methods + // OSR address needs to be at offset 0 because OSR and tier0 methods // need to agree on that. - assert(layout.OSRILOffset == 0); + assert(layout.OSRAddressOffset == 0); GenTree* newContinuation = m_compiler->gtNewLclvNode(GetNewContinuationVar(), TYP_REF); unsigned offset = OFFSETOF__CORINFO_Continuation__data; - GenTree* storePatchpointOffset = StoreAtOffset(newContinuation, offset, ilOffsetToStore, TYP_INT); - LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_compiler, storePatchpointOffset)); + GenTree* storeOSRAddressOffset = StoreAtOffset(newContinuation, offset, osrAddressToStore, TYP_I_IMPL); + LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_compiler, storeOSRAddressOffset)); } // Fill in data @@ -2871,7 +2874,7 @@ ContinuationLayoutBuilder* ContinuationLayoutBuilder::CreateSharedLayout(Compile for (const AsyncState& state : states) { ContinuationLayoutBuilder* layout = state.Layout; - sharedLayout->m_needsOSRILOffset |= layout->m_needsOSRILOffset; + sharedLayout->m_needsOSRAddress |= layout->m_needsOSRAddress; sharedLayout->m_needsException |= layout->m_needsException; sharedLayout->m_needsContinuationContext |= layout->m_needsContinuationContext; sharedLayout->m_needsKeepAlive |= layout->m_needsKeepAlive; @@ -3004,55 +3007,50 @@ void AsyncTransformation::CreateResumptionSwitch() { JITDUMP(" Method has patch points...\n"); // If we have patchpoints then first check if we need to resume in the OSR version. - BasicBlock* callHelperBB = m_compiler->fgNewBBafter(BBJ_THROW, m_compiler->fgLastBBInMainFunction(), false); - callHelperBB->bbSetRunRarely(); - callHelperBB->clearTryIndex(); - callHelperBB->clearHndIndex(); + BasicBlock* jmpOSR = m_compiler->fgNewBBafter(BBJ_THROW, m_compiler->fgLastBBInMainFunction(), false); + jmpOSR->bbSetRunRarely(); + jmpOSR->clearTryIndex(); + jmpOSR->clearHndIndex(); - JITDUMP(" Created " FMT_BB " for transitions back into OSR method\n", callHelperBB->bbNum); + JITDUMP(" Created " FMT_BB " for transitions back into OSR method\n", jmpOSR->bbNum); - BasicBlock* onContinuationBB = newEntryBB->GetTrueTarget(); - BasicBlock* checkILOffsetBB = m_compiler->fgNewBBbefore(BBJ_COND, onContinuationBB, true); + BasicBlock* onContinuationBB = newEntryBB->GetTrueTarget(); + BasicBlock* checkOSRAddressOffsetBB = m_compiler->fgNewBBbefore(BBJ_COND, onContinuationBB, true); JITDUMP(" Created " FMT_BB " to check whether we should transition immediately to OSR\n", - checkILOffsetBB->bbNum); + checkOSRAddressOffsetBB->bbNum); - // Redirect newEntryBB -> onContinuationBB into newEntryBB -> checkILOffsetBB -> onContinuationBB - m_compiler->fgRedirectEdge(newEntryBB->TrueEdgeRef(), checkILOffsetBB); + // Redirect newEntryBB -> onContinuationBB into newEntryBB -> checkOSRAddressOffsetBB -> onContinuationBB + m_compiler->fgRedirectEdge(newEntryBB->TrueEdgeRef(), checkOSRAddressOffsetBB); newEntryBB->GetTrueEdge()->setLikelihood(0); - checkILOffsetBB->inheritWeightPercentage(newEntryBB, 0); + checkOSRAddressOffsetBB->inheritWeightPercentage(newEntryBB, 0); - FlowEdge* toOnContinuationBB = m_compiler->fgAddRefPred(onContinuationBB, checkILOffsetBB); - FlowEdge* toCallHelperBB = m_compiler->fgAddRefPred(callHelperBB, checkILOffsetBB); - checkILOffsetBB->SetCond(toCallHelperBB, toOnContinuationBB); - toCallHelperBB->setLikelihood(0); + FlowEdge* toOnContinuationBB = m_compiler->fgAddRefPred(onContinuationBB, checkOSRAddressOffsetBB); + FlowEdge* toJmpOSRBB = m_compiler->fgAddRefPred(jmpOSR, checkOSRAddressOffsetBB); + checkOSRAddressOffsetBB->SetCond(toJmpOSRBB, toOnContinuationBB); + toJmpOSRBB->setLikelihood(0); toOnContinuationBB->setLikelihood(1); - callHelperBB->inheritWeightPercentage(checkILOffsetBB, 0); - - // We need to dispatch to the OSR version if the IL offset is non-negative. - continuationArg = m_compiler->gtNewLclvNode(m_compiler->lvaAsyncContinuationArg, TYP_REF); - unsigned offsetOfIlOffset = OFFSETOF__CORINFO_Continuation__data; - GenTree* ilOffset = LoadFromOffset(continuationArg, offsetOfIlOffset, TYP_INT); - unsigned ilOffsetLclNum = m_compiler->lvaGrabTemp(false DEBUGARG("IL offset for tier0 OSR method")); - m_compiler->lvaGetDesc(ilOffsetLclNum)->lvType = TYP_INT; - GenTree* storeIlOffset = m_compiler->gtNewStoreLclVarNode(ilOffsetLclNum, ilOffset); - LIR::AsRange(checkILOffsetBB).InsertAtEnd(LIR::SeqTree(m_compiler, storeIlOffset)); - - ilOffset = m_compiler->gtNewLclvNode(ilOffsetLclNum, TYP_INT); - GenTree* zero = m_compiler->gtNewIconNode(0); - GenTree* geZero = m_compiler->gtNewOperNode(GT_GE, TYP_INT, ilOffset, zero); - GenTree* jtrue = m_compiler->gtNewOperNode(GT_JTRUE, TYP_VOID, geZero); - LIR::AsRange(checkILOffsetBB).InsertAtEnd(ilOffset, zero, geZero, jtrue); + jmpOSR->inheritWeightPercentage(checkOSRAddressOffsetBB, 0); - ilOffset = m_compiler->gtNewLclvNode(ilOffsetLclNum, TYP_INT); + // We need to dispatch to the OSR version if the OSR address is non-zero. + continuationArg = m_compiler->gtNewLclvNode(m_compiler->lvaAsyncContinuationArg, TYP_REF); + unsigned offsetOfOSRAddressOffset = OFFSETOF__CORINFO_Continuation__data; + GenTree* osrAddress = LoadFromOffset(continuationArg, offsetOfOSRAddressOffset, TYP_I_IMPL); + unsigned osrAddressLclNum = m_compiler->lvaGrabTemp(false DEBUGARG("OSR address for tier0 OSR method")); + m_compiler->lvaGetDesc(osrAddressLclNum)->lvType = TYP_I_IMPL; + GenTree* storeOsrAddress = m_compiler->gtNewStoreLclVarNode(osrAddressLclNum, osrAddress); + LIR::AsRange(checkOSRAddressOffsetBB).InsertAtEnd(LIR::SeqTree(m_compiler, storeOsrAddress)); - GenTreeCall* callHelper = m_compiler->gtNewHelperCallNode(CORINFO_HELP_PATCHPOINT_FORCED, TYP_VOID, ilOffset); - callHelper->gtCallMoreFlags |= GTF_CALL_M_DOES_NOT_RETURN; + osrAddress = m_compiler->gtNewLclvNode(osrAddressLclNum, TYP_I_IMPL); + GenTree* zero = m_compiler->gtNewIconNode(0, TYP_I_IMPL); + GenTree* neZero = m_compiler->gtNewOperNode(GT_NE, TYP_INT, osrAddress, zero); + GenTree* jtrue = m_compiler->gtNewOperNode(GT_JTRUE, TYP_VOID, neZero); + LIR::AsRange(checkOSRAddressOffsetBB).InsertAtEnd(osrAddress, zero, neZero, jtrue); - m_compiler->compCurBB = callHelperBB; - m_compiler->fgMorphTree(callHelper); + osrAddress = m_compiler->gtNewLclvNode(osrAddressLclNum, TYP_I_IMPL); - LIR::AsRange(callHelperBB).InsertAtEnd(LIR::SeqTree(m_compiler, callHelper)); + GenTree* jmpOsr = m_compiler->gtNewOperNode(GT_NONLOCAL_JMP, TYP_VOID, osrAddress); + LIR::AsRange(jmpOSR).InsertAtEnd(LIR::SeqTree(m_compiler, jmpOsr)); } else if (m_compiler->opts.IsOSR()) { @@ -3061,33 +3059,33 @@ void AsyncTransformation::CreateResumptionSwitch() // version by normal means then we will see a non-zero continuation // here that belongs to the tier0 method. In that case we should just // ignore it, so create a BB that jumps back. - BasicBlock* onContinuationBB = newEntryBB->GetTrueTarget(); - BasicBlock* onNoContinuationBB = newEntryBB->GetFalseTarget(); - BasicBlock* checkILOffsetBB = m_compiler->fgNewBBbefore(BBJ_COND, onContinuationBB, true); + BasicBlock* onContinuationBB = newEntryBB->GetTrueTarget(); + BasicBlock* onNoContinuationBB = newEntryBB->GetFalseTarget(); + BasicBlock* checkOSRAddressOffsetBB = m_compiler->fgNewBBbefore(BBJ_COND, onContinuationBB, true); - // Switch newEntryBB -> onContinuationBB into newEntryBB -> checkILOffsetBB - m_compiler->fgRedirectEdge(newEntryBB->TrueEdgeRef(), checkILOffsetBB); + // Switch newEntryBB -> onContinuationBB into newEntryBB -> checkOSRAddressOffsetBB + m_compiler->fgRedirectEdge(newEntryBB->TrueEdgeRef(), checkOSRAddressOffsetBB); newEntryBB->GetTrueEdge()->setLikelihood(0); - checkILOffsetBB->inheritWeightPercentage(newEntryBB, 0); + checkOSRAddressOffsetBB->inheritWeightPercentage(newEntryBB, 0); - // Make checkILOffsetBB ->(true) onNoContinuationBB - // ->(false) onContinuationBB + // Make checkOSRAddressOffsetBB ->(true) onNoContinuationBB + // ->(false) onContinuationBB - FlowEdge* toOnContinuationBB = m_compiler->fgAddRefPred(onContinuationBB, checkILOffsetBB); - FlowEdge* toOnNoContinuationBB = m_compiler->fgAddRefPred(onNoContinuationBB, checkILOffsetBB); - checkILOffsetBB->SetCond(toOnNoContinuationBB, toOnContinuationBB); + FlowEdge* toOnContinuationBB = m_compiler->fgAddRefPred(onContinuationBB, checkOSRAddressOffsetBB); + FlowEdge* toOnNoContinuationBB = m_compiler->fgAddRefPred(onNoContinuationBB, checkOSRAddressOffsetBB); + checkOSRAddressOffsetBB->SetCond(toOnNoContinuationBB, toOnContinuationBB); toOnContinuationBB->setLikelihood(0); toOnNoContinuationBB->setLikelihood(1); - JITDUMP(" Created " FMT_BB " to check for Tier-0 continuations\n", checkILOffsetBB->bbNum); + JITDUMP(" Created " FMT_BB " to check for Tier-0 continuations\n", checkOSRAddressOffsetBB->bbNum); - continuationArg = m_compiler->gtNewLclvNode(m_compiler->lvaAsyncContinuationArg, TYP_REF); - unsigned offsetOfIlOffset = OFFSETOF__CORINFO_Continuation__data; - GenTree* ilOffset = LoadFromOffset(continuationArg, offsetOfIlOffset, TYP_INT); - GenTree* zero = m_compiler->gtNewIconNode(0); - GenTree* ltZero = m_compiler->gtNewOperNode(GT_LT, TYP_INT, ilOffset, zero); - GenTree* jtrue = m_compiler->gtNewOperNode(GT_JTRUE, TYP_VOID, ltZero); - LIR::AsRange(checkILOffsetBB).InsertAtEnd(LIR::SeqTree(m_compiler, jtrue)); + continuationArg = m_compiler->gtNewLclvNode(m_compiler->lvaAsyncContinuationArg, TYP_REF); + unsigned offsetOfOSRAddressOffset = OFFSETOF__CORINFO_Continuation__data; + GenTree* osrAddress = LoadFromOffset(continuationArg, offsetOfOSRAddressOffset, TYP_I_IMPL); + GenTree* zero = m_compiler->gtNewIconNode(0, TYP_I_IMPL); + GenTree* eqZero = m_compiler->gtNewOperNode(GT_EQ, TYP_INT, osrAddress, zero); + GenTree* jtrue = m_compiler->gtNewOperNode(GT_JTRUE, TYP_VOID, eqZero); + LIR::AsRange(checkOSRAddressOffsetBB).InsertAtEnd(LIR::SeqTree(m_compiler, jtrue)); if (ReuseContinuations()) { diff --git a/src/coreclr/jit/async.h b/src/coreclr/jit/async.h index b02993c8410b09..eb9a22b16ab212 100644 --- a/src/coreclr/jit/async.h +++ b/src/coreclr/jit/async.h @@ -53,7 +53,7 @@ struct ContinuationLayoutBuilder { private: Compiler* m_compiler; - bool m_needsOSRILOffset = false; + bool m_needsOSRAddress = false; bool m_needsException = false; bool m_needsContinuationContext = false; bool m_needsKeepAlive = false; @@ -70,13 +70,13 @@ struct ContinuationLayoutBuilder { } - void SetNeedsOSRILOffset() + void SetNeedsOSRAddress() { - m_needsOSRILOffset = true; + m_needsOSRAddress = true; } - bool NeedsOSRILOffset() const + bool NeedsOSRAddress() const { - return m_needsOSRILOffset; + return m_needsOSRAddress; } void SetNeedsException() { @@ -128,7 +128,7 @@ struct ContinuationLayoutBuilder struct ContinuationLayout { unsigned Size = 0; - unsigned OSRILOffset = UINT_MAX; + unsigned OSRAddressOffset = UINT_MAX; unsigned ExceptionOffset = UINT_MAX; unsigned ContinuationContextOffset = UINT_MAX; unsigned KeepAliveOffset = UINT_MAX; diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index f4e371c7ea4338..62974a9dc164c7 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -1246,6 +1246,7 @@ class CodeGen final : public CodeGenInterface void genJumpTable(GenTree* tree); void genTableBasedSwitch(GenTree* tree); void genAsyncResumeInfo(GenTreeVal* tree); + void genFtnEntry(GenTree* tree); UNATIVE_OFFSET genEmitAsyncResumeInfoTable(emitter::dataSection** dataSec); CORINFO_FIELD_HANDLE genEmitAsyncResumeInfo(unsigned stateNum); #if defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) @@ -1309,6 +1310,8 @@ class CodeGen final : public CodeGenInterface void genSwiftErrorReturn(GenTree* treeNode); #endif // SWIFT_SUPPORT + void genNonLocalJmp(GenTreeUnOp* treeNode); + #ifdef TARGET_XARCH void genStackPointerConstantAdjustment(ssize_t spDelta, bool trackSpAdjustments); void genStackPointerConstantAdjustmentWithProbe(ssize_t spDelta, bool trackSpAdjustments); diff --git a/src/coreclr/jit/codegenarm.cpp b/src/coreclr/jit/codegenarm.cpp index e28571af1629e2..635107af30ac26 100644 --- a/src/coreclr/jit/codegenarm.cpp +++ b/src/coreclr/jit/codegenarm.cpp @@ -643,6 +643,17 @@ void CodeGen::genTableBasedSwitch(GenTree* treeNode) GetEmitter()->emitIns_R_ARX(INS_ldr, EA_4BYTE, REG_PC, baseReg, idxReg, TARGET_POINTER_SIZE, 0); } +//------------------------------------------------------------------------ +// genNonLocalJmp: Not supported for arm32. +// +// Parameters: +// tree - the GT_NONLOCAL_JMP node +// +void CodeGen::genNonLocalJmp(GenTreeUnOp* tree) +{ + NYI_ARM("GT_NONLOCAL_JMP is not supported on arm32"); +} + //------------------------------------------------------------------------ // genJumpTable: emits the table and an instruction to get the address of the first element // @@ -669,6 +680,17 @@ void CodeGen::genAsyncResumeInfo(GenTreeVal* treeNode) genProduceReg(treeNode); } +//------------------------------------------------------------------------ +// genFtnEntry: Not supported for arm32. +// +// Parameters: +// treeNode - the GT_FTN_ENTRY node +// +void CodeGen::genFtnEntry(GenTree* treeNode) +{ + NYI_ARM("GT_FTN_ENTRY is not supported on arm32"); +} + //------------------------------------------------------------------------ // genGetInsForOper: Return instruction encoding of the operation tree. // diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 4f7eb08e69c1cf..47ecfbea7dc7de 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -3808,6 +3808,35 @@ void CodeGen::genAsyncResumeInfo(GenTreeVal* treeNode) genProduceReg(treeNode); } +//------------------------------------------------------------------------ +// genFtnEntry: emits address of the current function being compiled +// +// Parameters: +// treeNode - the GT_FTN_ENTRY node +// +void CodeGen::genFtnEntry(GenTree* treeNode) +{ + GetEmitter()->emitIns_R_L(INS_adr, EA_PTRSIZE, GetEmitter()->emitPrologIG, treeNode->GetRegNum()); + genProduceReg(treeNode); +} + +//------------------------------------------------------------------------ +// genNonLocalJmp: Emit jump to the specified address. +// +// Parameters: +// tree - the GT_NONLOCAL_JMP node +// +void CodeGen::genNonLocalJmp(GenTreeUnOp* tree) +{ + // Non-local jumps cannot handle the case where this function has been + // hijacked, since the VM may not restore the original LR at the right + // location in the new frame. + SetHasTailCalls(true); + + genConsumeOperands(tree->AsOp()); + GetEmitter()->emitIns_R(INS_br, EA_PTRSIZE, tree->gtGetOp1()->GetRegNum()); +} + //------------------------------------------------------------------------ // genLockedInstructions: Generate code for a GT_XADD, GT_XAND, GT_XORR or GT_XCHG node. // diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 7e7a259e855272..337ecc76369609 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -295,10 +295,6 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) break; #endif // SWIFT_SUPPORT - case GT_RETURN_SUSPEND: - genReturnSuspend(treeNode->AsUnOp()); - break; - case GT_LEA: // If we are here, it is the case where there is an LEA that cannot be folded into a parent instruction. genLeaInstruction(treeNode->AsAddrMode()); @@ -502,6 +498,18 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) case GT_CATCH_ARG: genCodeForCatchArg(treeNode); break; + case GT_LABEL: + genPendingCallLabel = genCreateTempLabel(); +#if defined(TARGET_ARM) + genMov32RelocatableDisplacement(genPendingCallLabel, targetReg); +#else + emit->emitIns_R_L(INS_adr, EA_PTRSIZE, genPendingCallLabel, targetReg); +#endif + break; + + case GT_RETURN_SUSPEND: + genReturnSuspend(treeNode->AsUnOp()); + break; case GT_ASYNC_CONTINUATION: genCodeForAsyncContinuation(treeNode); @@ -511,13 +519,16 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) genAsyncResumeInfo(treeNode->AsVal()); break; - case GT_LABEL: - genPendingCallLabel = genCreateTempLabel(); -#if defined(TARGET_ARM) - genMov32RelocatableDisplacement(genPendingCallLabel, targetReg); -#else - emit->emitIns_R_L(INS_adr, EA_PTRSIZE, genPendingCallLabel, targetReg); -#endif + case GT_RECORD_ASYNC_RESUME: + genRecordAsyncResume(treeNode->AsVal()); + break; + + case GT_FTN_ENTRY: + genFtnEntry(treeNode); + break; + + case GT_NONLOCAL_JMP: + genNonLocalJmp(treeNode->AsUnOp()); break; case GT_STORE_BLK: @@ -543,10 +554,6 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) // Do nothing; this node is a marker for debug info. break; - case GT_RECORD_ASYNC_RESUME: - genRecordAsyncResume(treeNode->AsVal()); - break; - default: { unreached(); diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 93b43354b57bf4..d7dc1e38aedfd6 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -5393,6 +5393,10 @@ void CodeGen::genFnProlog() // allocating the local frame. // extraFrameSize = m_compiler->compCalleeRegsPushed * REGSIZE_BYTES; + + // Simulate a return address being pushed by a call to get expected misalignment on entry. + GetEmitter()->emitIns_R(INS_push, EA_PTRSIZE, REG_EAX); + m_compiler->unwindAllocStack(REGSIZE_BYTES); #endif } diff --git a/src/coreclr/jit/codegenloongarch64.cpp b/src/coreclr/jit/codegenloongarch64.cpp index b7e548b874b233..cbbd5d748a5151 100644 --- a/src/coreclr/jit/codegenloongarch64.cpp +++ b/src/coreclr/jit/codegenloongarch64.cpp @@ -2340,6 +2340,36 @@ void CodeGen::genAsyncResumeInfo(GenTreeVal* treeNode) genProduceReg(treeNode); } +//------------------------------------------------------------------------ +// genFtnEntry: emits address of the current function being compiled +// +// Parameters: +// treeNode - the GT_FTN_ENTRY node +// +void CodeGen::genFtnEntry(GenTree* treeNode) +{ + GetEmitter()->emitIns_R_L(INS_lea, EA_PTRSIZE, GetEmitter()->emitPrologIG, treeNode->GetRegNum()); + genProduceReg(treeNode); +} + +//------------------------------------------------------------------------ +// genNonLocalJmp: Emit jump to the specified address. +// +// Parameters: +// tree - the GT_NONLOCAL_JMP node +// +void CodeGen::genNonLocalJmp(GenTreeUnOp* tree) +{ + // Non-local jumps cannot handle the case where this function has been + // hijacked, since the VM may not restore the original LR at the right + // location in the new frame. + SetHasTailCalls(true); + + genConsumeOperands(tree->AsOp()); + // jirl with rd=r0 is an indirect jump (no link) + GetEmitter()->emitIns_R_R_I(INS_jirl, EA_PTRSIZE, REG_R0, tree->gtGetOp1()->GetRegNum(), 0); +} + //------------------------------------------------------------------------ // genLockedInstructions: Generate code for a GT_XADD or GT_XCHG node. // @@ -4421,10 +4451,30 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) emit->emitIns_R_L(INS_ld_d, EA_PTRSIZE, genPendingCallLabel, targetReg); break; + case GT_RETURN_SUSPEND: + genReturnSuspend(treeNode->AsUnOp()); + break; + + case GT_ASYNC_CONTINUATION: + genCodeForAsyncContinuation(treeNode); + break; + case GT_ASYNC_RESUME_INFO: genAsyncResumeInfo(treeNode->AsVal()); break; + case GT_RECORD_ASYNC_RESUME: + genRecordAsyncResume(treeNode->AsVal()); + break; + + case GT_FTN_ENTRY: + genFtnEntry(treeNode); + break; + + case GT_NONLOCAL_JMP: + genNonLocalJmp(treeNode->AsUnOp()); + break; + case GT_STORE_BLK: genCodeForStoreBlk(treeNode->AsBlk()); break; @@ -4441,18 +4491,6 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) // Do nothing; this node is a marker for debug info. break; - case GT_RECORD_ASYNC_RESUME: - genRecordAsyncResume(treeNode->AsVal()); - break; - - case GT_ASYNC_CONTINUATION: - genCodeForAsyncContinuation(treeNode); - break; - - case GT_RETURN_SUSPEND: - genReturnSuspend(treeNode->AsUnOp()); - break; - default: { #ifdef DEBUG diff --git a/src/coreclr/jit/codegenriscv64.cpp b/src/coreclr/jit/codegenriscv64.cpp index 6732e8459e018a..1a429ae5335aca 100644 --- a/src/coreclr/jit/codegenriscv64.cpp +++ b/src/coreclr/jit/codegenriscv64.cpp @@ -2315,6 +2315,36 @@ void CodeGen::genAsyncResumeInfo(GenTreeVal* treeNode) genProduceReg(treeNode); } +//------------------------------------------------------------------------ +// genFtnEntry: emits address of the current function being compiled +// +// Parameters: +// treeNode - the GT_FTN_ENTRY node +// +void CodeGen::genFtnEntry(GenTree* treeNode) +{ + GetEmitter()->emitIns_R_L(INS_lea, EA_PTRSIZE, GetEmitter()->emitPrologIG, treeNode->GetRegNum()); + genProduceReg(treeNode); +} + +//------------------------------------------------------------------------ +// genNonLocalJmp: Emit jump to the specified address. +// +// Parameters: +// tree - the GT_NONLOCAL_JMP node +// +void CodeGen::genNonLocalJmp(GenTreeUnOp* tree) +{ + // Non-local jumps cannot handle the case where this function has been + // hijacked, since the VM may not restore the original LR at the right + // location in the new frame. + SetHasTailCalls(true); + + genConsumeOperands(tree->AsOp()); + // jalr with rd=x0 is an indirect jump (no link) + GetEmitter()->emitIns_R_R_I(INS_jalr, EA_PTRSIZE, REG_R0, tree->gtGetOp1()->GetRegNum(), 0); +} + //------------------------------------------------------------------------ // genLockedInstructions: Generate code for a GT_XADD, GT_XAND, GT_XORR or GT_XCHG node. // @@ -4207,10 +4237,30 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) emit->emitIns_R_L(INS_ld, EA_PTRSIZE, genPendingCallLabel, targetReg); break; + case GT_RETURN_SUSPEND: + genReturnSuspend(treeNode->AsUnOp()); + break; + + case GT_ASYNC_CONTINUATION: + genCodeForAsyncContinuation(treeNode); + break; + case GT_ASYNC_RESUME_INFO: genAsyncResumeInfo(treeNode->AsVal()); break; + case GT_RECORD_ASYNC_RESUME: + genRecordAsyncResume(treeNode->AsVal()); + break; + + case GT_FTN_ENTRY: + genFtnEntry(treeNode); + break; + + case GT_NONLOCAL_JMP: + genNonLocalJmp(treeNode->AsUnOp()); + break; + case GT_STORE_BLK: genCodeForStoreBlk(treeNode->AsBlk()); break; @@ -4227,18 +4277,6 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) // Do nothing; this node is a marker for debug info. break; - case GT_RECORD_ASYNC_RESUME: - genRecordAsyncResume(treeNode->AsVal()); - break; - - case GT_ASYNC_CONTINUATION: - genCodeForAsyncContinuation(treeNode); - break; - - case GT_RETURN_SUSPEND: - genReturnSuspend(treeNode->AsUnOp()); - break; - case GT_SH1ADD: case GT_SH1ADD_UW: case GT_SH2ADD: diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 30f278f1f32215..3ddb8389951247 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -1925,10 +1925,6 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) break; #endif // SWIFT_SUPPORT - case GT_RETURN_SUSPEND: - genReturnSuspend(treeNode->AsUnOp()); - break; - case GT_LEA: // If we are here, it is the case where there is an LEA that cannot be folded into a parent instruction. genLeaInstruction(treeNode->AsAddrMode()); @@ -2106,19 +2102,35 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) genCodeForCatchArg(treeNode); break; - case GT_ASYNC_CONTINUATION: - genCodeForAsyncContinuation(treeNode); - break; - case GT_LABEL: genPendingCallLabel = genCreateTempLabel(); emit->emitIns_R_L(INS_lea, EA_PTR_DSP_RELOC, genPendingCallLabel, treeNode->GetRegNum()); break; + case GT_RETURN_SUSPEND: + genReturnSuspend(treeNode->AsUnOp()); + break; + + case GT_ASYNC_CONTINUATION: + genCodeForAsyncContinuation(treeNode); + break; + case GT_ASYNC_RESUME_INFO: genAsyncResumeInfo(treeNode->AsVal()); break; + case GT_RECORD_ASYNC_RESUME: + genRecordAsyncResume(treeNode->AsVal()); + break; + + case GT_FTN_ENTRY: + genFtnEntry(treeNode); + break; + + case GT_NONLOCAL_JMP: + genNonLocalJmp(treeNode->AsUnOp()); + break; + case GT_STORE_BLK: genCodeForStoreBlk(treeNode->AsBlk()); break; @@ -2142,10 +2154,6 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) // Do nothing; this node is a marker for debug info. break; - case GT_RECORD_ASYNC_RESUME: - genRecordAsyncResume(treeNode->AsVal()); - break; - #if defined(TARGET_AMD64) case GT_CCMP: genCodeForCCMP(treeNode->AsCCMP()); @@ -4261,6 +4269,18 @@ void CodeGen::genTableBasedSwitch(GenTree* treeNode) GetEmitter()->emitIns_R(INS_i_jmp, emitTypeSize(TYP_I_IMPL), baseReg); } +//------------------------------------------------------------------------ +// genNonLocalJmp: Emit jump to the specified address. +// +// Parameters: +// tree - the GT_NONLOCAL_JMP node +// +void CodeGen::genNonLocalJmp(GenTreeUnOp* tree) +{ + genConsumeOperands(tree->AsOp()); + inst_TT(INS_i_jmp, EA_PTRSIZE, tree->gtGetOp1()); +} + // emits the table and an instruction to get the address of the first element void CodeGen::genJumpTable(GenTree* treeNode) { @@ -4286,6 +4306,18 @@ void CodeGen::genAsyncResumeInfo(GenTreeVal* treeNode) genProduceReg(treeNode); } +//------------------------------------------------------------------------ +// genFtnEntry: emits address of the current function being compiled +// +// Parameters: +// treeNode - the GT_FTN_ENTRY node +// +void CodeGen::genFtnEntry(GenTree* treeNode) +{ + GetEmitter()->emitIns_R_L(INS_lea, EA_PTR_DSP_RELOC, GetEmitter()->emitPrologIG, treeNode->GetRegNum()); + genProduceReg(treeNode); +} + //------------------------------------------------------------------------ // genCodeForLockAdd: Generate code for a GT_LOCKADD node // @@ -9909,13 +9941,14 @@ void CodeGen::genOSRHandleTier0CalleeSavedRegistersAndFrame() // We must account for the post-callee-saves push SP movement // done by the Tier0 frame and by the OSR transition. // - // tier0FrameSize is the Tier0 FP-SP delta plus the fake call slot added by - // JIT_Patchpoint. We add one slot to account for the saved FP. + // tier0FrameSize is the Tier0 FP-SP delta plus the fake call slot we push in genFnProlog. + // We subtract the fake call slot since we will add it as unwind after the instruction itself. + // We then add back one slot to account for the saved FP. // // We then need to subtract off the size the Tier0 callee saves as SP // adjusts for those will have been modelled by the unwind pushes above. // - int const tier0FrameSize = patchpointInfo->TotalFrameSize() + REGSIZE_BYTES; + int const tier0FrameSize = patchpointInfo->TotalFrameSize() - REGSIZE_BYTES + REGSIZE_BYTES; int const tier0NetSize = tier0FrameSize - tier0IntCalleeSaveUsedSize; m_compiler->unwindAllocStack(tier0NetSize); } diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index d5bafd88098f1b..a7e591163e109c 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -5622,8 +5622,8 @@ void Compiler::generatePatchpointInfo() // offset. #if defined(TARGET_AMD64) - // We add +TARGET_POINTER_SIZE here is to account for the slot that Jit_Patchpoint - // creates when it simulates calling the OSR method (the "pseudo return address" slot). + // We add +TARGET_POINTER_SIZE here to account for the return address slot that the OSR method prolog + // pushes on entry (the "pseudo return address" slot) to make the stack unaligned. // This is effectively a new slot at the bottom of the Tier0 frame. // const int totalFrameSize = codeGen->genTotalFrameSize() + TARGET_POINTER_SIZE; diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 7db9f05710b22b..b78299082bd300 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -12434,6 +12434,7 @@ class GenTreeVisitor case GT_ASYNC_RESUME_INFO: case GT_LABEL: case GT_FTN_ADDR: + case GT_FTN_ENTRY: case GT_RET_EXPR: case GT_CNS_INT: case GT_CNS_LNG: @@ -12506,6 +12507,7 @@ class GenTreeVisitor case GT_FIELD_ADDR: case GT_RETURN: case GT_RETURN_SUSPEND: + case GT_NONLOCAL_JMP: case GT_RETFILT: case GT_RUNTIMELOOKUP: case GT_ARR_ADDR: diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 0e0f988993c1e7..d3cc67314329d8 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -4405,6 +4405,7 @@ GenTree::VisitResult GenTree::VisitOperandUses(TVisitor visitor) case GT_ASYNC_RESUME_INFO: case GT_LABEL: case GT_FTN_ADDR: + case GT_FTN_ENTRY: case GT_RET_EXPR: case GT_CNS_INT: case GT_CNS_LNG: @@ -4478,6 +4479,7 @@ GenTree::VisitResult GenTree::VisitOperandUses(TVisitor visitor) case GT_KEEPALIVE: case GT_INC_SATURATE: case GT_RETURN_SUSPEND: + case GT_NONLOCAL_JMP: return visitor(&this->AsUnOp()->gtOp1); // Variadic nodes diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 9056f8e4ca70b3..2056d8dabee674 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -9150,6 +9150,64 @@ void emitter::emitIns_R_L(instruction ins, emitAttr attr, BasicBlock* dst, regNu appendToCurIG(id); } +//-------------------------------------------------------------------- +// emitIns_R_L: Emit an instruction with a label operand. +// +// Arguments: +// ins - The instruction +// attr - Size of the instruction +// dst - Instruction group +// reg - Register destination +// +void emitter::emitIns_R_L(instruction ins, emitAttr attr, insGroup* dst, regNumber reg) +{ + assert(dst != nullptr); + + insFormat fmt = IF_NONE; + + switch (ins) + { + case INS_adr: + fmt = IF_LARGEADR; + break; + default: + unreached(); + } + + instrDescJmp* id = emitNewInstrJmp(); + + id->idIns(ins); + id->idInsFmt(fmt); + id->idjShort = false; + id->idAddr()->iiaIGlabel = dst; + id->idSetIsBound(); // Mark as bound since we already have the target insGroup directly + id->idReg1(reg); + id->idOpSize(EA_PTRSIZE); + id->idjKeepLong = false; + +#ifdef DEBUG + if (m_compiler->opts.compLongAddress) + id->idjKeepLong = 1; +#endif // DEBUG + + /* Record the jump's IG and offset within it */ + + id->idjIG = emitCurIG; + id->idjOffs = emitCurIGsize; + + /* Append this jump to this IG's jump list */ + + id->idjNext = emitCurIGjmpList; + emitCurIGjmpList = id; + +#if EMITTER_STATS + emitTotalIGjmps++; +#endif + + dispIns(id); + appendToCurIG(id); +} + /***************************************************************************** * * Add a data label instruction. diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index 212de5f36f729d..6b54d0e643b641 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -1796,6 +1796,7 @@ void emitIns_C_R(instruction ins, emitAttr attr, CORINFO_FIELD_HANDLE fldHnd, re void emitIns_C_I(instruction ins, emitAttr attr, CORINFO_FIELD_HANDLE fdlHnd, ssize_t offs, ssize_t val); void emitIns_R_L(instruction ins, emitAttr attr, BasicBlock* dst, regNumber reg); +void emitIns_R_L(instruction ins, emitAttr attr, insGroup* dst, regNumber reg); void emitIns_R_D(instruction ins, emitAttr attr, unsigned offs, regNumber reg); diff --git a/src/coreclr/jit/emitloongarch64.cpp b/src/coreclr/jit/emitloongarch64.cpp index 8c49d04d266148..9ec429118953eb 100644 --- a/src/coreclr/jit/emitloongarch64.cpp +++ b/src/coreclr/jit/emitloongarch64.cpp @@ -2165,6 +2165,50 @@ void emitter::emitIns_R_L(instruction ins, emitAttr attr, BasicBlock* dst, regNu appendToCurIG(id); } +//-------------------------------------------------------------------- +// emitIns_R_L: Emit an instruction with a label operand. +// +// Arguments: +// ins - The instruction +// attr - Size of the instruction +// dst - Instruction group +// reg - Register destination +// +void emitter::emitIns_R_L(instruction ins, emitAttr attr, insGroup* dst, regNumber reg) +{ + assert(dst != nullptr); + + // if for reloc! 2-ins: + // pcaddu12i reg, offset-hi20 + // addi_d reg, reg, offset-lo12 + // + // else: 3-ins: + // lu12i_w r21, addr_bits[31:12] + // ori reg, r21, addr_bits[11:0] + // lu32i_d reg, addr_bits[50:32] + + instrDesc* id = emitNewInstr(attr); + + id->idIns(ins); + id->idInsOpt(INS_OPTS_RL); + id->idAddr()->iiaIGlabel = dst; + id->idSetIsBound(); // Mark as bound since we already have the target insGroup directly + + if (m_compiler->opts.compReloc) + { + id->idSetIsDspReloc(); + id->idCodeSize(8); + } + else + { + id->idCodeSize(12); + } + + id->idReg1(reg); + + appendToCurIG(id); +} + void emitter::emitIns_J_R(instruction ins, emitAttr attr, BasicBlock* dst, regNumber reg) { NYI_LOONGARCH64("emitIns_J_R-----unimplemented/unused on LOONGARCH64 yet----"); @@ -3546,8 +3590,14 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) // ori reg, r21, addr_bits[11:0] // lu32i_d reg, addr_bits[50:32] - insGroup* tgtIG = (insGroup*)emitCodeGetCookie(id->idAddr()->iiaBBlabel); - id->idAddr()->iiaIGlabel = tgtIG; + if (!id->idIsBound()) + { + insGroup* tgtIG = (insGroup*)emitCodeGetCookie(id->idAddr()->iiaBBlabel); + id->idAddr()->iiaIGlabel = tgtIG; + id->idSetIsBound(); + } + + insGroup* tgtIG = id->idAddr()->iiaIGlabel; regNumber reg1 = id->idReg1(); assert(isGeneralRegister(reg1)); diff --git a/src/coreclr/jit/emitloongarch64.h b/src/coreclr/jit/emitloongarch64.h index 49f7bb702a7422..2e3adad21ceb32 100644 --- a/src/coreclr/jit/emitloongarch64.h +++ b/src/coreclr/jit/emitloongarch64.h @@ -278,6 +278,7 @@ void emitIns_R_C( instruction ins, emitAttr attr, regNumber reg, regNumber tmpReg, CORINFO_FIELD_HANDLE fldHnd, int offs); void emitIns_R_L(instruction ins, emitAttr attr, BasicBlock* dst, regNumber reg); +void emitIns_R_L(instruction ins, emitAttr attr, insGroup* dst, regNumber reg); void emitIns_J_R(instruction ins, emitAttr attr, BasicBlock* dst, regNumber reg); diff --git a/src/coreclr/jit/emitriscv64.cpp b/src/coreclr/jit/emitriscv64.cpp index 057518229faba1..e25e252ecd07fb 100644 --- a/src/coreclr/jit/emitriscv64.cpp +++ b/src/coreclr/jit/emitriscv64.cpp @@ -1376,6 +1376,39 @@ void emitter::emitIns_R_L(instruction ins, emitAttr attr, BasicBlock* dst, regNu appendToCurIG(id); } +//-------------------------------------------------------------------- +// emitIns_R_L: Emit an instruction with a label operand. +// +// Arguments: +// ins - The instruction +// attr - Size of the instruction +// dst - Instruction group +// reg - Register destination +// +void emitter::emitIns_R_L(instruction ins, emitAttr attr, insGroup* dst, regNumber reg) +{ + assert(dst != nullptr); + + // 2-ins: + // auipc reg, offset-hi20 + // addi reg, reg, offset-lo12 + + instrDesc* id = emitNewInstr(attr); + + id->idIns(ins); + id->idInsOpt(INS_OPTS_RL); + id->idAddr()->iiaIGlabel = dst; + id->idSetIsBound(); // Mark as bound since we already have the target insGroup directly + + if (m_compiler->opts.compReloc) + id->idSetIsDspReloc(); + + id->idCodeSize(2 * sizeof(code_t)); + id->idReg1(reg); + + appendToCurIG(id); +} + //------------------------------------------------------------------------ // emitIns_R_R_Addr: emit instruction sequence for a long (address pointer) immediate // @@ -3301,8 +3334,14 @@ BYTE* emitter::emitOutputInstr_OptsRc(BYTE* dst, const instrDesc* id, instructio BYTE* emitter::emitOutputInstr_OptsRl(BYTE* dst, instrDesc* id, instruction* ins) { - insGroup* targetInsGroup = static_cast(emitCodeGetCookie(id->idAddr()->iiaBBlabel)); - id->idAddr()->iiaIGlabel = targetInsGroup; + if (!id->idIsBound()) + { + insGroup* targetInsGroup = static_cast(emitCodeGetCookie(id->idAddr()->iiaBBlabel)); + id->idAddr()->iiaIGlabel = targetInsGroup; + id->idSetIsBound(); + } + + insGroup* targetInsGroup = id->idAddr()->iiaIGlabel; const regNumber reg1 = id->idReg1(); const ssize_t igOffs = targetInsGroup->igOffs; diff --git a/src/coreclr/jit/emitriscv64.h b/src/coreclr/jit/emitriscv64.h index 904e7ecbf061ff..13efbe550b4e24 100644 --- a/src/coreclr/jit/emitriscv64.h +++ b/src/coreclr/jit/emitriscv64.h @@ -467,6 +467,7 @@ void emitIns_R_R_R_R(instruction ins, emitAttr attr, regNumber reg1, regNumber r void emitIns_R_C(instruction ins, emitAttr attr, regNumber destReg, regNumber addrReg, CORINFO_FIELD_HANDLE fldHnd); void emitIns_R_L(instruction ins, emitAttr attr, BasicBlock* dst, regNumber reg); +void emitIns_R_L(instruction ins, emitAttr attr, insGroup* dst, regNumber reg); void emitIns_R_R_Addr(instruction ins, emitAttr attr, regNumber regDest, regNumber regAddr, void* addr); diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index 43d74e91d53ea7..c4ddbfd26bc6d9 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -9121,6 +9121,68 @@ void emitter::emitIns_R_L(instruction ins, emitAttr attr, BasicBlock* dst, regNu emitCurIGsize += sz; } +//-------------------------------------------------------------------- +// emitIns_R_L: Emit an instruction with a label operand. +// +// Arguments: +// ins - The instruction +// attr - Size of the instruction +// dst - Instruction group +// reg - Register destination +// +void emitter::emitIns_R_L(instruction ins, emitAttr attr, insGroup* dst, regNumber reg) +{ + assert(ins == INS_lea); + + instrDescJmp* id = emitNewInstrJmp(); + + id->idIns(ins); + id->idReg1(reg); + id->idInsFmt(IF_RWR_LABEL); + id->idOpSize(EA_SIZE(attr)); // emitNewInstrJmp() sets the size (incorrectly) to EA_1BYTE + id->idAddr()->iiaIGlabel = dst; + id->idSetIsBound(); + + /* The label reference is always long */ + + id->idjShort = 0; + id->idjKeepLong = 1; + + /* Record the current IG and offset within it */ + + id->idjIG = emitCurIG; + id->idjOffs = emitCurIGsize; + + /* Append this instruction to this IG's jump list */ + + id->idjNext = emitCurIGjmpList; + emitCurIGjmpList = id; + +#ifdef DEBUG + // Mark the catch return + if (m_compiler->compCurBB->KindIs(BBJ_EHCATCHRET)) + { + id->idDebugOnlyInfo()->idCatchRet = true; + } +#endif // DEBUG + +#if EMITTER_STATS + emitTotalIGjmps++; +#endif + + // Set the relocation flags - these give hint to zap to perform + // relocation of the specified 32bit address. + // + // Note the relocation flags influence the size estimate. + id->idSetRelocFlags(attr); + + UNATIVE_OFFSET sz = emitInsSizeAM(id, insCodeRM(ins)); + id->idCodeSize(sz); + + dispIns(id); + emitCurIGsize += sz; +} + /***************************************************************************** * * The following adds instructions referencing address modes. @@ -12677,86 +12739,6 @@ void emitter::emitDispIns( #define ID_INFO_DSP_RELOC ((bool)(id->idIsDspReloc())) - /* Display a constant value if the instruction references one */ - - if (!isNew && id->idHasMemGen()) - { - /* Is this actually a reference to a data section? */ - int offs = Compiler::eeGetJitDataOffs(id->idAddr()->iiaFieldHnd); - - if (offs >= 0) - { - void* addr; - - /* Display a data section reference */ - - assert((unsigned)offs < emitConsDsc.dsdOffs); - addr = emitDataOffsetToPtr((UNATIVE_OFFSET)offs); - -#if 0 - // TODO-XArch-Cleanup: Fix or remove this code. - /* Is the operand an integer or floating-point value? */ - - bool isFP = false; - - if (CodeGen::instIsFP(id->idIns())) - { - switch (id->idIns()) - { - case INS_fild: - case INS_fildl: - break; - - default: - isFP = true; - break; - } - } - - if (offs & 1) - printf("@CNS%02u", offs); - else - printf("@RWD%02u", offs); - - printf(" "); - - if (addr) - { - addr = 0; - // TODO-XArch-Bug?: - // This was busted by switching the order - // in which we output the code block vs. - // the data blocks -- when we get here, - // the data block has not been filled in - // yet, so we'll display garbage. - - if (isFP) - { - if (id->idOpSize() == EA_4BYTE) - printf("DF %f \n", addr ? *(float *)addr : 0); - else - printf("DQ %lf\n", addr ? *(double *)addr : 0); - } - else - { - if (id->idOpSize() <= EA_4BYTE) - printf("DD %d \n", addr ? *(int *)addr : 0); - else - printf("DQ %D \n", addr ? *(int64_t *)addr : 0); - } - } -#endif - } - } - - // printf("[F=%s] " , emitIfName(id->idInsFmt())); - // printf("INS#%03u: ", id->idDebugOnlyInfo()->idNum); - // printf("[S=%02u] " , emitCurStackLvl); if (isNew) printf("[M=%02u] ", emitMaxStackDepth); - // printf("[S=%02u] " , emitCurStackLvl/sizeof(INT32)); - // printf("[A=%08X] " , emitSimpleStkMask); - // printf("[A=%08X] " , emitSimpleByrefStkMask); - // printf("[L=%02u] " , id->idCodeSize()); - if (!isNew && !asmfm) { doffs = true; @@ -16202,15 +16184,8 @@ BYTE* emitter::emitOutputCV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) else { // Special case: mov reg, fs:[ddd] or mov reg, [ddd] - if (jitStaticFldIsGlobAddr(fldh)) - { - addr = nullptr; - } - else - { - assert(jitStaticFldIsGlobAddr(fldh)); - addr = nullptr; - } + assert(jitStaticFldIsGlobAddr(fldh)); + addr = nullptr; } BYTE* target = (addr + offs); diff --git a/src/coreclr/jit/emitxarch.h b/src/coreclr/jit/emitxarch.h index 126d1a67a3593b..38ade6a38f3930 100644 --- a/src/coreclr/jit/emitxarch.h +++ b/src/coreclr/jit/emitxarch.h @@ -1102,6 +1102,7 @@ void emitIns_IJ(emitAttr attr, regNumber reg, unsigned base); void emitIns_J_S(instruction ins, emitAttr attr, BasicBlock* dst, int varx, int offs); void emitIns_R_L(instruction ins, emitAttr attr, BasicBlock* dst, regNumber reg); +void emitIns_R_L(instruction ins, emitAttr attr, insGroup* dst, regNumber reg); void emitIns_R_D(instruction ins, emitAttr attr, unsigned offs, regNumber reg); diff --git a/src/coreclr/jit/fgstmt.cpp b/src/coreclr/jit/fgstmt.cpp index 83bc11579b5219..80c31e4ab3370d 100644 --- a/src/coreclr/jit/fgstmt.cpp +++ b/src/coreclr/jit/fgstmt.cpp @@ -593,6 +593,7 @@ inline bool OperIsControlFlow(genTreeOps oper) case GT_RETFILT: case GT_SWIFT_ERROR_RET: case GT_RETURN_SUSPEND: + case GT_NONLOCAL_JMP: case GT_WASM_JEXCEPT: return true; diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 826fba0d2ce604..4e437351458b08 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -2769,9 +2769,12 @@ bool GenTree::Compare(GenTree* op1, GenTree* op2, bool swapOK) } return true; + case GT_ASYNC_RESUME_INFO: + return op1->AsVal()->gtVal1 == op2->AsVal()->gtVal1; + case GT_NOP: case GT_LABEL: - case GT_ASYNC_RESUME_INFO: + case GT_FTN_ENTRY: case GT_SWIFT_ERROR: case GT_GCPOLL: case GT_WASM_THROW_REF: @@ -8011,6 +8014,7 @@ bool GenTree::TryGetUse(GenTree* operand, GenTree*** pUse) case GT_ASYNC_RESUME_INFO: case GT_LABEL: case GT_FTN_ADDR: + case GT_FTN_ENTRY: case GT_RET_EXPR: case GT_CNS_INT: case GT_CNS_LNG: @@ -8072,6 +8076,7 @@ bool GenTree::TryGetUse(GenTree* operand, GenTree*** pUse) case GT_RETURN: case GT_RETFILT: case GT_RETURN_SUSPEND: + case GT_NONLOCAL_JMP: case GT_BSWAP: case GT_BSWAP16: case GT_KEEPALIVE: @@ -8343,6 +8348,7 @@ bool GenTree::OperRequiresCallFlag(Compiler* comp) const case GT_KEEPALIVE: case GT_ASYNC_CONTINUATION: case GT_RETURN_SUSPEND: + case GT_NONLOCAL_JMP: return true; case GT_SWIFT_ERROR: @@ -8686,6 +8692,7 @@ bool GenTree::OperRequiresGlobRefFlag(Compiler* comp) const case GT_KEEPALIVE: case GT_ASYNC_CONTINUATION: case GT_RETURN_SUSPEND: + case GT_NONLOCAL_JMP: case GT_SWIFT_ERROR: case GT_GCPOLL: return true; @@ -8751,6 +8758,7 @@ bool GenTree::OperSupportsOrderingSideEffect() const case GT_CATCH_ARG: case GT_ASYNC_CONTINUATION: case GT_RETURN_SUSPEND: + case GT_NONLOCAL_JMP: case GT_SWIFT_ERROR: return true; default: @@ -10911,6 +10919,7 @@ GenTree* Compiler::gtCloneExpr(GenTree* tree) case GT_NO_OP: case GT_NOP: case GT_LABEL: + case GT_FTN_ENTRY: case GT_SWIFT_ERROR: case GT_GCPOLL: case GT_WASM_THROW_REF: @@ -11661,6 +11670,7 @@ GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node) case GT_ASYNC_RESUME_INFO: case GT_LABEL: case GT_FTN_ADDR: + case GT_FTN_ENTRY: case GT_RET_EXPR: case GT_CNS_INT: case GT_CNS_LNG: @@ -11725,6 +11735,7 @@ GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node) case GT_INC_SATURATE: case GT_RETURNTRAP: case GT_RETURN_SUSPEND: + case GT_NONLOCAL_JMP: m_edge = &m_node->AsUnOp()->gtOp1; assert(*m_edge != nullptr); m_advance = &GenTreeUseEdgeIterator::Terminate; @@ -13698,6 +13709,7 @@ void Compiler::gtDispLeaf(GenTree* tree, IndentStack* indentStack) case GT_PROF_HOOK: case GT_CATCH_ARG: case GT_ASYNC_CONTINUATION: + case GT_FTN_ENTRY: case GT_MEMORYBARRIER: case GT_JMPTABLE: case GT_SWIFT_ERROR: diff --git a/src/coreclr/jit/gtlist.h b/src/coreclr/jit/gtlist.h index d14790d89fb622..08f75d6ff40d25 100644 --- a/src/coreclr/jit/gtlist.h +++ b/src/coreclr/jit/gtlist.h @@ -39,6 +39,7 @@ GTNODE(FTN_ADDR , GenTreeFptrVal ,0,0,GTK_LEAF) // Addre GTNODE(RET_EXPR , GenTreeRetExpr ,0,0,GTK_LEAF|DBK_NOTLIR) // Place holder for the return expression from an inline candidate GTNODE(GCPOLL , GenTree ,0,0,GTK_LEAF|GTK_NOVALUE|DBK_NOTLIR) GTNODE(ASYNC_RESUME_INFO, GenTreeVal ,0,0,GTK_LEAF) // Address of async resume info for a state +GTNODE(FTN_ENTRY , GenTree ,0,0,GTK_LEAF) // Address of this function's entry point //----------------------------------------------------------------------------- // Constant nodes: @@ -104,6 +105,8 @@ GTNODE(BSWAP16 , GenTreeOp ,0,0,GTK_UNOP) // Byt GTNODE(LZCNT , GenTreeOp ,0,0,GTK_UNOP) // Leading Zero Count - Only used for SIMD VN evaluation today +GTNODE(NONLOCAL_JMP , GenTreeOp ,0,0,GTK_UNOP|GTK_NOVALUE) // Non-local jump to specified address + //----------------------------------------------------------------------------- // Binary operators (2 operands): //----------------------------------------------------------------------------- diff --git a/src/coreclr/jit/liveness.cpp b/src/coreclr/jit/liveness.cpp index 6e1aca212fb5c1..741aebfca15533 100644 --- a/src/coreclr/jit/liveness.cpp +++ b/src/coreclr/jit/liveness.cpp @@ -2556,6 +2556,7 @@ void Liveness::ComputeLifeLIR(VARSET_TP& life, BasicBlock* block, VAR case GT_JTRUE: case GT_RETURN: case GT_RETURN_SUSPEND: + case GT_NONLOCAL_JMP: case GT_SWITCH: case GT_RETFILT: case GT_START_NONGC: diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index e66cf2cbfebb20..4d30b7d109967a 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -685,6 +685,10 @@ GenTree* Lowering::LowerNode(GenTree* node) LowerReturnSuspend(node); break; + case GT_NONLOCAL_JMP: + ContainCheckNonLocalJmp(node->AsUnOp()); + break; + #if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_ARM64) case GT_CNS_MSK: return LowerCnsMask(node->AsMskCon()); @@ -9922,6 +9926,9 @@ void Lowering::ContainCheckNode(GenTree* node) ContainCheckHWIntrinsic(node->AsHWIntrinsic()); break; #endif // FEATURE_HW_INTRINSICS + case GT_NONLOCAL_JMP: + ContainCheckNonLocalJmp(node->AsUnOp()); + break; default: break; } diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index af39e59ca66a64..d88ab35f9c2ad0 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -106,6 +106,7 @@ class Lowering final : public Phase void ContainCheckCallOperands(GenTreeCall* call); void ContainCheckIndir(GenTreeIndir* indirNode); void ContainCheckStoreIndir(GenTreeStoreInd* indirNode); + void ContainCheckNonLocalJmp(GenTreeUnOp* node); void ContainCheckMul(GenTreeOp* node); void ContainCheckShiftRotate(GenTreeOp* node); void ContainCheckStoreLoc(GenTreeLclVarCommon* storeLoc) const; diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index ffdd8b9729d73b..6d9fe77b48a6cf 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -2899,6 +2899,17 @@ void Lowering::ContainCheckDivOrMod(GenTreeOp* node) // ARM doesn't have a div instruction with an immediate operand } +//------------------------------------------------------------------------ +// ContainCheckNonLocalJmp: +// No-op for arm. +// +// Arguments: +// node - The GT_NONLOCAL_JMP node. +// +void Lowering::ContainCheckNonLocalJmp(GenTreeUnOp* node) +{ +} + //------------------------------------------------------------------------ // ContainCheckShiftRotate: Determine whether a mul op's operands should be contained. // diff --git a/src/coreclr/jit/lowerloongarch64.cpp b/src/coreclr/jit/lowerloongarch64.cpp index e18e08a413d5bd..98cc34bddb2407 100644 --- a/src/coreclr/jit/lowerloongarch64.cpp +++ b/src/coreclr/jit/lowerloongarch64.cpp @@ -749,6 +749,17 @@ void Lowering::ContainCheckDivOrMod(GenTreeOp* node) assert(node->OperIs(GT_MOD, GT_UMOD, GT_DIV, GT_UDIV)); } +//------------------------------------------------------------------------ +// ContainCheckNonLocalJmp: +// No-op for LA64. +// +// Arguments: +// node - The GT_NONLOCAL_JMP node. +// +void Lowering::ContainCheckNonLocalJmp(GenTreeUnOp* node) +{ +} + //------------------------------------------------------------------------ // ContainCheckShiftRotate: Determine whether a mul op's operands should be contained. // diff --git a/src/coreclr/jit/lowerriscv64.cpp b/src/coreclr/jit/lowerriscv64.cpp index e954b6be7d27ec..26849d90d15d85 100644 --- a/src/coreclr/jit/lowerriscv64.cpp +++ b/src/coreclr/jit/lowerriscv64.cpp @@ -1222,6 +1222,17 @@ void Lowering::ContainCheckDivOrMod(GenTreeOp* node) assert(node->OperIs(GT_MOD, GT_UMOD, GT_DIV, GT_UDIV)); } +//------------------------------------------------------------------------ +// ContainCheckNonLocalJmp: +// No-op for RISCV64. +// +// Arguments: +// node - The GT_NONLOCAL_JMP node. +// +void Lowering::ContainCheckNonLocalJmp(GenTreeUnOp* node) +{ +} + //------------------------------------------------------------------------ // ContainCheckShiftRotate: Determine whether a mul op's operands should be contained. // diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index b857dec2499425..ab9c56164fa1a8 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -377,6 +377,17 @@ void Lowering::ContainCheckCallOperands(GenTreeCall* call) { } +//------------------------------------------------------------------------ +// ContainCheckNonLocalJmp: +// No-op for wasm. +// +// Arguments: +// node - The GT_NONLOCAL_JMP node. +// +void Lowering::ContainCheckNonLocalJmp(GenTreeUnOp* node) +{ +} + //------------------------------------------------------------------------ // ContainCheckStoreIndir: determine whether the sources of a STOREIND node should be contained. // diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index 61fef20d231ccc..ea3684e0ff0ba0 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -7405,6 +7405,26 @@ void Lowering::ContainCheckIndir(GenTreeIndir* node) } } +//------------------------------------------------------------------------ +// ContainCheckNonLocalJmp: +// Check if we can contain the memory operand of a GT_NONLOCAL_JMP. +// +// Arguments: +// node - The GT_NONLOCAL_JMP node. +// +void Lowering::ContainCheckNonLocalJmp(GenTreeUnOp* node) +{ + GenTree* addr = node->gtGetOp1(); + if (IsContainableMemoryOp(addr) && IsSafeToContainMem(node, addr)) + { + MakeSrcContained(node, addr); + } + else if (IsSafeToMarkRegOptional(node, addr)) + { + MakeSrcRegOptional(node, addr); + } +} + //------------------------------------------------------------------------ // ContainCheckStoreIndir: determine whether the sources of a STOREIND node should be contained. // diff --git a/src/coreclr/vm/jithelpers.cpp b/src/coreclr/vm/jithelpers.cpp index fef4e1175a2235..383df6d195a755 100644 --- a/src/coreclr/vm/jithelpers.cpp +++ b/src/coreclr/vm/jithelpers.cpp @@ -1744,11 +1744,6 @@ extern "C" void JIT_PatchpointWorkerWorkerWithPolicy(TransitionBlock * pTransiti // use that to adjust the stack, likely saving some stack space. #if defined(TARGET_AMD64) - // If calls push the return address, we need to simulate that here, so the OSR - // method sees the "expected" SP misalgnment on entry. - _ASSERTE(currentSP % 16 == 0); - currentSP -= 8; - #if defined(TARGET_WINDOWS) DWORD64 ssp = GetSSP(pFrameContext); if (ssp != 0)