From 521aa0ef0970d8f755519cad8a27e3501477da65 Mon Sep 17 00:00:00 2001 From: petris Date: Sat, 23 Jul 2022 23:11:24 +0200 Subject: [PATCH 01/31] Convert MemoryMarshal.GetArrayDataReference to a JIT intrinsic Converts MemoryMarshal.GetArrayDataReference to an always expand JIT intrinsic and removes the VM intrinsics. Introduces JIT tests validating the correct behaviour. Fixes invalid codegen samples from: https://github.com/dotnet/runtime/issues/58312#issuecomment-993491291 --- .../InteropServices/MemoryMarshal.CoreCLR.cs | 3 +- src/coreclr/jit/importer.cpp | 26855 +++++++++++----- src/coreclr/jit/namedintrinsiclist.h | 2 + .../MemoryMarshal.NativeAot.cs | 3 +- src/coreclr/vm/corelib.h | 1 - src/coreclr/vm/jitinterface.cpp | 37 - .../MemoryMarshalGetArrayDataReference.cs | 150 + ...emoryMarshalGetArrayDataReference_r.csproj | 13 + ...moryMarshalGetArrayDataReference_ro.csproj | 13 + 9 files changed, 18173 insertions(+), 8904 deletions(-) create mode 100644 src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs create mode 100644 src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference_r.csproj create mode 100644 src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference_ro.csproj diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/InteropServices/MemoryMarshal.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/InteropServices/MemoryMarshal.CoreCLR.cs index 407fbc23678783..f4a87e587c55ef 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/InteropServices/MemoryMarshal.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/InteropServices/MemoryMarshal.CoreCLR.cs @@ -19,9 +19,8 @@ public static unsafe partial class MemoryMarshal /// [Intrinsic] [NonVersionable] - [MethodImpl(MethodImplOptions.AggressiveInlining)] public static ref T GetArrayDataReference(T[] array) => - ref Unsafe.As(ref Unsafe.As(array).Data); + ref GetArrayDataReference(array); /// /// Returns a reference to the 0th element of . If the array is empty, returns a reference to where the 0th element diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index eda9ce179b6397..4885752e1a0f9c 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -28,6 +28,36 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX } \ } while (0) +#define VerifyOrReturn(cond, msg) \ + do \ + { \ + if (!(cond)) \ + { \ + verRaiseVerifyExceptionIfNeeded(INDEBUG(msg) DEBUGARG(__FILE__) DEBUGARG(__LINE__)); \ + return; \ + } \ + } while (0) + +#define VerifyOrReturnSpeculative(cond, msg, speculative) \ + do \ + { \ + if (speculative) \ + { \ + if (!(cond)) \ + { \ + return false; \ + } \ + } \ + else \ + { \ + if (!(cond)) \ + { \ + verRaiseVerifyExceptionIfNeeded(INDEBUG(msg) DEBUGARG(__FILE__) DEBUGARG(__LINE__)); \ + return false; \ + } \ + } \ + } while (0) + /*****************************************************************************/ void Compiler::impInit() @@ -76,7 +106,7 @@ void Compiler::impPushOnStack(GenTree* tree, typeInfo ti) } } -void Compiler::impPushNullObjRefOnStack() +inline void Compiler::impPushNullObjRefOnStack() { impPushOnStack(gtNewIconNode(0, TYP_REF), typeInfo(TI_NULL)); } @@ -84,11 +114,11 @@ void Compiler::impPushNullObjRefOnStack() // This method gets called when we run into unverifiable code // (and we are verifying the method) -void Compiler::verRaiseVerifyExceptionIfNeeded(INDEBUG(const char* msg) DEBUGARG(const char* file) - DEBUGARG(unsigned line)) +inline void Compiler::verRaiseVerifyExceptionIfNeeded(INDEBUG(const char* msg) DEBUGARG(const char* file) + DEBUGARG(unsigned line)) { #ifdef DEBUG - const char* tail = strrchr(file, DIRECTORY_SEPARATOR_CHAR_A); + const char* tail = strrchr(file, '\\'); if (tail) { file = tail + 1; @@ -111,8 +141,8 @@ void Compiler::verRaiseVerifyExceptionIfNeeded(INDEBUG(const char* msg) DEBUGARG } } -void DECLSPEC_NORETURN Compiler::verRaiseVerifyException(INDEBUG(const char* msg) DEBUGARG(const char* file) - DEBUGARG(unsigned line)) +inline void DECLSPEC_NORETURN Compiler::verRaiseVerifyException(INDEBUG(const char* msg) DEBUGARG(const char* file) + DEBUGARG(unsigned line)) { JITLOG((LL_ERROR, "Verification failure: %s:%d : %s, while compiling %s opcode %s, IL offset %x\n", file, line, msg, info.compFullName, impCurOpcName, impCurOpcOffs)); @@ -336,7 +366,7 @@ void Compiler::impRestoreStackState(SavedStack* savePtr) //------------------------------------------------------------------------ // impBeginTreeList: Get the tree list started for a new basic block. // -void Compiler::impBeginTreeList() +inline void Compiler::impBeginTreeList() { assert(impStmtList == nullptr && impLastStmt == nullptr); } @@ -348,7 +378,7 @@ void Compiler::impBeginTreeList() * directly only for handling CEE_LEAVEs out of finally-protected try's. */ -void Compiler::impEndTreeList(BasicBlock* block, Statement* firstStmt, Statement* lastStmt) +inline void Compiler::impEndTreeList(BasicBlock* block, Statement* firstStmt, Statement* lastStmt) { /* Make the list circular, so that we can easily walk it backwards */ @@ -364,7 +394,7 @@ void Compiler::impEndTreeList(BasicBlock* block, Statement* firstStmt, Statement block->bbFlags |= BBF_IMPORTED; } -void Compiler::impEndTreeList(BasicBlock* block) +inline void Compiler::impEndTreeList(BasicBlock* block) { if (impStmtList == nullptr) { @@ -395,18 +425,18 @@ void Compiler::impEndTreeList(BasicBlock* block) * that this has only limited value as we can only check [0..chkLevel). */ -void Compiler::impAppendStmtCheck(Statement* stmt, unsigned chkLevel) +inline void Compiler::impAppendStmtCheck(Statement* stmt, unsigned chkLevel) { #ifndef DEBUG return; #else - if (chkLevel == CHECK_SPILL_ALL) + if (chkLevel == (unsigned)CHECK_SPILL_ALL) { chkLevel = verCurrentState.esStackDepth; } - if (verCurrentState.esStackDepth == 0 || chkLevel == 0 || chkLevel == CHECK_SPILL_NONE) + if (verCurrentState.esStackDepth == 0 || chkLevel == 0 || chkLevel == (unsigned)CHECK_SPILL_NONE) { return; } @@ -423,23 +453,25 @@ void Compiler::impAppendStmtCheck(Statement* stmt, unsigned chkLevel) } } - if (tree->OperIs(GT_ASG)) + if (tree->gtOper == GT_ASG) { // For an assignment to a local variable, all references of that // variable have to be spilled. If it is aliased, all calls and // indirect accesses have to be spilled - if (tree->AsOp()->gtOp1->OperIsLocal()) + if (tree->AsOp()->gtOp1->gtOper == GT_LCL_VAR) { unsigned lclNum = tree->AsOp()->gtOp1->AsLclVarCommon()->GetLclNum(); for (unsigned level = 0; level < chkLevel; level++) { - GenTree* stkTree = verCurrentState.esStack[level].val; - assert(!gtHasRef(stkTree, lclNum) || impIsInvariant(stkTree)); - assert(!lvaTable[lclNum].IsAddressExposed() || ((stkTree->gtFlags & GTF_SIDE_EFFECT) == 0)); + assert(!gtHasRef(verCurrentState.esStack[level].val, lclNum)); + assert(!lvaTable[lclNum].IsAddressExposed() || + (verCurrentState.esStack[level].val->gtFlags & GTF_SIDE_EFFECT) == 0); } } + // If the access may be to global memory, all side effects have to be spilled. + else if (tree->AsOp()->gtOp1->gtFlags & GTF_GLOB_REF) { for (unsigned level = 0; level < chkLevel; level++) @@ -458,7 +490,7 @@ void Compiler::impAppendStmtCheck(Statement* stmt, unsigned chkLevel) // Arguments: // stmt - The statement to add. // chkLevel - [0..chkLevel) is the portion of the stack which we will check -// for interference with stmt and spilled if needed. +// for interference with stmt and spill if needed. // checkConsumedDebugInfo - Whether to check for consumption of impCurStmtDI. impCurStmtDI // marks the debug info of the current boundary and is set when we // start importing IL at that boundary. If this parameter is true, @@ -468,88 +500,70 @@ void Compiler::impAppendStmtCheck(Statement* stmt, unsigned chkLevel) // void Compiler::impAppendStmt(Statement* stmt, unsigned chkLevel, bool checkConsumedDebugInfo) { - if (chkLevel == CHECK_SPILL_ALL) + if (chkLevel == (unsigned)CHECK_SPILL_ALL) { chkLevel = verCurrentState.esStackDepth; } - if ((chkLevel != 0) && (chkLevel != CHECK_SPILL_NONE)) + if ((chkLevel != 0) && (chkLevel != (unsigned)CHECK_SPILL_NONE)) { assert(chkLevel <= verCurrentState.esStackDepth); - // If the statement being appended has any side-effects, check the stack to see if anything - // needs to be spilled to preserve correct ordering. - // + /* If the statement being appended has any side-effects, check the stack + to see if anything needs to be spilled to preserve correct ordering. */ + GenTree* expr = stmt->GetRootNode(); GenTreeFlags flags = expr->gtFlags & GTF_GLOB_EFFECT; - // Assignments to unaliased locals require special handling. Here, we look for trees that - // can modify them and spill the references. In doing so, we make two assumptions: - // - // 1. All locals which can be modified indirectly are marked as address-exposed or with - // "lvHasLdAddrOp" -- we will rely on "impSpillSideEffects(spillGlobEffects: true)" - // below to spill them. - // 2. Trees that assign to unaliased locals are always top-level (this avoids having to - // walk down the tree here), and are a subset of what is recognized here. - // - // If any of the above are violated (say for some temps), the relevant code must spill - // things manually. - // - LclVarDsc* dstVarDsc = nullptr; - if (expr->OperIs(GT_ASG) && expr->AsOp()->gtOp1->OperIsLocal()) + // Assignment to (unaliased) locals don't count as a side-effect as + // we handle them specially using impSpillLclRefs(). Temp locals should + // be fine too. + + if ((expr->gtOper == GT_ASG) && (expr->AsOp()->gtOp1->gtOper == GT_LCL_VAR) && + ((expr->AsOp()->gtOp1->gtFlags & GTF_GLOB_REF) == 0) && !gtHasLocalsWithAddrOp(expr->AsOp()->gtOp2)) { - dstVarDsc = lvaGetDesc(expr->AsOp()->gtOp1->AsLclVarCommon()); + GenTreeFlags op2Flags = expr->AsOp()->gtOp2->gtFlags & GTF_GLOB_EFFECT; + assert(flags == (op2Flags | GTF_ASG)); + flags = op2Flags; } - else if (expr->OperIs(GT_CALL, GT_RET_EXPR)) // The special case of calls with return buffers. + + if (flags != 0) { - GenTree* call = expr->OperIs(GT_RET_EXPR) ? expr->AsRetExpr()->gtInlineCandidate : expr; + bool spillGlobEffects = false; - if (call->TypeIs(TYP_VOID) && call->AsCall()->TreatAsShouldHaveRetBufArg(this)) + if ((flags & GTF_CALL) != 0) { - GenTree* retBuf; - if (call->AsCall()->ShouldHaveRetBufArg()) - { - assert(call->AsCall()->gtArgs.HasRetBuffer()); - retBuf = call->AsCall()->gtArgs.GetRetBufferArg()->GetNode(); - } - else - { - assert(!call->AsCall()->gtArgs.HasThisPointer()); - retBuf = call->AsCall()->gtArgs.GetArgByIndex(0)->GetNode(); - } - - assert(retBuf->TypeIs(TYP_I_IMPL, TYP_BYREF)); - - GenTreeLclVarCommon* lclNode = retBuf->IsLocalAddrExpr(); - if (lclNode != nullptr) + // If there is a call, we have to spill global refs + spillGlobEffects = true; + } + else if (!expr->OperIs(GT_ASG)) + { + if ((flags & GTF_ASG) != 0) { - dstVarDsc = lvaGetDesc(lclNode); + // The expression is not an assignment node but it has an assignment side effect, it + // must be an atomic op, HW intrinsic or some other kind of node that stores to memory. + // Since we don't know what it assigns to, we need to spill global refs. + spillGlobEffects = true; } } - } - - if ((dstVarDsc != nullptr) && !dstVarDsc->IsAddressExposed() && !dstVarDsc->lvHasLdAddrOp) - { - impSpillLclRefs(lvaGetLclNum(dstVarDsc), chkLevel); - - if (expr->OperIs(GT_ASG)) + else { - // For assignments, limit the checking to what the RHS could modify/interfere with. - GenTree* rhs = expr->AsOp()->gtOp2; - flags = rhs->gtFlags & GTF_GLOB_EFFECT; + GenTree* lhs = expr->gtGetOp1(); + GenTree* rhs = expr->gtGetOp2(); - // We don't mark indirections off of "aliased" locals with GLOB_REF, but they must still be - // considered as such in the interference checking. - if (((flags & GTF_GLOB_REF) == 0) && !impIsAddressInLocal(rhs) && gtHasLocalsWithAddrOp(rhs)) + if (((rhs->gtFlags | lhs->gtFlags) & GTF_ASG) != 0) + { + // Either side of the assignment node has an assignment side effect. + // Since we don't know what it assigns to, we need to spill global refs. + spillGlobEffects = true; + } + else if ((lhs->gtFlags & GTF_GLOB_REF) != 0) { - flags |= GTF_GLOB_REF; + spillGlobEffects = true; } } - } - if (flags != 0) - { - impSpillSideEffects((flags & (GTF_ASG | GTF_CALL)) != 0, chkLevel DEBUGARG("impAppendStmt")); + impSpillSideEffects(spillGlobEffects, chkLevel DEBUGARG("impAppendStmt")); } else { @@ -596,7 +610,7 @@ void Compiler::impAppendStmt(Statement* stmt, unsigned chkLevel, bool checkConsu // Arguments: // stmt - the statement to add. // -void Compiler::impAppendStmt(Statement* stmt) +inline void Compiler::impAppendStmt(Statement* stmt) { if (impStmtList == nullptr) { @@ -641,7 +655,7 @@ Statement* Compiler::impExtractLastStmt() // stmt - a statement to insert; // stmtBefore - an insertion point to insert "stmt" before. // -void Compiler::impInsertStmtBefore(Statement* stmt, Statement* stmtBefore) +inline void Compiler::impInsertStmtBefore(Statement* stmt, Statement* stmtBefore) { assert(stmt != nullptr); assert(stmtBefore != nullptr); @@ -779,7 +793,7 @@ void Compiler::impAssignTempGen(unsigned tmpNum, // calls that may not actually be required - e.g. if we only access a field of a struct. GenTree* dst = gtNewLclvNode(tmpNum, varType); - asg = impAssignStruct(dst, val, curLevel, pAfterStmt, di, block); + asg = impAssignStruct(dst, val, structType, curLevel, pAfterStmt, di, block); } else { @@ -801,6 +815,138 @@ void Compiler::impAssignTempGen(unsigned tmpNum, } } +//------------------------------------------------------------------------ +// impPopCallArgs: +// Pop the given number of values from the stack and return a list node with +// their values. +// +// Parameters: +// sig - Signature used to figure out classes the runtime must load, and +// also to record exact receiving argument types that may be needed for ABI +// purposes later. +// call - The call to pop arguments into. +// +void Compiler::impPopCallArgs(CORINFO_SIG_INFO* sig, GenTreeCall* call) +{ + assert(call->gtArgs.IsEmpty()); + + if (impStackHeight() < sig->numArgs) + { + BADCODE("not enough arguments for call"); + } + + struct SigParamInfo + { + CorInfoType CorType; + CORINFO_CLASS_HANDLE ClassHandle; + }; + + SigParamInfo inlineParams[16]; + SigParamInfo* params = sig->numArgs <= 16 ? inlineParams : new (this, CMK_CallArgs) SigParamInfo[sig->numArgs]; + + // We will iterate and pop the args in reverse order as we sometimes need + // to spill some args. However, we need signature information and the + // JIT-EE interface only allows us to iterate the signature forwards. We + // will collect the needed information here and at the same time notify the + // EE that the signature types need to be loaded. + CORINFO_ARG_LIST_HANDLE sigArg = sig->args; + for (unsigned i = 0; i < sig->numArgs; i++) + { + params[i].CorType = strip(info.compCompHnd->getArgType(sig, sigArg, ¶ms[i].ClassHandle)); + + if (params[i].CorType != CORINFO_TYPE_CLASS && params[i].CorType != CORINFO_TYPE_BYREF && + params[i].CorType != CORINFO_TYPE_PTR && params[i].CorType != CORINFO_TYPE_VAR) + { + CORINFO_CLASS_HANDLE argRealClass = info.compCompHnd->getArgClass(sig, sigArg); + if (argRealClass != nullptr) + { + // Make sure that all valuetypes (including enums) that we push are loaded. + // This is to guarantee that if a GC is triggered from the prestub of this methods, + // all valuetypes in the method signature are already loaded. + // We need to be able to find the size of the valuetypes, but we cannot + // do a class-load from within GC. + info.compCompHnd->classMustBeLoadedBeforeCodeIsRun(argRealClass); + } + } + + sigArg = info.compCompHnd->getArgNext(sigArg); + } + + if ((sig->retTypeSigClass != nullptr) && (sig->retType != CORINFO_TYPE_CLASS) && + (sig->retType != CORINFO_TYPE_BYREF) && (sig->retType != CORINFO_TYPE_PTR) && + (sig->retType != CORINFO_TYPE_VAR)) + { + // Make sure that all valuetypes (including enums) that we push are loaded. + // This is to guarantee that if a GC is triggerred from the prestub of this methods, + // all valuetypes in the method signature are already loaded. + // We need to be able to find the size of the valuetypes, but we cannot + // do a class-load from within GC. + info.compCompHnd->classMustBeLoadedBeforeCodeIsRun(sig->retTypeSigClass); + } + + // Now create the arguments in reverse. + for (unsigned i = sig->numArgs; i > 0; i--) + { + StackEntry se = impPopStack(); + typeInfo ti = se.seTypeInfo; + GenTree* argNode = se.val; + + var_types jitSigType = JITtype2varType(params[i - 1].CorType); + CORINFO_CLASS_HANDLE classHnd = params[i - 1].ClassHandle; + + if (!impCheckImplicitArgumentCoercion(jitSigType, argNode->TypeGet())) + { + BADCODE("the call argument has a type that can't be implicitly converted to the signature type"); + } + + if (varTypeIsStruct(argNode)) + { + // Morph trees that aren't already OBJs or MKREFANY to be OBJs + assert(ti.IsType(TI_STRUCT)); + + JITDUMP("Calling impNormStructVal on:\n"); + DISPTREE(argNode); + + argNode = impNormStructVal(argNode, classHnd, CHECK_SPILL_ALL); + // For SIMD types the normalization can normalize TYP_STRUCT to + // e.g. TYP_SIMD16 which we keep (along with the class handle) in + // the CallArgs. + jitSigType = argNode->TypeGet(); + + JITDUMP("resulting tree:\n"); + DISPTREE(argNode); + } + else + { + // insert implied casts (from float to double or double to float) + if ((jitSigType == TYP_DOUBLE) && argNode->TypeIs(TYP_FLOAT)) + { + argNode = gtNewCastNode(TYP_DOUBLE, argNode, false, TYP_DOUBLE); + } + else if ((jitSigType == TYP_FLOAT) && argNode->TypeIs(TYP_DOUBLE)) + { + argNode = gtNewCastNode(TYP_FLOAT, argNode, false, TYP_FLOAT); + } + + // insert any widening or narrowing casts for backwards compatibility + argNode = impImplicitIorI4Cast(argNode, jitSigType); + } + + NewCallArg arg; + if (varTypeIsStruct(jitSigType)) + { + arg = NewCallArg::Struct(argNode, jitSigType, classHnd); + } + else + { + arg = NewCallArg::Primitive(argNode, jitSigType); + } + + call->gtArgs.PushFront(this, arg); + call->gtFlags |= argNode->gtFlags & GTF_GLOB_EFFECT; + } +} + static bool TypeIs(var_types type1, var_types type2) { return type1 == type2; @@ -906,15 +1052,31 @@ bool Compiler::impCheckImplicitArgumentCoercion(var_types sigType, var_types nod return false; } +/***************************************************************************** + * + * Pop the given number of values from the stack in reverse order (STDCALL/CDECL etc.) + * The first "skipReverseCount" items are not reversed. + */ + +void Compiler::impPopReverseCallArgs(CORINFO_SIG_INFO* sig, GenTreeCall* call, unsigned skipReverseCount) +{ + assert(skipReverseCount <= sig->numArgs); + + impPopCallArgs(sig, call); + + call->gtArgs.Reverse(skipReverseCount, sig->numArgs - skipReverseCount); +} + //------------------------------------------------------------------------ // impAssignStruct: Create a struct assignment // // Arguments: // dest - the destination of the assignment // src - the value to be assigned +// structHnd - handle representing the struct type // curLevel - stack level for which a spill may be being done // pAfterStmt - statement to insert any additional statements after -// di - debug info for new statements +// ilOffset - il offset for new statements // block - block to insert any additional statements in // // Return Value: @@ -922,35 +1084,141 @@ bool Compiler::impCheckImplicitArgumentCoercion(var_types sigType, var_types nod // // Notes: // Temp assignments may be appended to impStmtList if spilling is necessary. -// -GenTree* Compiler::impAssignStruct(GenTree* dest, - GenTree* src, - unsigned curLevel, - Statement** pAfterStmt, /* = nullptr */ - const DebugInfo& di, /* = DebugInfo() */ - BasicBlock* block /* = nullptr */ + +GenTree* Compiler::impAssignStruct(GenTree* dest, + GenTree* src, + CORINFO_CLASS_HANDLE structHnd, + unsigned curLevel, + Statement** pAfterStmt, /* = nullptr */ + const DebugInfo& di, /* = DebugInfo() */ + BasicBlock* block /* = nullptr */ ) { - assert(varTypeIsStruct(dest) && (dest->OperIsLocal() || dest->OperIsIndir() || dest->OperIs(GT_FIELD))); + assert(varTypeIsStruct(dest)); + + DebugInfo usedDI = di; + if (!usedDI.IsValid()) + { + usedDI = impCurStmtDI; + } + + while (dest->gtOper == GT_COMMA) + { + // Second thing is the struct. + assert(varTypeIsStruct(dest->AsOp()->gtOp2)); + + // Append all the op1 of GT_COMMA trees before we evaluate op2 of the GT_COMMA tree. + if (pAfterStmt) + { + Statement* newStmt = gtNewStmt(dest->AsOp()->gtOp1, usedDI); + fgInsertStmtAfter(block, *pAfterStmt, newStmt); + *pAfterStmt = newStmt; + } + else + { + impAppendTree(dest->AsOp()->gtOp1, curLevel, usedDI); // do the side effect + } + + // set dest to the second thing + dest = dest->AsOp()->gtOp2; + } + + assert(dest->gtOper == GT_LCL_VAR || dest->gtOper == GT_RETURN || dest->gtOper == GT_FIELD || + dest->gtOper == GT_IND || dest->gtOper == GT_OBJ); + + // Return a NOP if this is a self-assignment. + if (dest->OperGet() == GT_LCL_VAR && src->OperGet() == GT_LCL_VAR && + src->AsLclVarCommon()->GetLclNum() == dest->AsLclVarCommon()->GetLclNum()) + { + return gtNewNothingNode(); + } - assert(dest->TypeGet() == src->TypeGet()); - if (dest->TypeIs(TYP_STRUCT)) + // TODO-1stClassStructs: Avoid creating an address if it is not needed, + // or re-creating a Blk node if it is. + GenTree* destAddr; + + if (dest->gtOper == GT_IND || dest->OperIsBlk()) + { + destAddr = dest->AsOp()->gtOp1; + } + else { - assert(ClassLayout::AreCompatible(dest->GetLayout(this), src->GetLayout(this))); + destAddr = gtNewOperNode(GT_ADDR, TYP_BYREF, dest); } + return (impAssignStructPtr(destAddr, src, structHnd, curLevel, pAfterStmt, usedDI, block)); +} + +//------------------------------------------------------------------------ +// impAssignStructPtr: Assign (copy) the structure from 'src' to 'destAddr'. +// +// Arguments: +// destAddr - address of the destination of the assignment +// src - source of the assignment +// structHnd - handle representing the struct type +// curLevel - stack level for which a spill may be being done +// pAfterStmt - statement to insert any additional statements after +// di - debug info for new statements +// block - block to insert any additional statements in +// +// Return Value: +// The tree that should be appended to the statement list that represents the assignment. +// +// Notes: +// Temp assignments may be appended to impStmtList if spilling is necessary. + +GenTree* Compiler::impAssignStructPtr(GenTree* destAddr, + GenTree* src, + CORINFO_CLASS_HANDLE structHnd, + unsigned curLevel, + Statement** pAfterStmt, /* = NULL */ + const DebugInfo& di, /* = DebugInfo() */ + BasicBlock* block /* = NULL */ + ) +{ + GenTree* dest = nullptr; + GenTreeFlags destFlags = GTF_EMPTY; + DebugInfo usedDI = di; if (!usedDI.IsValid()) { usedDI = impCurStmtDI; } - if (src->IsCall()) +#ifdef DEBUG +#ifdef FEATURE_HW_INTRINSICS + if (src->OperIs(GT_HWINTRINSIC)) + { + const GenTreeHWIntrinsic* intrinsic = src->AsHWIntrinsic(); + + if (HWIntrinsicInfo::IsMultiReg(intrinsic->GetHWIntrinsicId())) + { + assert(src->TypeGet() == TYP_STRUCT); + } + else + { + assert(varTypeIsSIMD(src)); + } + } + else +#endif // FEATURE_HW_INTRINSICS + { + assert(src->OperIs(GT_LCL_VAR, GT_LCL_FLD, GT_FIELD, GT_IND, GT_OBJ, GT_CALL, GT_MKREFANY, GT_RET_EXPR, + GT_COMMA, GT_CNS_VEC) || + ((src->TypeGet() != TYP_STRUCT) && src->OperIsSIMD())); + } +#endif // DEBUG + + var_types asgType = src->TypeGet(); + + if (src->gtOper == GT_CALL) { GenTreeCall* srcCall = src->AsCall(); if (srcCall->TreatAsShouldHaveRetBufArg(this)) { - // Case of call returning a struct via hidden retbuf arg. + // Case of call returning a struct via hidden retbuf arg + CLANG_FORMAT_COMMENT_ANCHOR; + // Some calls have an "out buffer" that is not actually a ret buff // in the ABI sense. We take the path here for those but it should // not be marked as the ret buff arg since it always follow the @@ -958,8 +1226,7 @@ GenTree* Compiler::impAssignStruct(GenTree* dest, WellKnownArg wellKnownArgType = srcCall->ShouldHaveRetBufArg() ? WellKnownArg::RetBuffer : WellKnownArg::None; - GenTree* destAddr = gtNewOperNode(GT_ADDR, TYP_BYREF, dest); - NewCallArg newArg = NewCallArg::Primitive(destAddr).WellKnown(wellKnownArgType); + NewCallArg newArg = NewCallArg::Primitive(destAddr).WellKnown(wellKnownArgType); #if !defined(TARGET_ARM) // Unmanaged instance methods on Windows or Unix X86 need the retbuf arg after the first (this) parameter @@ -1013,7 +1280,7 @@ GenTree* Compiler::impAssignStruct(GenTree* dest, } else { - srcCall->gtArgs.InsertAfter(this, srcCall->gtArgs.Args().begin().GetArg(), newArg); + srcCall->gtArgs.InsertAfter(this, &*srcCall->gtArgs.Args().begin(), newArg); } #endif } @@ -1043,24 +1310,66 @@ GenTree* Compiler::impAssignStruct(GenTree* dest, // return the morphed call node return src; } - -#ifdef UNIX_AMD64_ABI - if (dest->OperIs(GT_LCL_VAR)) + else { - // TODO-Cleanup: delete this quirk. - lvaGetDesc(dest->AsLclVar())->lvIsMultiRegRet = true; + // Case of call returning a struct in one or more registers. + + var_types returnType = (var_types)srcCall->gtReturnType; + + // First we try to change this to "LclVar/LclFld = call" + // + if ((destAddr->gtOper == GT_ADDR) && (destAddr->AsOp()->gtOp1->gtOper == GT_LCL_VAR)) + { + // If it is a multi-reg struct return, don't change the oper to GT_LCL_FLD. + // That is, the IR will be of the form lclVar = call for multi-reg return + // + GenTreeLclVar* lcl = destAddr->AsOp()->gtOp1->AsLclVar(); + unsigned lclNum = lcl->GetLclNum(); + LclVarDsc* varDsc = lvaGetDesc(lclNum); + if (src->AsCall()->HasMultiRegRetVal()) + { + // Mark the struct LclVar as used in a MultiReg return context + // which currently makes it non promotable. + // TODO-1stClassStructs: Eliminate this pessimization when we can more generally + // handle multireg returns. + lcl->gtFlags |= GTF_DONT_CSE; + varDsc->lvIsMultiRegRet = true; + } + + dest = lcl; + +#if defined(TARGET_ARM) + // TODO-Cleanup: This should have been taken care of in the above HasMultiRegRetVal() case, + // but that method has not been updadted to include ARM. + impMarkLclDstNotPromotable(lclNum, src, structHnd); + lcl->gtFlags |= GTF_DONT_CSE; +#elif defined(UNIX_AMD64_ABI) + // Not allowed for FEATURE_CORCLR which is the only SKU available for System V OSs. + assert(!src->AsCall()->IsVarargs() && "varargs not allowed for System V OSs."); + + // Make the struct non promotable. The eightbytes could contain multiple fields. + // TODO-1stClassStructs: Eliminate this pessimization when we can more generally + // handle multireg returns. + // TODO-Cleanup: Why is this needed here? This seems that it will set this even for + // non-multireg returns. + lcl->gtFlags |= GTF_DONT_CSE; + varDsc->lvIsMultiRegRet = true; +#endif + } + else // we don't have a GT_ADDR of a GT_LCL_VAR + { + asgType = returnType; + } } -#endif // UNIX_AMD64_ABI } - else if (src->OperIs(GT_RET_EXPR)) + else if (src->gtOper == GT_RET_EXPR) { - assert(src->AsRetExpr()->gtInlineCandidate->OperIs(GT_CALL)); - GenTreeCall* call = src->AsRetExpr()->gtInlineCandidate; + noway_assert(src->AsRetExpr()->gtInlineCandidate->OperIs(GT_CALL)); + GenTreeCall* call = src->AsRetExpr()->gtInlineCandidate->AsCall(); if (call->ShouldHaveRetBufArg()) { // insert the return value buffer into the argument list as first byref parameter after 'this' - GenTree* destAddr = gtNewOperNode(GT_ADDR, TYP_BYREF, dest); call->gtArgs.InsertAfterThisOrFirst(this, NewCallArg::Primitive(destAddr).WellKnown(WellKnownArg::RetBuffer)); @@ -1072,15 +1381,26 @@ GenTree* Compiler::impAssignStruct(GenTree* dest, // So now we just return an empty node (pruning the GT_RET_EXPR) return src; } + else + { + // Case of inline method returning a struct in one or more registers. + // We won't need a return buffer + asgType = src->gtType; + } + } + else if (src->OperIsBlk()) + { + asgType = impNormStructType(structHnd); + assert(ClassLayout::AreCompatible(src->AsBlk()->GetLayout(), typGetObjLayout(structHnd))); } - else if (src->OperIs(GT_MKREFANY)) + else if (src->gtOper == GT_MKREFANY) { - // Since we are assigning the result of a GT_MKREFANY, "destAddr" must point to a refany. - // TODO-CQ: we can do this without address-exposing the local on the LHS. - GenTree* destAddr = gtNewOperNode(GT_ADDR, TYP_BYREF, dest); + // Since we are assigning the result of a GT_MKREFANY, + // "destAddr" must point to a refany. + GenTree* destAddrClone; - destAddr = impCloneExpr(destAddr, &destAddrClone, NO_CLASS_HANDLE, curLevel, - pAfterStmt DEBUGARG("MKREFANY assignment")); + destAddr = + impCloneExpr(destAddr, &destAddrClone, structHnd, curLevel, pAfterStmt DEBUGARG("MKREFANY assignment")); assert(OFFSETOF__CORINFO_TypedReference__dataPtr == 0); assert(destAddr->gtType == TYP_I_IMPL || destAddr->gtType == TYP_BYREF); @@ -1107,8 +1427,10 @@ GenTree* Compiler::impAssignStruct(GenTree* dest, // return the assign of the type value, to be appended return gtNewAssignNode(typeSlot, src->AsOp()->gtOp2); } - else if (src->OperIs(GT_COMMA)) + else if (src->gtOper == GT_COMMA) { + // The second thing is the struct or its address. + assert(varTypeIsStruct(src->AsOp()->gtOp2) || src->AsOp()->gtOp2->gtType == TYP_BYREF); if (pAfterStmt) { // Insert op1 after '*pAfterStmt' @@ -1126,56 +1448,98 @@ GenTree* Compiler::impAssignStruct(GenTree* dest, // In this case we have neither been given a statement to insert after, nor are we // in the importer where we can append the side effect. // Instead, we're going to sink the assignment below the COMMA. - src->AsOp()->gtOp2 = impAssignStruct(dest, src->AsOp()->gtOp2, curLevel, pAfterStmt, usedDI, block); + src->AsOp()->gtOp2 = + impAssignStructPtr(destAddr, src->AsOp()->gtOp2, structHnd, curLevel, pAfterStmt, usedDI, block); src->AddAllEffectsFlags(src->AsOp()->gtOp2); return src; } // Evaluate the second thing using recursion. - return impAssignStruct(dest, src->AsOp()->gtOp2, curLevel, pAfterStmt, usedDI, block); + return impAssignStructPtr(destAddr, src->AsOp()->gtOp2, structHnd, curLevel, pAfterStmt, usedDI, block); + } + else if (src->IsLocal()) + { + asgType = src->TypeGet(); + } + else if (asgType == TYP_STRUCT) + { + // It should already have the appropriate type. + assert(asgType == impNormStructType(structHnd)); + } + if ((dest == nullptr) && (destAddr->OperGet() == GT_ADDR)) + { + GenTree* destNode = destAddr->gtGetOp1(); + // If the actual destination is a local, or a block node, + // don't insert an OBJ(ADDR) if it already has the right type. + if (destNode->OperIs(GT_LCL_VAR) || destNode->OperIsBlk()) + { + var_types destType = destNode->TypeGet(); + // If one or both types are TYP_STRUCT (one may not yet be normalized), they are compatible + // iff their handles are the same. + // Otherwise, they are compatible if their types are the same. + bool typesAreCompatible = + ((destType == TYP_STRUCT) || (asgType == TYP_STRUCT)) + ? ((gtGetStructHandleIfPresent(destNode) == structHnd) && varTypeIsStruct(asgType)) + : (destType == asgType); + if (typesAreCompatible) + { + dest = destNode; + if (destType != TYP_STRUCT) + { + // Use a normalized type if available. We know from above that they're equivalent. + asgType = destType; + } + } + } + } + + if (dest == nullptr) + { + if (asgType == TYP_STRUCT) + { + dest = gtNewObjNode(structHnd, destAddr); + gtSetObjGcInfo(dest->AsObj()); + // Although an obj as a call argument was always assumed to be a globRef + // (which is itself overly conservative), that is not true of the operands + // of a block assignment. + dest->gtFlags &= ~GTF_GLOB_REF; + dest->gtFlags |= (destAddr->gtFlags & GTF_GLOB_REF); + } + else + { + dest = gtNewOperNode(GT_IND, asgType, destAddr); + } } - if (dest->OperIs(GT_LCL_VAR) && src->IsMultiRegNode()) + if (dest->OperIs(GT_LCL_VAR) && + (src->IsMultiRegNode() || + (src->OperIs(GT_RET_EXPR) && src->AsRetExpr()->gtInlineCandidate->AsCall()->HasMultiRegRetVal()))) { - lvaGetDesc(dest->AsLclVar())->lvIsMultiRegRet = true; + if (lvaEnregMultiRegVars && varTypeIsStruct(dest)) + { + dest->AsLclVar()->SetMultiReg(); + } + if (src->OperIs(GT_CALL)) + { + lvaGetDesc(dest->AsLclVar())->lvIsMultiRegRet = true; + } } - // Return a store node, to be appended. - GenTree* storeNode = gtNewBlkOpNode(dest, src); + dest->gtFlags |= destFlags; + destFlags = dest->gtFlags; - return storeNode; -} - -//------------------------------------------------------------------------ -// impAssignStructPtr: Assign (copy) the structure from 'src' to 'destAddr'. -// -// Arguments: -// destAddr - address of the destination of the assignment -// src - source of the assignment -// structHnd - handle representing the struct type -// curLevel - stack level for which a spill may be being done -// pAfterStmt - statement to insert any additional statements after -// di - debug info for new statements -// block - block to insert any additional statements in -// -// Return Value: -// The tree that should be appended to the statement list that represents the assignment. -// -// Notes: -// Temp assignments may be appended to impStmtList if spilling is necessary. -// -GenTree* Compiler::impAssignStructPtr(GenTree* destAddr, - GenTree* src, - CORINFO_CLASS_HANDLE structHnd, - unsigned curLevel, - Statement** pAfterStmt, /* = NULL */ - const DebugInfo& di, /* = DebugInfo() */ - BasicBlock* block /* = NULL */ - ) -{ - GenTree* dst = gtNewStructVal(typGetObjLayout(structHnd), destAddr); - return impAssignStruct(dst, src, curLevel, pAfterStmt, di, block); + // return an assignment node, to be appended + GenTree* asgNode = gtNewAssignNode(dest, src); + gtBlockOpInit(asgNode, dest, src, false); + + // TODO-1stClassStructs: Clean up the settings of GTF_DONT_CSE on the lhs + // of assignments. + if ((destFlags & GTF_DONT_CSE) == 0) + { + dest->gtFlags &= ~(GTF_DONT_CSE); + } + return asgNode; } /***************************************************************************** @@ -1192,20 +1556,30 @@ GenTree* Compiler::impGetStructAddr(GenTree* structVal, { assert(varTypeIsStruct(structVal) || eeIsValueClass(structHnd)); - var_types type = structVal->TypeGet(); + var_types type = structVal->TypeGet(); + genTreeOps oper = structVal->gtOper; - if (oper == GT_CALL || oper == GT_RET_EXPR || (oper == GT_OBJ && !willDeref) || oper == GT_MKREFANY || - structVal->OperIsSimdOrHWintrinsic() || structVal->IsCnsVec()) + if (oper == GT_OBJ && willDeref) + { + assert(structVal->AsObj()->GetLayout()->GetClassHandle() == structHnd); + return (structVal->AsObj()->Addr()); + } + else if (oper == GT_CALL || oper == GT_RET_EXPR || oper == GT_OBJ || oper == GT_MKREFANY || + structVal->OperIsSimdOrHWintrinsic() || structVal->IsCnsVec()) { unsigned tmpNum = lvaGrabTemp(true DEBUGARG("struct address for call/obj")); impAssignTempGen(tmpNum, structVal, structHnd, curLevel); - // The 'return value' is now address of the temp itself. - return gtNewLclVarAddrNode(tmpNum, TYP_BYREF); + // The 'return value' is now the temp itself + + type = genActualType(lvaTable[tmpNum].TypeGet()); + GenTree* temp = gtNewLclvNode(tmpNum, type); + temp = gtNewOperNode(GT_ADDR, TYP_BYREF, temp); + return temp; } - if (oper == GT_COMMA) + else if (oper == GT_COMMA) { assert(structVal->AsOp()->gtOp2->gtType == type); // Second thing is the struct @@ -1238,7 +1612,7 @@ GenTree* Compiler::impGetStructAddr(GenTree* structVal, return (structVal); } - return gtNewOperNode(GT_ADDR, TYP_BYREF, structVal); + return (gtNewOperNode(GT_ADDR, TYP_BYREF, structVal)); } //------------------------------------------------------------------------ @@ -1321,15 +1695,31 @@ GenTree* Compiler::impNormStructVal(GenTree* structVal, CORINFO_CLASS_HANDLE str { structType = impNormStructType(structHnd); } - GenTreeLclVarCommon* structLcl = nullptr; + bool alreadyNormalized = false; + GenTreeLclVarCommon* structLcl = nullptr; - switch (structVal->OperGet()) + genTreeOps oper = structVal->OperGet(); + switch (oper) { + // GT_MKREFANY is supported directly by args morphing. + case GT_MKREFANY: + alreadyNormalized = true; + break; + case GT_CALL: case GT_RET_EXPR: makeTemp = true; break; + case GT_FIELD: + // Wrap it in a GT_OBJ, if needed. + structVal->gtType = structType; + if (structType == TYP_STRUCT) + { + structVal = gtNewObjNode(structHnd, gtNewOperNode(GT_ADDR, TYP_BYREF, structVal)); + } + break; + case GT_LCL_VAR: case GT_LCL_FLD: structLcl = structVal->AsLclVarCommon(); @@ -1337,21 +1727,35 @@ GenTree* Compiler::impNormStructVal(GenTree* structVal, CORINFO_CLASS_HANDLE str structVal = gtNewObjNode(structHnd, gtNewOperNode(GT_ADDR, TYP_BYREF, structVal)); FALLTHROUGH; - case GT_IND: case GT_OBJ: case GT_BLK: - case GT_FIELD: + // These should already have the appropriate type. + assert(structVal->gtType == structType); + alreadyNormalized = true; + break; + + case GT_IND: + assert(structVal->gtType == structType); + structVal = gtNewObjNode(structHnd, structVal->gtGetOp1()); + alreadyNormalized = true; + break; + case GT_CNS_VEC: + assert(varTypeIsSIMD(structVal) && (structVal->gtType == structType)); + break; + #ifdef FEATURE_SIMD case GT_SIMD: -#endif + assert(varTypeIsSIMD(structVal) && (structVal->gtType == structType)); + break; +#endif // FEATURE_SIMD #ifdef FEATURE_HW_INTRINSICS case GT_HWINTRINSIC: -#endif - case GT_MKREFANY: - // These should already have the appropriate type. - assert(structVal->TypeGet() == structType); + assert(structVal->gtType == structType); + assert(varTypeIsSIMD(structVal) || + HWIntrinsicInfo::IsMultiReg(structVal->AsHWIntrinsic()->GetHWIntrinsicId())); break; +#endif case GT_COMMA: { @@ -1372,32 +1776,40 @@ GenTree* Compiler::impNormStructVal(GenTree* structVal, CORINFO_CLASS_HANDLE str } while (blockNode->OperGet() == GT_COMMA); } + if (blockNode->OperGet() == GT_FIELD) + { + // If we have a GT_FIELD then wrap it in a GT_OBJ. + blockNode = gtNewObjNode(structHnd, gtNewOperNode(GT_ADDR, TYP_BYREF, blockNode)); + } + #ifdef FEATURE_SIMD if (blockNode->OperIsSimdOrHWintrinsic() || blockNode->IsCnsVec()) { parent->AsOp()->gtOp2 = impNormStructVal(blockNode, structHnd, curLevel); + alreadyNormalized = true; } else #endif { - noway_assert(blockNode->OperIsBlk() || blockNode->OperIs(GT_FIELD)); + noway_assert(blockNode->OperIsBlk()); // Sink the GT_COMMA below the blockNode addr. - // That is GT_COMMA(op1, op2=blockNode) is transformed into + // That is GT_COMMA(op1, op2=blockNode) is tranformed into // blockNode(GT_COMMA(TYP_BYREF, op1, op2's op1)). // // In case of a chained GT_COMMA case, we sink the last // GT_COMMA below the blockNode addr. GenTree* blockNodeAddr = blockNode->AsOp()->gtOp1; - assert(blockNodeAddr->TypeIs(TYP_BYREF, TYP_I_IMPL)); + assert(blockNodeAddr->gtType == TYP_BYREF); GenTree* commaNode = parent; - commaNode->gtType = blockNodeAddr->gtType; + commaNode->gtType = TYP_BYREF; commaNode->AsOp()->gtOp2 = blockNodeAddr; blockNode->AsOp()->gtOp1 = commaNode; if (parent == structVal) { structVal = blockNode; } + alreadyNormalized = true; } } break; @@ -1406,19 +1818,22 @@ GenTree* Compiler::impNormStructVal(GenTree* structVal, CORINFO_CLASS_HANDLE str noway_assert(!"Unexpected node in impNormStructVal()"); break; } + structVal->gtType = structType; - if (makeTemp) + if (!alreadyNormalized) { - unsigned tmpNum = lvaGrabTemp(true DEBUGARG("struct address for call/obj")); - - impAssignTempGen(tmpNum, structVal, structHnd, curLevel); + if (makeTemp) + { + unsigned tmpNum = lvaGrabTemp(true DEBUGARG("struct address for call/obj")); - // The structVal is now the temp itself + impAssignTempGen(tmpNum, structVal, structHnd, curLevel); - structLcl = gtNewLclvNode(tmpNum, structType)->AsLclVarCommon(); - structVal = structLcl; + // The structVal is now the temp itself - if (structType == TYP_STRUCT) + structLcl = gtNewLclvNode(tmpNum, structType)->AsLclVarCommon(); + structVal = structLcl; + } + if ((structType == TYP_STRUCT) && !structVal->OperIsBlk()) { // Wrap it in a GT_OBJ structVal = gtNewObjNode(structHnd, gtNewOperNode(GT_ADDR, TYP_BYREF, structVal)); @@ -1790,7 +2205,7 @@ GenTree* Compiler::impRuntimeLookupToTree(CORINFO_RESOLVED_TOKEN* pResolvedToken if (pRuntimeLookup->testForNull) { - slotPtrTree = impCloneExpr(ctxTree, &ctxTree, NO_CLASS_HANDLE, CHECK_SPILL_ALL, + slotPtrTree = impCloneExpr(ctxTree, &ctxTree, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, nullptr DEBUGARG("impRuntimeLookup slot")); } @@ -1802,7 +2217,7 @@ GenTree* Compiler::impRuntimeLookupToTree(CORINFO_RESOLVED_TOKEN* pResolvedToken { if ((i == 1 && pRuntimeLookup->indirectFirstOffset) || (i == 2 && pRuntimeLookup->indirectSecondOffset)) { - indOffTree = impCloneExpr(slotPtrTree, &slotPtrTree, NO_CLASS_HANDLE, CHECK_SPILL_ALL, + indOffTree = impCloneExpr(slotPtrTree, &slotPtrTree, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, nullptr DEBUGARG("impRuntimeLookup indirectOffset")); } @@ -1829,7 +2244,7 @@ GenTree* Compiler::impRuntimeLookupToTree(CORINFO_RESOLVED_TOKEN* pResolvedToken { if (isLastIndirectionWithSizeCheck) { - lastIndOfTree = impCloneExpr(slotPtrTree, &slotPtrTree, NO_CLASS_HANDLE, CHECK_SPILL_ALL, + lastIndOfTree = impCloneExpr(slotPtrTree, &slotPtrTree, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, nullptr DEBUGARG("impRuntimeLookup indirectOffset")); } @@ -1857,7 +2272,7 @@ GenTree* Compiler::impRuntimeLookupToTree(CORINFO_RESOLVED_TOKEN* pResolvedToken impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("bubbling QMark0")); unsigned slotLclNum = lvaGrabTemp(true DEBUGARG("impRuntimeLookup test")); - impAssignTempGen(slotLclNum, slotPtrTree, NO_CLASS_HANDLE, CHECK_SPILL_ALL, nullptr, impCurStmtDI); + impAssignTempGen(slotLclNum, slotPtrTree, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, nullptr, impCurStmtDI); GenTree* slot = gtNewLclvNode(slotLclNum, TYP_I_IMPL); // downcast the pointer to a TYP_INT on 64-bit targets @@ -1877,7 +2292,7 @@ GenTree* Compiler::impRuntimeLookupToTree(CORINFO_RESOLVED_TOKEN* pResolvedToken GenTree* asg = gtNewAssignNode(slot, indir); GenTreeColon* colon = new (this, GT_COLON) GenTreeColon(TYP_VOID, gtNewNothingNode(), asg); GenTreeQmark* qmark = gtNewQmarkNode(TYP_VOID, relop, colon); - impAppendTree(qmark, CHECK_SPILL_NONE, impCurStmtDI); + impAppendTree(qmark, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); return gtNewLclvNode(slotLclNum, TYP_I_IMPL); } @@ -1938,10 +2353,18 @@ GenTree* Compiler::impRuntimeLookupToTree(CORINFO_RESOLVED_TOKEN* pResolvedToken unsigned tmp = lvaGrabTemp(true DEBUGARG("spilling Runtime Lookup tree")); - impAssignTempGen(tmp, result, CHECK_SPILL_NONE); + impAssignTempGen(tmp, result, (unsigned)CHECK_SPILL_NONE); return gtNewLclvNode(tmp, TYP_I_IMPL); } +/****************************************************************************** + * Spills the stack at verCurrentState.esStack[level] and replaces it with a temp. + * If tnum!=BAD_VAR_NUM, the temp var used to replace the tree is tnum, + * else, grab a new temp. + * For structs (which can be pushed on the stack using obj, etc), + * special handling is needed + */ + struct RecursiveGuard { public: @@ -2075,9 +2498,9 @@ void Compiler::impSpillStackEnsure(bool spillLeaves) * On return the stack is guaranteed to be empty. */ -void Compiler::impEvalSideEffects() +inline void Compiler::impEvalSideEffects() { - impSpillSideEffects(false, CHECK_SPILL_ALL DEBUGARG("impEvalSideEffects")); + impSpillSideEffects(false, (unsigned)CHECK_SPILL_ALL DEBUGARG("impEvalSideEffects")); verCurrentState.esStackDepth = 0; } @@ -2097,7 +2520,7 @@ void Compiler::impSpillSideEffect(bool spillGlobEffects, unsigned i DEBUGARG(con if ((tree->gtFlags & spillFlags) != 0 || (spillGlobEffects && // Only consider the following when spillGlobEffects == true - !impIsAddressInLocal(tree) && // No need to spill the LCL_ADDR nodes. + !impIsAddressInLocal(tree) && // No need to spill the GT_ADDR node on a local. gtHasLocalsWithAddrOp(tree))) // Spill if we still see GT_LCL_VAR that contains lvHasLdAddrOp or // lvAddrTaken flag. { @@ -2112,16 +2535,16 @@ void Compiler::impSpillSideEffect(bool spillGlobEffects, unsigned i DEBUGARG(con * [0..chkLevel) is the portion of the stack which will be checked and spilled. */ -void Compiler::impSpillSideEffects(bool spillGlobEffects, unsigned chkLevel DEBUGARG(const char* reason)) +inline void Compiler::impSpillSideEffects(bool spillGlobEffects, unsigned chkLevel DEBUGARG(const char* reason)) { - assert(chkLevel != CHECK_SPILL_NONE); + assert(chkLevel != (unsigned)CHECK_SPILL_NONE); /* Before we make any appends to the tree list we must spill the * "special" side effects (GTF_ORDER_SIDEEFF on a GT_CATCH_ARG) */ impSpillSpecialSideEff(); - if (chkLevel == CHECK_SPILL_ALL) + if (chkLevel == (unsigned)CHECK_SPILL_ALL) { chkLevel = verCurrentState.esStackDepth; } @@ -2140,7 +2563,7 @@ void Compiler::impSpillSideEffects(bool spillGlobEffects, unsigned chkLevel DEBU * those trees to temps and replace them on the stack with refs to their temps. */ -void Compiler::impSpillSpecialSideEff() +inline void Compiler::impSpillSpecialSideEff() { // Only exception objects need to be carefully handled @@ -2160,27 +2583,21 @@ void Compiler::impSpillSpecialSideEff() } } -//------------------------------------------------------------------------ -// impSpillLclRefs: Spill all trees referencing the given local. -// -// Arguments: -// lclNum - The local's number -// chkLevel - Height (exclusive) of the portion of the stack to check -// -void Compiler::impSpillLclRefs(unsigned lclNum, unsigned chkLevel) -{ - // Before we make any appends to the tree list we must spill the - // "special" side effects (GTF_ORDER_SIDEEFF) - GT_CATCH_ARG. - impSpillSpecialSideEff(); +/***************************************************************************** + * + * If the stack contains any trees with references to local #lclNum, assign + * those trees to temps and replace their place on the stack with refs to + * their temps. + */ - if (chkLevel == CHECK_SPILL_ALL) - { - chkLevel = verCurrentState.esStackDepth; - } +void Compiler::impSpillLclRefs(unsigned lclNum) +{ + /* Before we make any appends to the tree list we must spill the + * "special" side effects (GTF_ORDER_SIDEEFF) - GT_CATCH_ARG */ - assert(chkLevel <= verCurrentState.esStackDepth); + impSpillSpecialSideEff(); - for (unsigned level = 0; level < chkLevel; level++) + for (unsigned level = 0; level < verCurrentState.esStackDepth; level++) { GenTree* tree = verCurrentState.esStack[level].val; @@ -2251,7 +2668,7 @@ BasicBlock* Compiler::impPushCatchArgOnStack(BasicBlock* hndBlk, CORINFO_CLASS_H #if defined(JIT32_GCENCODER) const bool forceInsertNewBlock = isSingleBlockFilter || compStressCompile(STRESS_CATCH_ARG, 5); #else - const bool forceInsertNewBlock = compStressCompile(STRESS_CATCH_ARG, 5); + const bool forceInsertNewBlock = compStressCompile(STRESS_CATCH_ARG, 5); #endif // defined(JIT32_GCENCODER) /* Spill GT_CATCH_ARG to a temp if there are jumps to the beginning of the handler */ @@ -2379,7 +2796,7 @@ DebugInfo Compiler::impCreateDIWithCurrentStackInfo(IL_OFFSET offs, bool isCall) // statements to report debug info for to the EE: for other statements, they // will have no debug information attached. // -void Compiler::impCurStmtOffsSet(IL_OFFSET offs) +inline void Compiler::impCurStmtOffsSet(IL_OFFSET offs) { if (offs == BAD_IL_OFFSET) { @@ -2452,7 +2869,7 @@ void Compiler::impNoteBranchOffs() { if (opts.compDbgCode) { - impAppendTree(gtNewNothingNode(), CHECK_SPILL_NONE, impCurStmtDI); + impAppendTree(gtNewNothingNode(), (unsigned)CHECK_SPILL_NONE, impCurStmtDI); } } @@ -2557,7 +2974,7 @@ bool Compiler::impOpcodeIsCallOpcode(OPCODE opcode) /*****************************************************************************/ -static bool impOpcodeIsCallSiteBoundary(OPCODE opcode) +static inline bool impOpcodeIsCallSiteBoundary(OPCODE opcode) { switch (opcode) { @@ -2691,7249 +3108,14108 @@ GenTree* Compiler::impImplicitR4orR8Cast(GenTree* tree, var_types dstTyp) return tree; } -GenTree* Compiler::impTypeIsAssignable(GenTree* typeTo, GenTree* typeFrom) +//------------------------------------------------------------------------ +// impInitializeArrayIntrinsic: Attempts to replace a call to InitializeArray +// with a GT_COPYBLK node. +// +// Arguments: +// sig - The InitializeArray signature. +// +// Return Value: +// A pointer to the newly created GT_COPYBLK node if the replacement succeeds or +// nullptr otherwise. +// +// Notes: +// The function recognizes the following IL pattern: +// ldc or a list of ldc / +// newarr or newobj +// dup +// ldtoken +// call InitializeArray +// The lower bounds need not be constant except when the array rank is 1. +// The function recognizes all kinds of arrays thus enabling a small runtime +// such as NativeAOT to skip providing an implementation for InitializeArray. + +GenTree* Compiler::impInitializeArrayIntrinsic(CORINFO_SIG_INFO* sig) { - // Optimize patterns like: + assert(sig->numArgs == 2); + + GenTree* fieldTokenNode = impStackTop(0).val; + GenTree* arrayLocalNode = impStackTop(1).val; + // - // typeof(TTo).IsAssignableFrom(typeof(TTFrom)) - // valueTypeVar.GetType().IsAssignableFrom(typeof(TTFrom)) - // typeof(TTFrom).IsAssignableTo(typeof(TTo)) - // typeof(TTFrom).IsAssignableTo(valueTypeVar.GetType()) + // Verify that the field token is known and valid. Note that it's also + // possible for the token to come from reflection, in which case we cannot do + // the optimization and must therefore revert to calling the helper. You can + // see an example of this in bvt\DynIL\initarray2.exe (in Main). // - // to true/false - // make sure both arguments are `typeof()` - CORINFO_CLASS_HANDLE hClassTo = NO_CLASS_HANDLE; - CORINFO_CLASS_HANDLE hClassFrom = NO_CLASS_HANDLE; - if (gtIsTypeof(typeTo, &hClassTo) && gtIsTypeof(typeFrom, &hClassFrom)) + // Check to see if the ldtoken helper call is what we see here. + if (fieldTokenNode->gtOper != GT_CALL || (fieldTokenNode->AsCall()->gtCallType != CT_HELPER) || + (fieldTokenNode->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_FIELDDESC_TO_STUBRUNTIMEFIELD))) { - TypeCompareState castResult = info.compCompHnd->compareTypesForCast(hClassFrom, hClassTo); - if (castResult == TypeCompareState::May) - { - // requires runtime check - // e.g. __Canon, COMObjects, Nullable - return nullptr; - } - - GenTreeIntCon* retNode = gtNewIconNode((castResult == TypeCompareState::Must) ? 1 : 0); - impPopStack(); // drop both CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE calls - impPopStack(); - - return retNode; + return nullptr; } - return nullptr; -} - -/***************************************************************************** - * 'logMsg' is true if a log message needs to be logged. false if the caller has - * already logged it (presumably in a more detailed fashion than done here) - */ - -void Compiler::verConvertBBToThrowVerificationException(BasicBlock* block DEBUGARG(bool logMsg)) -{ - block->bbJumpKind = BBJ_THROW; - block->bbFlags |= BBF_FAILED_VERIFICATION; - block->bbFlags &= ~BBF_IMPORTED; - - impCurStmtOffsSet(block->bbCodeOffs); - - // Clear the statement list as it exists so far; we're only going to have a verification exception. - impStmtList = impLastStmt = nullptr; + // Strip helper call away + fieldTokenNode = fieldTokenNode->AsCall()->gtArgs.GetArgByIndex(0)->GetEarlyNode(); -#ifdef DEBUG - if (logMsg) + if (fieldTokenNode->gtOper == GT_IND) { - JITLOG((LL_ERROR, "Verification failure: while compiling %s near IL offset %x..%xh \n", info.compFullName, - block->bbCodeOffs, block->bbCodeOffsEnd)); - if (verbose) - { - printf("\n\nVerification failure: %s near IL %xh \n", info.compFullName, block->bbCodeOffs); - } + fieldTokenNode = fieldTokenNode->AsOp()->gtOp1; } - if (JitConfig.DebugBreakOnVerificationFailure()) + // Check for constant + if (fieldTokenNode->gtOper != GT_CNS_INT) { - DebugBreak(); + return nullptr; } -#endif - impBeginTreeList(); - - // if the stack is non-empty evaluate all the side-effects - if (verCurrentState.esStackDepth > 0) + CORINFO_FIELD_HANDLE fieldToken = (CORINFO_FIELD_HANDLE)fieldTokenNode->AsIntCon()->gtCompileTimeHandle; + if (!fieldTokenNode->IsIconHandle(GTF_ICON_FIELD_HDL) || (fieldToken == nullptr)) { - impEvalSideEffects(); + return nullptr; } - assert(verCurrentState.esStackDepth == 0); - - GenTree* op1 = gtNewHelperCallNode(CORINFO_HELP_VERIFICATION, TYP_VOID, gtNewIconNode(block->bbCodeOffs)); - // verCurrentState.esStackDepth = 0; - impAppendTree(op1, CHECK_SPILL_NONE, impCurStmtDI); - - // The inliner is not able to handle methods that require throw block, so - // make sure this methods never gets inlined. - info.compCompHnd->setMethodAttribs(info.compMethodHnd, CORINFO_FLG_BAD_INLINEE); -} - -/***************************************************************************** - * - */ -void Compiler::verHandleVerificationFailure(BasicBlock* block DEBUGARG(bool logMsg)) -{ - verResetCurrentState(block, &verCurrentState); - verConvertBBToThrowVerificationException(block DEBUGARG(logMsg)); - -#ifdef DEBUG - impNoteLastILoffs(); // Remember at which BC offset the tree was finished -#endif // DEBUG -} -typeInfo Compiler::verMakeTypeInfoForLocal(unsigned lclNum) -{ - LclVarDsc* varDsc = lvaGetDesc(lclNum); + // + // We need to get the number of elements in the array and the size of each element. + // We verify that the newarr statement is exactly what we expect it to be. + // If it's not then we just return NULL and we don't optimize this call + // - if ((varDsc->TypeGet() == TYP_BLK) || (varDsc->TypeGet() == TYP_LCLBLK)) - { - return typeInfo(); - } - if (varDsc->TypeGet() == TYP_BYREF) + // It is possible the we don't have any statements in the block yet. + if (impLastStmt == nullptr) { - // Pretend all byrefs are pointing to bytes. - return typeInfo(TI_BYTE).MakeByRef(); + return nullptr; } - if (varTypeIsStruct(varDsc)) + + // + // We start by looking at the last statement, making sure it's an assignment, and + // that the target of the assignment is the array passed to InitializeArray. + // + GenTree* arrayAssignment = impLastStmt->GetRootNode(); + if ((arrayAssignment->gtOper != GT_ASG) || (arrayAssignment->AsOp()->gtOp1->gtOper != GT_LCL_VAR) || + (arrayLocalNode->gtOper != GT_LCL_VAR) || (arrayAssignment->AsOp()->gtOp1->AsLclVarCommon()->GetLclNum() != + arrayLocalNode->AsLclVarCommon()->GetLclNum())) { - return typeInfo(TI_STRUCT, varDsc->GetStructHnd()); + return nullptr; } - return typeInfo(varDsc->TypeGet()); -} - -typeInfo Compiler::verMakeTypeInfo(CorInfoType ciType, CORINFO_CLASS_HANDLE clsHnd) -{ - assert(ciType < CORINFO_TYPE_COUNT); + // + // Make sure that the object being assigned is a helper call. + // - typeInfo tiResult; - switch (ciType) + GenTree* newArrayCall = arrayAssignment->AsOp()->gtOp2; + if ((newArrayCall->gtOper != GT_CALL) || (newArrayCall->AsCall()->gtCallType != CT_HELPER)) { - case CORINFO_TYPE_STRING: - case CORINFO_TYPE_CLASS: - tiResult = verMakeTypeInfo(clsHnd); - if (!tiResult.IsType(TI_REF)) - { // type must be consistent with element type - return typeInfo(); - } - break; + return nullptr; + } - case CORINFO_TYPE_VALUECLASS: - case CORINFO_TYPE_REFANY: - tiResult = verMakeTypeInfo(clsHnd); - // type must be constant with element type; - if (!tiResult.IsValueClass()) - { - return typeInfo(); - } - break; - case CORINFO_TYPE_VAR: - return verMakeTypeInfo(clsHnd); + // + // Verify that it is one of the new array helpers. + // - case CORINFO_TYPE_PTR: // for now, pointers are treated as an error - case CORINFO_TYPE_VOID: - return typeInfo(); - break; + bool isMDArray = false; - case CORINFO_TYPE_BYREF: + if (newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEWARR_1_DIRECT) && + newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEWARR_1_OBJ) && + newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEWARR_1_VC) && + newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEWARR_1_ALIGN8) +#ifdef FEATURE_READYTORUN + && newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_READYTORUN_NEWARR_1) +#endif + ) + { + if (newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEW_MDARR)) { - CORINFO_CLASS_HANDLE childClassHandle; - CorInfoType childType = info.compCompHnd->getChildType(clsHnd, &childClassHandle); - return ByRef(verMakeTypeInfo(childType, childClassHandle)); + return nullptr; } - break; - default: - if (clsHnd) - { // If we have more precise information, use it - return typeInfo(TI_STRUCT, clsHnd); - } - else - { - return typeInfo(JITtype2tiType(ciType)); - } + isMDArray = true; } - return tiResult; -} -/******************************************************************************/ - -typeInfo Compiler::verMakeTypeInfo(CORINFO_CLASS_HANDLE clsHnd) -{ - if (clsHnd == NO_CLASS_HANDLE) - { - return typeInfo(); - } + CORINFO_CLASS_HANDLE arrayClsHnd = (CORINFO_CLASS_HANDLE)newArrayCall->AsCall()->compileTimeHelperArgumentHandle; - // Byrefs should only occur in method and local signatures, which are accessed - // using ICorClassInfo and ICorClassInfo.getChildType. - // So findClass() and getClassAttribs() should not be called for byrefs + // + // Make sure we found a compile time handle to the array + // - if (JITtype2varType(info.compCompHnd->asCorInfoType(clsHnd)) == TYP_BYREF) + if (!arrayClsHnd) { - assert(!"Did findClass() return a Byref?"); - return typeInfo(); + return nullptr; } - unsigned attribs = info.compCompHnd->getClassAttribs(clsHnd); + unsigned rank = 0; + S_UINT32 numElements; - if (attribs & CORINFO_FLG_VALUECLASS) + if (isMDArray) { - CorInfoType t = info.compCompHnd->getTypeForPrimitiveValueClass(clsHnd); + rank = info.compCompHnd->getArrayRank(arrayClsHnd); - // Meta-data validation should ensure that CORINF_TYPE_BYREF should - // not occur here, so we may want to change this to an assert instead. - if (t == CORINFO_TYPE_VOID || t == CORINFO_TYPE_BYREF || t == CORINFO_TYPE_PTR) + if (rank == 0) { - return typeInfo(); + return nullptr; } - if (t != CORINFO_TYPE_UNDEF) + assert(newArrayCall->AsCall()->gtArgs.CountArgs() == 3); + GenTree* numArgsArg = newArrayCall->AsCall()->gtArgs.GetArgByIndex(1)->GetNode(); + GenTree* argsArg = newArrayCall->AsCall()->gtArgs.GetArgByIndex(2)->GetNode(); + + // + // The number of arguments should be a constant between 1 and 64. The rank can't be 0 + // so at least one length must be present and the rank can't exceed 32 so there can + // be at most 64 arguments - 32 lengths and 32 lower bounds. + // + + if ((!numArgsArg->IsCnsIntOrI()) || (numArgsArg->AsIntCon()->IconValue() < 1) || + (numArgsArg->AsIntCon()->IconValue() > 64)) { - return (typeInfo(JITtype2tiType(t))); + return nullptr; } - else + + unsigned numArgs = static_cast(numArgsArg->AsIntCon()->IconValue()); + bool lowerBoundsSpecified; + + if (numArgs == rank * 2) { - return (typeInfo(TI_STRUCT, clsHnd)); + lowerBoundsSpecified = true; } - } - else - { - return (typeInfo(TI_REF, clsHnd)); - } -} + else if (numArgs == rank) + { + lowerBoundsSpecified = false; -/***************************************************************************** - */ -typeInfo Compiler::verParseArgSigToTypeInfo(CORINFO_SIG_INFO* sig, CORINFO_ARG_LIST_HANDLE args) -{ - CORINFO_CLASS_HANDLE classHandle; - CorInfoType ciType = strip(info.compCompHnd->getArgType(sig, args, &classHandle)); + // + // If the rank is 1 and a lower bound isn't specified then the runtime creates + // a SDArray. Note that even if a lower bound is specified it can be 0 and then + // we get a SDArray as well, see the for loop below. + // - var_types type = JITtype2varType(ciType); - if (varTypeIsGC(type)) - { - // For efficiency, getArgType only returns something in classHandle for - // value types. For other types that have addition type info, you - // have to call back explicitly - classHandle = info.compCompHnd->getArgClass(sig, args); - if (!classHandle) + if (rank == 1) + { + isMDArray = false; + } + } + else { - NO_WAY("Could not figure out Class specified in argument or local signature"); + return nullptr; } - } - return verMakeTypeInfo(ciType, classHandle); -} + // + // The rank is known to be at least 1 so we can start with numElements being 1 + // to avoid the need to special case the first dimension. + // -bool Compiler::verIsByRefLike(const typeInfo& ti) -{ - if (ti.IsByRef()) - { - return true; - } - if (!ti.IsType(TI_STRUCT)) - { - return false; - } - return info.compCompHnd->getClassAttribs(ti.GetClassHandleForValueClass()) & CORINFO_FLG_BYREF_LIKE; -} + numElements = S_UINT32(1); -/***************************************************************************** - * - * Check if a TailCall is legal. - */ + struct Match + { + static bool IsArgsFieldInit(GenTree* tree, unsigned index, unsigned lvaNewObjArrayArgs) + { + return (tree->OperGet() == GT_ASG) && IsArgsFieldIndir(tree->gtGetOp1(), index, lvaNewObjArrayArgs) && + IsArgsAddr(tree->gtGetOp1()->gtGetOp1()->gtGetOp1(), lvaNewObjArrayArgs); + } -bool Compiler::verCheckTailCallConstraint(OPCODE opcode, - CORINFO_RESOLVED_TOKEN* pResolvedToken, - CORINFO_RESOLVED_TOKEN* pConstrainedResolvedToken) -{ - DWORD mflags; - CORINFO_SIG_INFO sig; - unsigned int popCount = 0; // we can't pop the stack since impImportCall needs it, so - // this counter is used to keep track of how many items have been - // virtually popped + static bool IsArgsFieldIndir(GenTree* tree, unsigned index, unsigned lvaNewObjArrayArgs) + { + return (tree->OperGet() == GT_IND) && (tree->gtGetOp1()->OperGet() == GT_ADD) && + (tree->gtGetOp1()->gtGetOp2()->IsIntegralConst(sizeof(INT32) * index)) && + IsArgsAddr(tree->gtGetOp1()->gtGetOp1(), lvaNewObjArrayArgs); + } - CORINFO_METHOD_HANDLE methodHnd = nullptr; - CORINFO_CLASS_HANDLE methodClassHnd = nullptr; - unsigned methodClassFlgs = 0; + static bool IsArgsAddr(GenTree* tree, unsigned lvaNewObjArrayArgs) + { + return (tree->OperGet() == GT_ADDR) && (tree->gtGetOp1()->OperGet() == GT_LCL_VAR) && + (tree->gtGetOp1()->AsLclVar()->GetLclNum() == lvaNewObjArrayArgs); + } - assert(impOpcodeIsCallOpcode(opcode)); + static bool IsComma(GenTree* tree) + { + return (tree != nullptr) && (tree->OperGet() == GT_COMMA); + } + }; - if (compIsForInlining()) - { - return false; - } + unsigned argIndex = 0; + GenTree* comma; - // For calli, check that this is not a virtual method. - if (opcode == CEE_CALLI) - { - /* Get the call sig */ - eeGetSig(pResolvedToken->token, pResolvedToken->tokenScope, pResolvedToken->tokenContext, &sig); + for (comma = argsArg; Match::IsComma(comma); comma = comma->gtGetOp2()) + { + if (lowerBoundsSpecified) + { + // + // In general lower bounds can be ignored because they're not needed to + // calculate the total number of elements. But for single dimensional arrays + // we need to know if the lower bound is 0 because in this case the runtime + // creates a SDArray and this affects the way the array data offset is calculated. + // - // We don't know the target method, so we have to infer the flags, or - // assume the worst-case. - mflags = (sig.callConv & CORINFO_CALLCONV_HASTHIS) ? 0 : CORINFO_FLG_STATIC; - } - else - { - methodHnd = pResolvedToken->hMethod; + if (rank == 1) + { + GenTree* lowerBoundAssign = comma->gtGetOp1(); + assert(Match::IsArgsFieldInit(lowerBoundAssign, argIndex, lvaNewObjArrayArgs)); + GenTree* lowerBoundNode = lowerBoundAssign->gtGetOp2(); - mflags = info.compCompHnd->getMethodAttribs(methodHnd); + if (lowerBoundNode->IsIntegralConst(0)) + { + isMDArray = false; + } + } - // In generic code we pair the method handle with its owning class to get the exact method signature. - methodClassHnd = pResolvedToken->hClass; - assert(methodClassHnd != NO_CLASS_HANDLE); + comma = comma->gtGetOp2(); + argIndex++; + } - eeGetMethodSig(methodHnd, &sig, methodClassHnd); + GenTree* lengthNodeAssign = comma->gtGetOp1(); + assert(Match::IsArgsFieldInit(lengthNodeAssign, argIndex, lvaNewObjArrayArgs)); + GenTree* lengthNode = lengthNodeAssign->gtGetOp2(); - // opcode specific check - methodClassFlgs = info.compCompHnd->getClassAttribs(methodClassHnd); - } + if (!lengthNode->IsCnsIntOrI()) + { + return nullptr; + } - // We must have got the methodClassHnd if opcode is not CEE_CALLI - assert((methodHnd != nullptr && methodClassHnd != NO_CLASS_HANDLE) || opcode == CEE_CALLI); + numElements *= S_SIZE_T(lengthNode->AsIntCon()->IconValue()); + argIndex++; + } - if ((sig.callConv & CORINFO_CALLCONV_MASK) == CORINFO_CALLCONV_VARARG) - { - eeGetCallSiteSig(pResolvedToken->token, pResolvedToken->tokenScope, pResolvedToken->tokenContext, &sig); - } + assert((comma != nullptr) && Match::IsArgsAddr(comma, lvaNewObjArrayArgs)); - // Check compatibility of the arguments. - unsigned int argCount = sig.numArgs; - CORINFO_ARG_LIST_HANDLE args; - args = sig.args; - while (argCount--) + if (argIndex != numArgs) + { + return nullptr; + } + } + else { - typeInfo tiDeclared = verParseArgSigToTypeInfo(&sig, args).NormaliseForStack(); + // + // Make sure there are exactly two arguments: the array class and + // the number of elements. + // + + GenTree* arrayLengthNode; - // Check that the argument is not a byref for tailcalls. - if (verIsByRefLike(tiDeclared)) +#ifdef FEATURE_READYTORUN + if (newArrayCall->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_READYTORUN_NEWARR_1)) { - return false; + // Array length is 1st argument for readytorun helper + arrayLengthNode = newArrayCall->AsCall()->gtArgs.GetArgByIndex(0)->GetNode(); + } + else +#endif + { + // Array length is 2nd argument for regular helper + arrayLengthNode = newArrayCall->AsCall()->gtArgs.GetArgByIndex(1)->GetNode(); } - // For unsafe code, we might have parameters containing pointer to the stack location. - // Disallow the tailcall for this kind. - CORINFO_CLASS_HANDLE classHandle; - CorInfoType ciType = strip(info.compCompHnd->getArgType(&sig, args, &classHandle)); - if (ciType == CORINFO_TYPE_PTR) + // + // This optimization is only valid for a constant array size. + // + if (arrayLengthNode->gtOper != GT_CNS_INT) { - return false; + return nullptr; } - args = info.compCompHnd->getArgNext(args); - } + numElements = S_SIZE_T(arrayLengthNode->AsIntCon()->gtIconVal); - // Update popCount. - popCount += sig.numArgs; + if (!info.compCompHnd->isSDArray(arrayClsHnd)) + { + return nullptr; + } + } - // Check for 'this' which is on non-static methods, not called via NEWOBJ - if (!(mflags & CORINFO_FLG_STATIC)) - { - // Always update the popCount. This is crucial for the stack calculation to be correct. - typeInfo tiThis = impStackTop(popCount).seTypeInfo; - popCount++; + CORINFO_CLASS_HANDLE elemClsHnd; + var_types elementType = JITtype2varType(info.compCompHnd->getChildType(arrayClsHnd, &elemClsHnd)); - if (opcode == CEE_CALLI) - { - // For CALLI, we don't know the methodClassHnd. Therefore, let's check the "this" object - // on the stack. - if (tiThis.IsValueClass()) - { - tiThis.MakeByRef(); - } + // + // Note that genTypeSize will return zero for non primitive types, which is exactly + // what we want (size will then be 0, and we will catch this in the conditional below). + // Note that we don't expect this to fail for valid binaries, so we assert in the + // non-verification case (the verification case should not assert but rather correctly + // handle bad binaries). This assert is not guarding any specific invariant, but rather + // saying that we don't expect this to happen, and if it is hit, we need to investigate + // why. + // - if (verIsByRefLike(tiThis)) - { - return false; - } - } - else - { - // Check type compatibility of the this argument - typeInfo tiDeclaredThis = verMakeTypeInfo(methodClassHnd); - if (tiDeclaredThis.IsValueClass()) - { - tiDeclaredThis.MakeByRef(); - } + S_UINT32 elemSize(genTypeSize(elementType)); + S_UINT32 size = elemSize * S_UINT32(numElements); - if (verIsByRefLike(tiDeclaredThis)) - { - return false; - } - } + if (size.IsOverflow()) + { + return nullptr; } - // Tail calls on constrained calls should be illegal too: - // when instantiated at a value type, a constrained call may pass the address of a stack allocated value - if (pConstrainedResolvedToken != nullptr) + if ((size.Value() == 0) || (varTypeIsGC(elementType))) { - return false; + return nullptr; } - // Get the exact view of the signature for an array method - if (sig.retType != CORINFO_TYPE_VOID) + void* initData = info.compCompHnd->getArrayInitializationData(fieldToken, size.Value()); + if (!initData) { - if (methodClassFlgs & CORINFO_FLG_ARRAY) - { - assert(opcode != CEE_CALLI); - eeGetCallSiteSig(pResolvedToken->token, pResolvedToken->tokenScope, pResolvedToken->tokenContext, &sig); - } + return nullptr; } - var_types calleeRetType = genActualType(JITtype2varType(sig.retType)); - var_types callerRetType = genActualType(JITtype2varType(info.compMethodInfo->args.retType)); + // + // At this point we are ready to commit to implementing the InitializeArray + // intrinsic using a struct assignment. Pop the arguments from the stack and + // return the struct assignment node. + // + + impPopStack(); + impPopStack(); - // Normalize TYP_FLOAT to TYP_DOUBLE (it is ok to return one as the other and vice versa). - calleeRetType = (calleeRetType == TYP_FLOAT) ? TYP_DOUBLE : calleeRetType; - callerRetType = (callerRetType == TYP_FLOAT) ? TYP_DOUBLE : callerRetType; + const unsigned blkSize = size.Value(); + unsigned dataOffset; - // Make sure the types match. - if (calleeRetType != callerRetType) + if (isMDArray) { - return false; + dataOffset = eeGetMDArrayDataOffset(rank); } - else if ((callerRetType == TYP_STRUCT) && (sig.retTypeClass != info.compMethodInfo->args.retTypeClass)) + else { - return false; + dataOffset = eeGetArrayDataOffset(); } - // For tailcall, stack must be empty. - if (verCurrentState.esStackDepth != popCount) - { - return false; - } + GenTree* dstAddr = gtNewOperNode(GT_ADD, TYP_BYREF, arrayLocalNode, gtNewIconNode(dataOffset, TYP_I_IMPL)); + GenTree* dst = new (this, GT_BLK) GenTreeBlk(GT_BLK, TYP_STRUCT, dstAddr, typGetBlkLayout(blkSize)); + GenTree* src = gtNewIndOfIconHandleNode(TYP_STRUCT, (size_t)initData, GTF_ICON_CONST_PTR, true); - return true; // Yes, tailcall is legal +#ifdef DEBUG + src->gtGetOp1()->AsIntCon()->gtTargetHandle = THT_InitializeArrayIntrinsics; +#endif + + return gtNewBlkOpNode(dst, // dst + src, // src + false, // volatile + true); // copyBlock } -GenTree* Compiler::impImportLdvirtftn(GenTree* thisPtr, - CORINFO_RESOLVED_TOKEN* pResolvedToken, - CORINFO_CALL_INFO* pCallInfo) +GenTree* Compiler::impCreateSpanIntrinsic(CORINFO_SIG_INFO* sig) { - if ((pCallInfo->methodFlags & CORINFO_FLG_EnC) && !(pCallInfo->classFlags & CORINFO_FLG_INTERFACE)) + assert(sig->numArgs == 1); + assert(sig->sigInst.methInstCount == 1); + + GenTree* fieldTokenNode = impStackTop(0).val; + + // + // Verify that the field token is known and valid. Note that it's also + // possible for the token to come from reflection, in which case we cannot do + // the optimization and must therefore revert to calling the helper. You can + // see an example of this in bvt\DynIL\initarray2.exe (in Main). + // + + // Check to see if the ldtoken helper call is what we see here. + if (fieldTokenNode->gtOper != GT_CALL || (fieldTokenNode->AsCall()->gtCallType != CT_HELPER) || + (fieldTokenNode->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_FIELDDESC_TO_STUBRUNTIMEFIELD))) { - NO_WAY("Virtual call to a function added via EnC is not supported"); + return nullptr; } - // NativeAOT generic virtual method - if ((pCallInfo->sig.sigInst.methInstCount != 0) && IsTargetAbi(CORINFO_NATIVEAOT_ABI)) + // Strip helper call away + fieldTokenNode = fieldTokenNode->AsCall()->gtArgs.GetArgByIndex(0)->GetNode(); + if (fieldTokenNode->gtOper == GT_IND) { - GenTree* runtimeMethodHandle = - impLookupToTree(pResolvedToken, &pCallInfo->codePointerLookup, GTF_ICON_METHOD_HDL, pCallInfo->hMethod); - return gtNewHelperCallNode(CORINFO_HELP_GVMLOOKUP_FOR_SLOT, TYP_I_IMPL, thisPtr, runtimeMethodHandle); + fieldTokenNode = fieldTokenNode->AsOp()->gtOp1; } -#ifdef FEATURE_READYTORUN - if (opts.IsReadyToRun()) + // Check for constant + if (fieldTokenNode->gtOper != GT_CNS_INT) { - if (!pCallInfo->exactContextNeedsRuntimeLookup) - { - GenTreeCall* call = gtNewHelperCallNode(CORINFO_HELP_READYTORUN_VIRTUAL_FUNC_PTR, TYP_I_IMPL, thisPtr); + return nullptr; + } - call->setEntryPoint(pCallInfo->codePointerLookup.constLookup); + CORINFO_FIELD_HANDLE fieldToken = (CORINFO_FIELD_HANDLE)fieldTokenNode->AsIntCon()->gtCompileTimeHandle; + if (!fieldTokenNode->IsIconHandle(GTF_ICON_FIELD_HDL) || (fieldToken == nullptr)) + { + return nullptr; + } - return call; - } + CORINFO_CLASS_HANDLE fieldOwnerHnd = info.compCompHnd->getFieldClass(fieldToken); - // We need a runtime lookup. NativeAOT has a ReadyToRun helper for that too. - if (IsTargetAbi(CORINFO_NATIVEAOT_ABI)) - { - GenTree* ctxTree = getRuntimeContextTree(pCallInfo->codePointerLookup.lookupKind.runtimeLookupKind); + CORINFO_CLASS_HANDLE fieldClsHnd; + var_types fieldElementType = + JITtype2varType(info.compCompHnd->getFieldType(fieldToken, &fieldClsHnd, fieldOwnerHnd)); + unsigned totalFieldSize; - return impReadyToRunHelperToTree(pResolvedToken, CORINFO_HELP_READYTORUN_GENERIC_HANDLE, TYP_I_IMPL, - &pCallInfo->codePointerLookup.lookupKind, ctxTree); - } + // Most static initialization data fields are of some structure, but it is possible for them to be of various + // primitive types as well + if (fieldElementType == var_types::TYP_STRUCT) + { + totalFieldSize = info.compCompHnd->getClassSize(fieldClsHnd); + } + else + { + totalFieldSize = genTypeSize(fieldElementType); } -#endif - // Get the exact descriptor for the static callsite - GenTree* exactTypeDesc = impParentClassTokenToHandle(pResolvedToken); - if (exactTypeDesc == nullptr) - { // compDonotInline() + // Limit to primitive or enum type - see ArrayNative::GetSpanDataFrom() + CORINFO_CLASS_HANDLE targetElemHnd = sig->sigInst.methInst[0]; + if (info.compCompHnd->getTypeForPrimitiveValueClass(targetElemHnd) == CORINFO_TYPE_UNDEF) + { return nullptr; } - GenTree* exactMethodDesc = impTokenToHandle(pResolvedToken); - if (exactMethodDesc == nullptr) - { // compDonotInline() + const unsigned targetElemSize = info.compCompHnd->getClassSize(targetElemHnd); + assert(targetElemSize != 0); + + const unsigned count = totalFieldSize / targetElemSize; + if (count == 0) + { return nullptr; } - // Call helper function. This gets the target address of the final destination callsite. + void* data = info.compCompHnd->getArrayInitializationData(fieldToken, totalFieldSize); + if (!data) + { + return nullptr; + } - return gtNewHelperCallNode(CORINFO_HELP_VIRTUAL_FUNC_PTR, TYP_I_IMPL, thisPtr, exactTypeDesc, exactMethodDesc); -} + // + // Ready to commit to the work + // + + impPopStack(); + + // Turn count and pointer value into constants. + GenTree* lengthValue = gtNewIconNode(count, TYP_INT); + GenTree* pointerValue = gtNewIconHandleNode((size_t)data, GTF_ICON_CONST_PTR); + + // Construct ReadOnlySpan to return. + CORINFO_CLASS_HANDLE spanHnd = sig->retTypeClass; + unsigned spanTempNum = lvaGrabTemp(true DEBUGARG("ReadOnlySpan for CreateSpan")); + lvaSetStruct(spanTempNum, spanHnd, false); + + GenTreeLclFld* pointerField = gtNewLclFldNode(spanTempNum, TYP_BYREF, 0); + GenTree* pointerFieldAsg = gtNewAssignNode(pointerField, pointerValue); + + GenTreeLclFld* lengthField = gtNewLclFldNode(spanTempNum, TYP_INT, TARGET_POINTER_SIZE); + GenTree* lengthFieldAsg = gtNewAssignNode(lengthField, lengthValue); + + // Now append a few statements the initialize the span + impAppendTree(lengthFieldAsg, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); + impAppendTree(pointerFieldAsg, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); + + // And finally create a tree that points at the span. + return impCreateLocalNode(spanTempNum DEBUGARG(0)); +} //------------------------------------------------------------------------ -// impBoxPatternMatch: match and import common box idioms +// impIntrinsic: possibly expand intrinsic call into alternate IR sequence // // Arguments: -// pResolvedToken - resolved token from the box operation -// codeAddr - position in IL stream after the box instruction -// codeEndp - end of IL stream -// opts - dictate pattern matching behavior +// newobjThis - for constructor calls, the tree for the newly allocated object +// clsHnd - handle for the intrinsic method's class +// method - handle for the intrinsic method +// sig - signature of the intrinsic method +// methodFlags - CORINFO_FLG_XXX flags of the intrinsic method +// memberRef - the token for the intrinsic method +// readonlyCall - true if call has a readonly prefix +// tailCall - true if call is in tail position +// pConstrainedResolvedToken -- resolved token for constrained call, or nullptr +// if call is not constrained +// constraintCallThisTransform -- this transform to apply for a constrained call +// pIntrinsicName [OUT] -- intrinsic name (see enumeration in namedintrinsiclist.h) +// for "traditional" jit intrinsics +// isSpecialIntrinsic [OUT] -- set true if intrinsic expansion is a call +// that is amenable to special downstream optimization opportunities // -// Return Value: -// Number of IL bytes matched and imported, -1 otherwise +// Returns: +// IR tree to use in place of the call, or nullptr if the jit should treat +// the intrinsic call like a normal call. +// +// pIntrinsicName set to non-illegal value if the call is recognized as a +// traditional jit intrinsic, even if the intrinsic is not expaned. +// +// isSpecial set true if the expansion is subject to special +// optimizations later in the jit processing // // Notes: -// pResolvedToken is known to be a value type; ref type boxing -// is handled in the CEE_BOX clause. - -int Compiler::impBoxPatternMatch(CORINFO_RESOLVED_TOKEN* pResolvedToken, - const BYTE* codeAddr, - const BYTE* codeEndp, - BoxPatterns opts) +// On success the IR tree may be a call to a different method or an inline +// sequence. If it is a call, then the intrinsic processing here is responsible +// for handling all the special cases, as upon return to impImportCall +// expanded intrinsics bypass most of the normal call processing. +// +// Intrinsics are generally not recognized in minopts and debug codegen. +// +// However, certain traditional intrinsics are identifed as "must expand" +// if there is no fallback implementation to invoke; these must be handled +// in all codegen modes. +// +// New style intrinsics (where the fallback implementation is in IL) are +// identified as "must expand" if they are invoked from within their +// own method bodies. +// +GenTree* Compiler::impIntrinsic(GenTree* newobjThis, + CORINFO_CLASS_HANDLE clsHnd, + CORINFO_METHOD_HANDLE method, + CORINFO_SIG_INFO* sig, + unsigned methodFlags, + int memberRef, + bool readonlyCall, + bool tailCall, + CORINFO_RESOLVED_TOKEN* pConstrainedResolvedToken, + CORINFO_THIS_TRANSFORM constraintCallThisTransform, + NamedIntrinsic* pIntrinsicName, + bool* isSpecialIntrinsic) { - if (codeAddr >= codeEndp) + assert((methodFlags & CORINFO_FLG_INTRINSIC) != 0); + + bool mustExpand = false; + bool isSpecial = false; + NamedIntrinsic ni = NI_Illegal; + + if ((methodFlags & CORINFO_FLG_INTRINSIC) != 0) { - return -1; + // The recursive non-virtual calls to Jit intrinsics are must-expand by convention. + mustExpand = mustExpand || (gtIsRecursiveCall(method) && !(methodFlags & CORINFO_FLG_VIRTUAL)); + + ni = lookupNamedIntrinsic(method); + + // We specially support the following on all platforms to allow for dead + // code optimization and to more generally support recursive intrinsics. + + if (ni == NI_IsSupported_True) + { + assert(sig->numArgs == 0); + return gtNewIconNode(true); + } + + if (ni == NI_IsSupported_False) + { + assert(sig->numArgs == 0); + return gtNewIconNode(false); + } + + if (ni == NI_Throw_PlatformNotSupportedException) + { + return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand); + } + + if ((ni > NI_SRCS_UNSAFE_START) && (ni < NI_SRCS_UNSAFE_END)) + { + assert(!mustExpand); + return impSRCSUnsafeIntrinsic(ni, clsHnd, method, sig); + } + +#ifdef FEATURE_HW_INTRINSICS + if ((ni > NI_HW_INTRINSIC_START) && (ni < NI_HW_INTRINSIC_END)) + { + GenTree* hwintrinsic = impHWIntrinsic(ni, clsHnd, method, sig, mustExpand); + + if (mustExpand && (hwintrinsic == nullptr)) + { + return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_NOT_IMPLEMENTED, method, sig, mustExpand); + } + + return hwintrinsic; + } + + if ((ni > NI_SIMD_AS_HWINTRINSIC_START) && (ni < NI_SIMD_AS_HWINTRINSIC_END)) + { + // These intrinsics aren't defined recursively and so they will never be mustExpand + // Instead, they provide software fallbacks that will be executed instead. + + assert(!mustExpand); + return impSimdAsHWIntrinsic(ni, clsHnd, method, sig, newobjThis); + } +#endif // FEATURE_HW_INTRINSICS } - switch (codeAddr[0]) + *pIntrinsicName = ni; + + if (ni == NI_System_StubHelpers_GetStubContext) { - case CEE_UNBOX_ANY: - // box + unbox.any - if (codeAddr + 1 + sizeof(mdToken) <= codeEndp) + // must be done regardless of DbgCode and MinOpts + return gtNewLclvNode(lvaStubArgumentVar, TYP_I_IMPL); + } + + if (ni == NI_System_StubHelpers_NextCallReturnAddress) + { + // For now we just avoid inlining anything into these methods since + // this intrinsic is only rarely used. We could do this better if we + // wanted to by trying to match which call is the one we need to get + // the return address of. + info.compHasNextCallRetAddr = true; + return new (this, GT_LABEL) GenTree(GT_LABEL, TYP_I_IMPL); + } + + switch (ni) + { + // CreateSpan must be expanded for NativeAOT + case NI_System_Runtime_CompilerServices_RuntimeHelpers_CreateSpan: + case NI_System_Runtime_CompilerServices_RuntimeHelpers_InitializeArray: + mustExpand |= IsTargetAbi(CORINFO_NATIVEAOT_ABI); + break; + + case NI_Internal_Runtime_MethodTable_Of: + case NI_System_Activator_AllocatorOf: + case NI_System_Activator_DefaultConstructorOf: + case NI_System_EETypePtr_EETypePtrOf: + mustExpand = true; + break; + + default: + break; + } + + GenTree* retNode = nullptr; + + // Under debug and minopts, only expand what is required. + // NextCallReturnAddress intrinsic returns the return address of the next call. + // If that call is an intrinsic and is expanded, codegen for NextCallReturnAddress will fail. + // To avoid that we conservatively expand only required intrinsics in methods that call + // the NextCallReturnAddress intrinsic. + if (!mustExpand && (opts.OptimizationDisabled() || info.compHasNextCallRetAddr)) + { + *pIntrinsicName = NI_Illegal; + return retNode; + } + + CorInfoType callJitType = sig->retType; + var_types callType = JITtype2varType(callJitType); + + /* First do the intrinsics which are always smaller than a call */ + + if (ni != NI_Illegal) + { + assert(retNode == nullptr); + switch (ni) + { + case NI_Array_Address: + case NI_Array_Get: + case NI_Array_Set: + retNode = impArrayAccessIntrinsic(clsHnd, sig, memberRef, readonlyCall, ni); + break; + + case NI_System_String_Equals: { - if (opts == BoxPatterns::MakeInlineObservation) + retNode = impStringEqualsOrStartsWith(/*startsWith:*/ false, sig, methodFlags); + break; + } + + case NI_System_MemoryExtensions_Equals: + case NI_System_MemoryExtensions_SequenceEqual: + { + retNode = impSpanEqualsOrStartsWith(/*startsWith:*/ false, sig, methodFlags); + break; + } + + case NI_System_String_StartsWith: + { + retNode = impStringEqualsOrStartsWith(/*startsWith:*/ true, sig, methodFlags); + break; + } + + case NI_System_MemoryExtensions_StartsWith: + { + retNode = impSpanEqualsOrStartsWith(/*startsWith:*/ true, sig, methodFlags); + break; + } + + case NI_System_MemoryExtensions_AsSpan: + case NI_System_String_op_Implicit: + { + assert(sig->numArgs == 1); + isSpecial = impStackTop().val->OperIs(GT_CNS_STR); + break; + } + + case NI_System_String_get_Chars: + { + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + GenTree* addr = gtNewIndexAddr(op1, op2, TYP_USHORT, NO_CLASS_HANDLE, OFFSETOF__CORINFO_String__chars, + OFFSETOF__CORINFO_String__stringLen); + retNode = gtNewIndexIndir(addr->AsIndexAddr()); + break; + } + + case NI_System_String_get_Length: + { + GenTree* op1 = impPopStack().val; + if (op1->OperIs(GT_CNS_STR)) { - compInlineResult->Note(InlineObservation::CALLEE_FOLDABLE_BOX); - return 1 + sizeof(mdToken); + // Optimize `ldstr + String::get_Length()` to CNS_INT + // e.g. "Hello".Length => 5 + GenTreeIntCon* iconNode = gtNewStringLiteralLength(op1->AsStrCon()); + if (iconNode != nullptr) + { + retNode = iconNode; + break; + } } + GenTreeArrLen* arrLen = gtNewArrLen(TYP_INT, op1, OFFSETOF__CORINFO_String__stringLen, compCurBB); + op1 = arrLen; - CORINFO_RESOLVED_TOKEN unboxResolvedToken; + // Getting the length of a null string should throw + op1->gtFlags |= GTF_EXCEPT; - impResolveToken(codeAddr + 1, &unboxResolvedToken, CORINFO_TOKENKIND_Class); + retNode = op1; + break; + } - // See if the resolved tokens describe types that are equal. - const TypeCompareState compare = - info.compCompHnd->compareTypesForEquality(unboxResolvedToken.hClass, pResolvedToken->hClass); + case NI_System_Runtime_CompilerServices_RuntimeHelpers_CreateSpan: + { + retNode = impCreateSpanIntrinsic(sig); + break; + } - bool optimize = false; + case NI_System_Runtime_CompilerServices_RuntimeHelpers_InitializeArray: + { + retNode = impInitializeArrayIntrinsic(sig); + break; + } - // If so, box/unbox.any is a nop. - if (compare == TypeCompareState::Must) + case NI_System_Runtime_CompilerServices_RuntimeHelpers_IsKnownConstant: + { + GenTree* op1 = impPopStack().val; + if (op1->OperIsConst()) { - optimize = true; + // op1 is a known constant, replace with 'true'. + retNode = gtNewIconNode(1); + JITDUMP("\nExpanding RuntimeHelpers.IsKnownConstant to true early\n"); + // We can also consider FTN_ADDR and typeof(T) here } - else if (compare == TypeCompareState::MustNot) + else { - // An attempt to catch cases where we mix enums and primitives, e.g.: - // (IntEnum)(object)myInt - // (byte)(object)myByteEnum - // - CorInfoType typ = info.compCompHnd->getTypeForPrimitiveValueClass(unboxResolvedToken.hClass); - if ((typ >= CORINFO_TYPE_BYTE) && (typ <= CORINFO_TYPE_ULONG) && - (info.compCompHnd->getTypeForPrimitiveValueClass(pResolvedToken->hClass) == typ)) - { - optimize = true; - } + // op1 is not a known constant, we'll do the expansion in morph + retNode = new (this, GT_INTRINSIC) GenTreeIntrinsic(TYP_INT, op1, ni, method); + JITDUMP("\nConverting RuntimeHelpers.IsKnownConstant to:\n"); + DISPTREE(retNode); } + break; + } - if (optimize) + case NI_System_Runtime_InteropService_MemoryMarshal_GetArrayDataReference: + { + assert(sig->numArgs == 1); + + GenTree* array = impPopStack().val; + CORINFO_CLASS_HANDLE elemHnd = sig->sigInst.classInst[0]; + CorInfoType jitType = info.compCompHnd->asCorInfoType(elemHnd); + var_type elemType = JITtype2varType(jitType); + + GenTree* index = gtNewIconNode(0, TYP_I_IMPL); + GenTreeIndexAddr* indexAddr = gtNewArrayIndexAddr(array, index, elemType, elemHnd); + indexAddr->gtFlags &= ~GTF_INX_RNGCHK; + break; + } + + case NI_Internal_Runtime_MethodTable_Of: + case NI_System_Activator_AllocatorOf: + case NI_System_Activator_DefaultConstructorOf: + case NI_System_EETypePtr_EETypePtrOf: + { + assert(IsTargetAbi(CORINFO_NATIVEAOT_ABI)); // Only NativeAOT supports it. + CORINFO_RESOLVED_TOKEN resolvedToken; + resolvedToken.tokenContext = impTokenLookupContextHandle; + resolvedToken.tokenScope = info.compScopeHnd; + resolvedToken.token = memberRef; + resolvedToken.tokenType = CORINFO_TOKENKIND_Method; + + CORINFO_GENERICHANDLE_RESULT embedInfo; + info.compCompHnd->expandRawHandleIntrinsic(&resolvedToken, &embedInfo); + + GenTree* rawHandle = impLookupToTree(&resolvedToken, &embedInfo.lookup, gtTokenToIconFlags(memberRef), + embedInfo.compileTimeHandle); + if (rawHandle == nullptr) { - JITDUMP("\n Importing BOX; UNBOX.ANY as NOP\n"); - // Skip the next unbox.any instruction - return 1 + sizeof(mdToken); + return nullptr; } + + noway_assert(genTypeSize(rawHandle->TypeGet()) == genTypeSize(TYP_I_IMPL)); + + unsigned rawHandleSlot = lvaGrabTemp(true DEBUGARG("rawHandle")); + impAssignTempGen(rawHandleSlot, rawHandle, clsHnd, (unsigned)CHECK_SPILL_NONE); + + GenTree* lclVar = gtNewLclvNode(rawHandleSlot, TYP_I_IMPL); + GenTree* lclVarAddr = gtNewOperNode(GT_ADDR, TYP_I_IMPL, lclVar); + var_types resultType = JITtype2varType(sig->retType); + retNode = gtNewOperNode(GT_IND, resultType, lclVarAddr); + + break; } - break; - case CEE_BRTRUE: - case CEE_BRTRUE_S: - case CEE_BRFALSE: - case CEE_BRFALSE_S: - // box + br_true/false - if ((codeAddr + ((codeAddr[0] >= CEE_BRFALSE) ? 5 : 2)) <= codeEndp) + case NI_System_Span_get_Item: + case NI_System_ReadOnlySpan_get_Item: { - if (opts == BoxPatterns::MakeInlineObservation) + // Have index, stack pointer-to Span s on the stack. Expand to: + // + // For Span + // Comma + // BoundsCheck(index, s->_length) + // s->_reference + index * sizeof(T) + // + // For ReadOnlySpan -- same expansion, as it now returns a readonly ref + // + // Signature should show one class type parameter, which + // we need to examine. + assert(sig->sigInst.classInstCount == 1); + assert(sig->numArgs == 1); + CORINFO_CLASS_HANDLE spanElemHnd = sig->sigInst.classInst[0]; + const unsigned elemSize = info.compCompHnd->getClassSize(spanElemHnd); + assert(elemSize > 0); + + const bool isReadOnly = (ni == NI_System_ReadOnlySpan_get_Item); + + JITDUMP("\nimpIntrinsic: Expanding %sSpan.get_Item, T=%s, sizeof(T)=%u\n", + isReadOnly ? "ReadOnly" : "", eeGetClassName(spanElemHnd), elemSize); + + GenTree* index = impPopStack().val; + GenTree* ptrToSpan = impPopStack().val; + GenTree* indexClone = nullptr; + GenTree* ptrToSpanClone = nullptr; + assert(genActualType(index) == TYP_INT); + assert(ptrToSpan->TypeGet() == TYP_BYREF); + +#if defined(DEBUG) + if (verbose) { - compInlineResult->Note(InlineObservation::CALLEE_FOLDABLE_BOX); - return 0; + printf("with ptr-to-span\n"); + gtDispTree(ptrToSpan); + printf("and index\n"); + gtDispTree(index); } +#endif // defined(DEBUG) - GenTree* const treeToBox = impStackTop().val; - bool canOptimize = true; - GenTree* treeToNullcheck = nullptr; + // We need to use both index and ptr-to-span twice, so clone or spill. + index = impCloneExpr(index, &indexClone, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, + nullptr DEBUGARG("Span.get_Item index")); + ptrToSpan = impCloneExpr(ptrToSpan, &ptrToSpanClone, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, + nullptr DEBUGARG("Span.get_Item ptrToSpan")); - // Can the thing being boxed cause a side effect? - if ((treeToBox->gtFlags & GTF_SIDE_EFFECT) != 0) + // Bounds check + CORINFO_FIELD_HANDLE lengthHnd = info.compCompHnd->getFieldInClass(clsHnd, 1); + const unsigned lengthOffset = info.compCompHnd->getFieldOffset(lengthHnd); + GenTree* length = gtNewFieldRef(TYP_INT, lengthHnd, ptrToSpan, lengthOffset); + GenTree* boundsCheck = new (this, GT_BOUNDS_CHECK) GenTreeBoundsChk(index, length, SCK_RNGCHK_FAIL); + + // Element access + index = indexClone; + +#ifdef TARGET_64BIT + if (index->OperGet() == GT_CNS_INT) { - // Is this a side effect we can replicate cheaply? - if (((treeToBox->gtFlags & GTF_SIDE_EFFECT) == GTF_EXCEPT) && - treeToBox->OperIs(GT_OBJ, GT_BLK, GT_IND)) - { - // If the only side effect comes from the dereference itself, yes. - GenTree* const addr = treeToBox->AsOp()->gtGetOp1(); + index->gtType = TYP_I_IMPL; + } + else + { + index = gtNewCastNode(TYP_I_IMPL, index, true, TYP_I_IMPL); + } +#endif - if ((addr->gtFlags & GTF_SIDE_EFFECT) != 0) - { - canOptimize = false; - } - else if (fgAddrCouldBeNull(addr)) - { - treeToNullcheck = addr; - } - } - else - { - canOptimize = false; - } + if (elemSize != 1) + { + GenTree* sizeofNode = gtNewIconNode(static_cast(elemSize), TYP_I_IMPL); + index = gtNewOperNode(GT_MUL, TYP_I_IMPL, index, sizeofNode); } - if (canOptimize) + CORINFO_FIELD_HANDLE ptrHnd = info.compCompHnd->getFieldInClass(clsHnd, 0); + const unsigned ptrOffset = info.compCompHnd->getFieldOffset(ptrHnd); + GenTree* data = gtNewFieldRef(TYP_BYREF, ptrHnd, ptrToSpanClone, ptrOffset); + GenTree* result = gtNewOperNode(GT_ADD, TYP_BYREF, data, index); + + // Prepare result + var_types resultType = JITtype2varType(sig->retType); + assert(resultType == result->TypeGet()); + retNode = gtNewOperNode(GT_COMMA, resultType, boundsCheck, result); + + break; + } + + case NI_System_RuntimeTypeHandle_GetValueInternal: + { + GenTree* op1 = impStackTop(0).val; + if (op1->gtOper == GT_CALL && (op1->AsCall()->gtCallType == CT_HELPER) && + gtIsTypeHandleToRuntimeTypeHandleHelper(op1->AsCall())) { - if ((opts == BoxPatterns::IsByRefLike) || - info.compCompHnd->getBoxHelper(pResolvedToken->hClass) == CORINFO_HELP_BOX) - { - JITDUMP("\n Importing BOX; BR_TRUE/FALSE as %sconstant\n", - treeToNullcheck == nullptr ? "" : "nullcheck+"); - impPopStack(); + // Old tree + // Helper-RuntimeTypeHandle -> TreeToGetNativeTypeHandle + // + // New tree + // TreeToGetNativeTypeHandle - GenTree* result = gtNewIconNode(1); + // Remove call to helper and return the native TypeHandle pointer that was the parameter + // to that helper. - if (treeToNullcheck != nullptr) - { - GenTree* nullcheck = gtNewNullCheck(treeToNullcheck, compCurBB); - result = gtNewOperNode(GT_COMMA, TYP_INT, nullcheck, result); - } + op1 = impPopStack().val; - impPushOnStack(result, typeInfo(TI_INT)); - return 0; - } + // Get native TypeHandle argument to old helper + assert(op1->AsCall()->gtArgs.CountArgs() == 1); + op1 = op1->AsCall()->gtArgs.GetArgByIndex(0)->GetNode(); + retNode = op1; } + // Call the regular function. + break; } - break; - case CEE_ISINST: - if (codeAddr + 1 + sizeof(mdToken) + 1 <= codeEndp) + case NI_System_Type_GetTypeFromHandle: { - const BYTE* nextCodeAddr = codeAddr + 1 + sizeof(mdToken); - - switch (nextCodeAddr[0]) + GenTree* op1 = impStackTop(0).val; + CorInfoHelpFunc typeHandleHelper; + if (op1->gtOper == GT_CALL && (op1->AsCall()->gtCallType == CT_HELPER) && + gtIsTypeHandleToRuntimeTypeHandleHelper(op1->AsCall(), &typeHandleHelper)) { - // box + isinst + br_true/false - case CEE_BRTRUE: - case CEE_BRTRUE_S: - case CEE_BRFALSE: - case CEE_BRFALSE_S: - if ((nextCodeAddr + ((nextCodeAddr[0] >= CEE_BRFALSE) ? 5 : 2)) <= codeEndp) - { - if (opts == BoxPatterns::MakeInlineObservation) - { - compInlineResult->Note(InlineObservation::CALLEE_FOLDABLE_BOX); - return 1 + sizeof(mdToken); - } - - if ((impStackTop().val->gtFlags & GTF_SIDE_EFFECT) == 0) - { - CorInfoHelpFunc foldAsHelper; - if (opts == BoxPatterns::IsByRefLike) - { - // Treat ByRefLike types as if they were regular boxing operations - // so they can be elided. - foldAsHelper = CORINFO_HELP_BOX; - } - else - { - foldAsHelper = info.compCompHnd->getBoxHelper(pResolvedToken->hClass); - } - - if (foldAsHelper == CORINFO_HELP_BOX) - { - CORINFO_RESOLVED_TOKEN isInstResolvedToken; - - impResolveToken(codeAddr + 1, &isInstResolvedToken, CORINFO_TOKENKIND_Casting); + op1 = impPopStack().val; + // Replace helper with a more specialized helper that returns RuntimeType + if (typeHandleHelper == CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPEHANDLE) + { + typeHandleHelper = CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE; + } + else + { + assert(typeHandleHelper == CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPEHANDLE_MAYBENULL); + typeHandleHelper = CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE_MAYBENULL; + } + assert(op1->AsCall()->gtArgs.CountArgs() == 1); + op1 = gtNewHelperCallNode(typeHandleHelper, TYP_REF, + op1->AsCall()->gtArgs.GetArgByIndex(0)->GetEarlyNode()); + op1->gtType = TYP_REF; + retNode = op1; + } + break; + } - TypeCompareState castResult = - info.compCompHnd->compareTypesForCast(pResolvedToken->hClass, - isInstResolvedToken.hClass); - if (castResult != TypeCompareState::May) - { - JITDUMP("\n Importing BOX; ISINST; BR_TRUE/FALSE as constant\n"); - impPopStack(); + case NI_System_Type_op_Equality: + case NI_System_Type_op_Inequality: + { + JITDUMP("Importing Type.op_*Equality intrinsic\n"); + GenTree* op1 = impStackTop(1).val; + GenTree* op2 = impStackTop(0).val; + GenTree* optTree = gtFoldTypeEqualityCall(ni == NI_System_Type_op_Equality, op1, op2); + if (optTree != nullptr) + { + // Success, clean up the evaluation stack. + impPopStack(); + impPopStack(); - impPushOnStack(gtNewIconNode((castResult == TypeCompareState::Must) ? 1 : 0), - typeInfo(TI_INT)); + // See if we can optimize even further, to a handle compare. + optTree = gtFoldTypeCompare(optTree); - // Skip the next isinst instruction - return 1 + sizeof(mdToken); - } - } - else if (foldAsHelper == CORINFO_HELP_BOX_NULLABLE) - { - // For nullable we're going to fold it to "ldfld hasValue + brtrue/brfalse" or - // "ldc.i4.0 + brtrue/brfalse" in case if the underlying type is not castable to - // the target type. - CORINFO_RESOLVED_TOKEN isInstResolvedToken; - impResolveToken(codeAddr + 1, &isInstResolvedToken, CORINFO_TOKENKIND_Casting); + // See if we can now fold a handle compare to a constant. + optTree = gtFoldExpr(optTree); - CORINFO_CLASS_HANDLE nullableCls = pResolvedToken->hClass; - CORINFO_CLASS_HANDLE underlyingCls = info.compCompHnd->getTypeForBox(nullableCls); + retNode = optTree; + } + else + { + // Retry optimizing these later + isSpecial = true; + } + break; + } - TypeCompareState castResult = - info.compCompHnd->compareTypesForCast(underlyingCls, - isInstResolvedToken.hClass); + case NI_System_Enum_HasFlag: + { + GenTree* thisOp = impStackTop(1).val; + GenTree* flagOp = impStackTop(0).val; + GenTree* optTree = gtOptimizeEnumHasFlag(thisOp, flagOp); - if (castResult == TypeCompareState::Must) - { - const CORINFO_FIELD_HANDLE hasValueFldHnd = - info.compCompHnd->getFieldInClass(nullableCls, 0); + if (optTree != nullptr) + { + // Optimization successful. Pop the stack for real. + impPopStack(); + impPopStack(); + retNode = optTree; + } + else + { + // Retry optimizing this during morph. + isSpecial = true; + } - assert(info.compCompHnd->getFieldOffset(hasValueFldHnd) == 0); + break; + } - GenTree* objToBox = impPopStack().val; + case NI_System_Type_IsAssignableFrom: + { + GenTree* typeTo = impStackTop(1).val; + GenTree* typeFrom = impStackTop(0).val; - // Spill struct to get its address (to access hasValue field) - objToBox = impGetStructAddr(objToBox, nullableCls, CHECK_SPILL_ALL, true); + retNode = impTypeIsAssignable(typeTo, typeFrom); + break; + } - impPushOnStack(gtNewFieldRef(TYP_BOOL, hasValueFldHnd, objToBox, 0), - typeInfo(TI_INT)); + case NI_System_Type_IsAssignableTo: + { + GenTree* typeTo = impStackTop(0).val; + GenTree* typeFrom = impStackTop(1).val; - JITDUMP("\n Importing BOX; ISINST; BR_TRUE/FALSE as nullableVT.hasValue\n"); - return 1 + sizeof(mdToken); - } - else if (castResult == TypeCompareState::MustNot) - { - impPopStack(); - impPushOnStack(gtNewIconNode(0), typeInfo(TI_INT)); - JITDUMP("\n Importing BOX; ISINST; BR_TRUE/FALSE as constant (false)\n"); - return 1 + sizeof(mdToken); - } - } - } - } - break; + retNode = impTypeIsAssignable(typeTo, typeFrom); + break; + } - // box + isinst + unbox.any - case CEE_UNBOX_ANY: - if ((nextCodeAddr + 1 + sizeof(mdToken)) <= codeEndp) + case NI_System_Type_get_IsValueType: + case NI_System_Type_get_IsByRefLike: + { + // Optimize + // + // call Type.GetTypeFromHandle (which is replaced with CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE) + // call Type.IsXXX + // + // to `true` or `false` + // e.g., `typeof(int).IsValueType` => `true` + // e.g., `typeof(Span).IsByRefLike` => `true` + if (impStackTop().val->IsCall()) + { + GenTreeCall* call = impStackTop().val->AsCall(); + if (call->gtCallMethHnd == eeFindHelper(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE)) + { + assert(call->gtArgs.CountArgs() == 1); + CORINFO_CLASS_HANDLE hClass = + gtGetHelperArgClassHandle(call->gtArgs.GetArgByIndex(0)->GetEarlyNode()); + if (hClass != NO_CLASS_HANDLE) { - if (opts == BoxPatterns::MakeInlineObservation) + switch (ni) { - compInlineResult->Note(InlineObservation::CALLEE_FOLDABLE_BOX); - return 2 + sizeof(mdToken) * 2; + case NI_System_Type_get_IsValueType: + retNode = gtNewIconNode( + (eeIsValueClass(hClass) && + // pointers are not value types (e.g. typeof(int*).IsValueType is false) + info.compCompHnd->asCorInfoType(hClass) != CORINFO_TYPE_PTR) + ? 1 + : 0); + break; + case NI_System_Type_get_IsByRefLike: + retNode = gtNewIconNode( + (info.compCompHnd->getClassAttribs(hClass) & CORINFO_FLG_BYREF_LIKE) ? 1 : 0); + break; + default: + NO_WAY("Intrinsic not supported in this path."); } + impPopStack(); // drop CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE call + } + } + } + break; + } - // See if the resolved tokens in box, isinst and unbox.any describe types that are equal. - CORINFO_RESOLVED_TOKEN isinstResolvedToken = {}; - impResolveToken(codeAddr + 1, &isinstResolvedToken, CORINFO_TOKENKIND_Class); - - if (info.compCompHnd->compareTypesForEquality(isinstResolvedToken.hClass, - pResolvedToken->hClass) == - TypeCompareState::Must) - { - CORINFO_RESOLVED_TOKEN unboxResolvedToken = {}; - impResolveToken(nextCodeAddr + 1, &unboxResolvedToken, CORINFO_TOKENKIND_Class); - - // If so, box + isinst + unbox.any is a nop. - if (info.compCompHnd->compareTypesForEquality(unboxResolvedToken.hClass, - pResolvedToken->hClass) == - TypeCompareState::Must) - { - JITDUMP("\n Importing BOX; ISINST, UNBOX.ANY as NOP\n"); - return 2 + sizeof(mdToken) * 2; - } - } + case NI_System_Threading_Thread_get_ManagedThreadId: + { + if (impStackTop().val->OperIs(GT_RET_EXPR)) + { + GenTreeCall* call = impStackTop().val->AsRetExpr()->gtInlineCandidate->AsCall(); + if (call->gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC) + { + if (lookupNamedIntrinsic(call->gtCallMethHnd) == NI_System_Threading_Thread_get_CurrentThread) + { + // drop get_CurrentThread() call + impPopStack(); + call->ReplaceWith(gtNewNothingNode(), this); + retNode = gtNewHelperCallNode(CORINFO_HELP_GETCURRENTMANAGEDTHREADID, TYP_INT); } - break; + } } + break; } - break; - default: - break; - } +#ifdef TARGET_ARM64 + // Intrinsify Interlocked.Or and Interlocked.And only for arm64-v8.1 (and newer) + // TODO-CQ: Implement for XArch (https://github.com/dotnet/runtime/issues/32239). + case NI_System_Threading_Interlocked_Or: + case NI_System_Threading_Interlocked_And: + { + if (compOpportunisticallyDependsOn(InstructionSet_Atomics)) + { + assert(sig->numArgs == 2); + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + genTreeOps op = (ni == NI_System_Threading_Interlocked_Or) ? GT_XORR : GT_XAND; + retNode = gtNewOperNode(op, genActualType(callType), op1, op2); + retNode->gtFlags |= GTF_GLOB_REF | GTF_ASG; + } + break; + } +#endif // TARGET_ARM64 - return -1; -} +#if defined(TARGET_XARCH) || defined(TARGET_ARM64) + // TODO-ARM-CQ: reenable treating InterlockedCmpXchg32 operation as intrinsic + case NI_System_Threading_Interlocked_CompareExchange: + { + var_types retType = JITtype2varType(sig->retType); + if ((retType == TYP_LONG) && (TARGET_POINTER_SIZE == 4)) + { + break; + } + if ((retType != TYP_INT) && (retType != TYP_LONG)) + { + break; + } -//------------------------------------------------------------------------ -// impImportAndPushBox: build and import a value-type box -// -// Arguments: -// pResolvedToken - resolved token from the box operation -// -// Return Value: -// None. -// -// Side Effects: -// The value to be boxed is popped from the stack, and a tree for -// the boxed value is pushed. This method may create upstream -// statements, spill side effecting trees, and create new temps. -// -// If importing an inlinee, we may also discover the inline must -// fail. If so there is no new value pushed on the stack. Callers -// should use CompDoNotInline after calling this method to see if -// ongoing importation should be aborted. -// -// Notes: -// Boxing of ref classes results in the same value as the value on -// the top of the stack, so is handled inline in impImportBlockCode -// for the CEE_BOX case. Only value or primitive type boxes make it -// here. -// -// Boxing for nullable types is done via a helper call; boxing -// of other value types is expanded inline or handled via helper -// call, depending on the jit's codegen mode. -// -// When the jit is operating in size and time constrained modes, -// using a helper call here can save jit time and code size. But it -// also may inhibit cleanup optimizations that could have also had a -// even greater benefit effect on code size and jit time. An optimal -// strategy may need to peek ahead and see if it is easy to tell how -// the box is being used. For now, we defer. + assert(callType != TYP_STRUCT); + assert(sig->numArgs == 3); -void Compiler::impImportAndPushBox(CORINFO_RESOLVED_TOKEN* pResolvedToken) -{ - // Spill any special side effects - impSpillSpecialSideEff(); + GenTree* op3 = impPopStack().val; // comparand + GenTree* op2 = impPopStack().val; // value + GenTree* op1 = impPopStack().val; // location - // Get get the expression to box from the stack. - GenTree* op1 = nullptr; - GenTree* op2 = nullptr; - StackEntry se = impPopStack(); - CORINFO_CLASS_HANDLE operCls = se.seTypeInfo.GetClassHandle(); - GenTree* exprToBox = se.val; + GenTree* node = new (this, GT_CMPXCHG) GenTreeCmpXchg(genActualType(callType), op1, op2, op3); - // Look at what helper we should use. - CorInfoHelpFunc boxHelper = info.compCompHnd->getBoxHelper(pResolvedToken->hClass); + node->AsCmpXchg()->gtOpLocation->gtFlags |= GTF_DONT_CSE; + retNode = node; + break; + } - // Determine what expansion to prefer. - // - // In size/time/debuggable constrained modes, the helper call - // expansion for box is generally smaller and is preferred, unless - // the value to box is a struct that comes from a call. In that - // case the call can construct its return value directly into the - // box payload, saving possibly some up-front zeroing. - // - // Currently primitive type boxes always get inline expanded. We may - // want to do the same for small structs if they don't come from - // calls and don't have GC pointers, since explicitly copying such - // structs is cheap. - JITDUMP("\nCompiler::impImportAndPushBox -- handling BOX(value class) via"); - bool canExpandInline = (boxHelper == CORINFO_HELP_BOX); - bool optForSize = !exprToBox->IsCall() && (operCls != nullptr) && opts.OptimizationDisabled(); - bool expandInline = canExpandInline && !optForSize; + case NI_System_Threading_Interlocked_Exchange: + case NI_System_Threading_Interlocked_ExchangeAdd: + { + assert(callType != TYP_STRUCT); + assert(sig->numArgs == 2); - if (expandInline) - { - JITDUMP(" inline allocate/copy sequence\n"); + var_types retType = JITtype2varType(sig->retType); + if ((retType == TYP_LONG) && (TARGET_POINTER_SIZE == 4)) + { + break; + } + if ((retType != TYP_INT) && (retType != TYP_LONG)) + { + break; + } - // we are doing 'normal' boxing. This means that we can inline the box operation - // Box(expr) gets morphed into - // temp = new(clsHnd) - // cpobj(temp+4, expr, clsHnd) - // push temp - // The code paths differ slightly below for structs and primitives because - // "cpobj" differs in these cases. In one case you get - // impAssignStructPtr(temp+4, expr, clsHnd) - // and the other you get - // *(temp+4) = expr + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; - if (opts.OptimizationDisabled()) - { - // For minopts/debug code, try and minimize the total number - // of box temps by reusing an existing temp when possible. - if (impBoxTempInUse || impBoxTemp == BAD_VAR_NUM) + // This creates: + // val + // XAdd + // addr + // field (for example) + // + // In the case where the first argument is the address of a local, we might + // want to make this *not* make the var address-taken -- but atomic instructions + // on a local are probably pretty useless anyway, so we probably don't care. + + op1 = gtNewOperNode(ni == NI_System_Threading_Interlocked_ExchangeAdd ? GT_XADD : GT_XCHG, + genActualType(callType), op1, op2); + op1->gtFlags |= GTF_GLOB_REF | GTF_ASG; + retNode = op1; + break; + } +#endif // defined(TARGET_XARCH) || defined(TARGET_ARM64) + + case NI_System_Threading_Interlocked_MemoryBarrier: + case NI_System_Threading_Interlocked_ReadMemoryBarrier: { - impBoxTemp = lvaGrabTemp(true DEBUGARG("Reusable Box Helper")); + assert(sig->numArgs == 0); + + GenTree* op1 = new (this, GT_MEMORYBARRIER) GenTree(GT_MEMORYBARRIER, TYP_VOID); + op1->gtFlags |= GTF_GLOB_REF | GTF_ASG; + + // On XARCH `NI_System_Threading_Interlocked_ReadMemoryBarrier` fences need not be emitted. + // However, we still need to capture the effect on reordering. + if (ni == NI_System_Threading_Interlocked_ReadMemoryBarrier) + { + op1->gtFlags |= GTF_MEMORYBARRIER_LOAD; + } + + retNode = op1; + break; } - } - else - { - // When optimizing, use a new temp for each box operation - // since we then know the exact class of the box temp. - impBoxTemp = lvaGrabTemp(true DEBUGARG("Single-def Box Helper")); - lvaTable[impBoxTemp].lvType = TYP_REF; - lvaTable[impBoxTemp].lvSingleDef = 1; - JITDUMP("Marking V%02u as a single def local\n", impBoxTemp); - const bool isExact = true; - lvaSetClass(impBoxTemp, pResolvedToken->hClass, isExact); - } - // needs to stay in use until this box expression is appended - // some other node. We approximate this by keeping it alive until - // the opcode stack becomes empty - impBoxTempInUse = true; +#ifdef FEATURE_HW_INTRINSICS + case NI_System_Math_FusedMultiplyAdd: + { +#ifdef TARGET_XARCH + if (compExactlyDependsOn(InstructionSet_FMA)) + { + assert(varTypeIsFloating(callType)); + + // We are constructing a chain of intrinsics similar to: + // return FMA.MultiplyAddScalar( + // Vector128.CreateScalarUnsafe(x), + // Vector128.CreateScalarUnsafe(y), + // Vector128.CreateScalarUnsafe(z) + // ).ToScalar(); + + GenTree* op3 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, impPopStack().val, + NI_Vector128_CreateScalarUnsafe, callJitType, 16); + GenTree* op2 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, impPopStack().val, + NI_Vector128_CreateScalarUnsafe, callJitType, 16); + GenTree* op1 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, impPopStack().val, + NI_Vector128_CreateScalarUnsafe, callJitType, 16); + GenTree* res = + gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, op3, NI_FMA_MultiplyAddScalar, callJitType, 16); + + retNode = gtNewSimdHWIntrinsicNode(callType, res, NI_Vector128_ToScalar, callJitType, 16); + break; + } +#elif defined(TARGET_ARM64) + if (compExactlyDependsOn(InstructionSet_AdvSimd)) + { + assert(varTypeIsFloating(callType)); - // Remember the current last statement in case we need to move - // a range of statements to ensure the box temp is initialized - // before it's used. - // - Statement* const cursor = impLastStmt; + // We are constructing a chain of intrinsics similar to: + // return AdvSimd.FusedMultiplyAddScalar( + // Vector64.Create{ScalarUnsafe}(z), + // Vector64.Create{ScalarUnsafe}(y), + // Vector64.Create{ScalarUnsafe}(x) + // ).ToScalar(); - const bool useParent = false; - op1 = gtNewAllocObjNode(pResolvedToken, useParent); - if (op1 == nullptr) - { - // If we fail to create the newobj node, we must be inlining - // and have run across a type we can't describe. - // - assert(compDonotInline()); - return; - } + NamedIntrinsic createVector64 = + (callType == TYP_DOUBLE) ? NI_Vector64_Create : NI_Vector64_CreateScalarUnsafe; - // Remember that this basic block contains 'new' of an object, - // and so does this method - // - compCurBB->bbFlags |= BBF_HAS_NEWOBJ; - optMethodFlags |= OMF_HAS_NEWOBJ; + constexpr unsigned int simdSize = 8; - // Assign the boxed object to the box temp. - // - GenTree* asg = gtNewTempAssign(impBoxTemp, op1); - Statement* asgStmt = impAppendTree(asg, CHECK_SPILL_NONE, impCurStmtDI); + GenTree* op3 = + gtNewSimdHWIntrinsicNode(TYP_SIMD8, impPopStack().val, createVector64, callJitType, simdSize); + GenTree* op2 = + gtNewSimdHWIntrinsicNode(TYP_SIMD8, impPopStack().val, createVector64, callJitType, simdSize); + GenTree* op1 = + gtNewSimdHWIntrinsicNode(TYP_SIMD8, impPopStack().val, createVector64, callJitType, simdSize); - // If the exprToBox is a call that returns its value via a ret buf arg, - // move the assignment statement(s) before the call (which must be a top level tree). - // - // We do this because impAssignStructPtr (invoked below) will - // back-substitute into a call when it sees a GT_RET_EXPR and the call - // has a hidden buffer pointer, So we need to reorder things to avoid - // creating out-of-sequence IR. - // - if (varTypeIsStruct(exprToBox) && exprToBox->OperIs(GT_RET_EXPR)) - { - GenTreeCall* const call = exprToBox->AsRetExpr()->gtInlineCandidate->AsCall(); + // Note that AdvSimd.FusedMultiplyAddScalar(op1,op2,op3) corresponds to op1 + op2 * op3 + // while Math{F}.FusedMultiplyAddScalar(op1,op2,op3) corresponds to op1 * op2 + op3 + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op3, op2, op1, NI_AdvSimd_FusedMultiplyAddScalar, + callJitType, simdSize); - if (call->ShouldHaveRetBufArg()) + retNode = gtNewSimdHWIntrinsicNode(callType, retNode, NI_Vector64_ToScalar, callJitType, simdSize); + break; + } +#endif + + // TODO-CQ-XArch: Ideally we would create a GT_INTRINSIC node for fma, however, that currently + // requires more extensive changes to valuenum to support methods with 3 operands + + // We want to generate a GT_INTRINSIC node in the case the call can't be treated as + // a target intrinsic so that we can still benefit from CSE and constant folding. + + break; + } +#endif // FEATURE_HW_INTRINSICS + + case NI_System_Math_Abs: + case NI_System_Math_Acos: + case NI_System_Math_Acosh: + case NI_System_Math_Asin: + case NI_System_Math_Asinh: + case NI_System_Math_Atan: + case NI_System_Math_Atanh: + case NI_System_Math_Atan2: + case NI_System_Math_Cbrt: + case NI_System_Math_Ceiling: + case NI_System_Math_Cos: + case NI_System_Math_Cosh: + case NI_System_Math_Exp: + case NI_System_Math_Floor: + case NI_System_Math_FMod: + case NI_System_Math_ILogB: + case NI_System_Math_Log: + case NI_System_Math_Log2: + case NI_System_Math_Log10: { - JITDUMP("Must insert newobj stmts for box before call [%06u]\n", dspTreeID(call)); + retNode = impMathIntrinsic(method, sig, callType, ni, tailCall); + break; + } +#if defined(TARGET_ARM64) + // ARM64 has fmax/fmin which are IEEE754:2019 minimum/maximum compatible + // TODO-XARCH-CQ: Enable this for XARCH when one of the arguments is a constant + // so we can then emit maxss/minss and avoid NaN/-0.0 handling + case NI_System_Math_Max: + case NI_System_Math_Min: +#endif - // Walk back through the statements in this block, looking for the one - // that has this call as the root node. - // - // Because gtNewTempAssign (above) may have added statements that - // feed into the actual assignment we need to move this set of added - // statements as a group. - // - // Note boxed allocations are side-effect free (no com or finalizer) so - // our only worries here are (correctness) not overlapping the box temp - // lifetime and (perf) stretching the temp lifetime across the inlinee - // body. - // - // Since this is an inline candidate, we must be optimizing, and so we have - // a unique box temp per call. So no worries about overlap. - // - assert(!opts.OptimizationDisabled()); +#if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_XARCH) + case NI_System_Math_Max: + case NI_System_Math_Min: + { + assert(varTypeIsFloating(callType)); + assert(sig->numArgs == 2); - // Lifetime stretching could addressed with some extra cleverness--sinking - // the allocation back down to just before the copy, once we figure out - // where the copy is. We defer for now. - // - Statement* insertBeforeStmt = cursor; - noway_assert(insertBeforeStmt != nullptr); + GenTreeDblCon* cnsNode = nullptr; + GenTree* otherNode = nullptr; - while (true) + GenTree* op2 = impStackTop().val; + GenTree* op1 = impStackTop(1).val; + + if (op2->IsCnsFltOrDbl()) { - if (insertBeforeStmt->GetRootNode() == call) + cnsNode = op2->AsDblCon(); + otherNode = op1; + } + else if (op1->IsCnsFltOrDbl()) + { + cnsNode = op1->AsDblCon(); + otherNode = op2; + } + + if (cnsNode == nullptr) + { + // no constant node, nothing to do + break; + } + + if (otherNode->IsCnsFltOrDbl()) + { + // both are constant, we can fold this operation completely. Pop both peeked values + + if (ni == NI_System_Math_Max) { - break; + cnsNode->gtDconVal = + FloatingPointUtils::maximum(cnsNode->gtDconVal, otherNode->AsDblCon()->gtDconVal); + } + else + { + assert(ni == NI_System_Math_Min); + cnsNode->gtDconVal = + FloatingPointUtils::minimum(cnsNode->gtDconVal, otherNode->AsDblCon()->gtDconVal); } - // If we've searched all the statements in the block and failed to - // find the call, then something's wrong. - // - noway_assert(insertBeforeStmt != impStmtList); + retNode = cnsNode; - insertBeforeStmt = insertBeforeStmt->GetPrevStmt(); + impPopStack(); + impPopStack(); + DEBUG_DESTROY_NODE(otherNode); + + break; } - // Found the call. Move the statements comprising the assignment. - // - JITDUMP("Moving " FMT_STMT "..." FMT_STMT " before " FMT_STMT "\n", cursor->GetNextStmt()->GetID(), - asgStmt->GetID(), insertBeforeStmt->GetID()); - assert(asgStmt == impLastStmt); - do + // only one is constant, we can fold in specialized scenarios + + if (cnsNode->IsFloatNaN()) { - Statement* movingStmt = impExtractLastStmt(); - impInsertStmtBefore(movingStmt, insertBeforeStmt); - insertBeforeStmt = movingStmt; - } while (impLastStmt != cursor); - } - } + impSpillSideEffects(false, (unsigned)CHECK_SPILL_ALL DEBUGARG( + "spill side effects before propagating NaN")); - // Create a pointer to the box payload in op1. - // - op1 = gtNewLclvNode(impBoxTemp, TYP_REF); - op2 = gtNewIconNode(TARGET_POINTER_SIZE, TYP_I_IMPL); - op1 = gtNewOperNode(GT_ADD, TYP_BYREF, op1, op2); + // maxsd, maxss, minsd, and minss all return op2 if either is NaN + // we require NaN to be propagated so ensure the known NaN is op2 - // Copy from the exprToBox to the box payload. - // - if (varTypeIsStruct(exprToBox)) - { - assert(info.compCompHnd->getClassSize(pResolvedToken->hClass) == info.compCompHnd->getClassSize(operCls)); - op1 = impAssignStructPtr(op1, exprToBox, operCls, CHECK_SPILL_ALL); - } - else - { - var_types lclTyp = exprToBox->TypeGet(); - if (lclTyp == TYP_BYREF) - { - lclTyp = TYP_I_IMPL; - } - CorInfoType jitType = info.compCompHnd->asCorInfoType(pResolvedToken->hClass); - if (impIsPrimitive(jitType)) - { - lclTyp = JITtype2varType(jitType); - } + impPopStack(); + impPopStack(); + DEBUG_DESTROY_NODE(otherNode); - var_types srcTyp = exprToBox->TypeGet(); - var_types dstTyp = lclTyp; + retNode = cnsNode; + break; + } - // We allow float <-> double mismatches and implicit truncation for small types. - assert((genActualType(srcTyp) == genActualType(dstTyp)) || - (varTypeIsFloating(srcTyp) == varTypeIsFloating(dstTyp))); + if (ni == NI_System_Math_Max) + { + // maxsd, maxss return op2 if both inputs are 0 of either sign + // we require +0 to be greater than -0, so we can't handle if + // the known constant is +0. This is because if the unknown value + // is -0, we'd need the cns to be op2. But if the unknown value + // is NaN, we'd need the cns to be op1 instead. - // Note regarding small types. - // We are going to store to the box here via an indirection, so the cast added below is - // redundant, since the store has an implicit truncation semantic. The reason we still - // add this cast is so that the code which deals with GT_BOX optimizations does not have - // to account for this implicit truncation (e. g. understand that BOX(0xFF + 1) is - // actually BOX(0) or deal with signedness mismatch and other GT_CAST complexities). - if (srcTyp != dstTyp) - { - exprToBox = gtNewCastNode(genActualType(dstTyp), exprToBox, false, dstTyp); - } + if (cnsNode->IsFloatPositiveZero()) + { + break; + } - op1 = gtNewAssignNode(gtNewOperNode(GT_IND, dstTyp, op1), exprToBox); - } + // Given the checks, op1 can safely be the cns and op2 the other node - // Spill eval stack to flush out any pending side effects. - impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("impImportAndPushBox")); + ni = (callType == TYP_DOUBLE) ? NI_SSE2_Max : NI_SSE_Max; - // Set up this copy as a second assignment. - Statement* copyStmt = impAppendTree(op1, CHECK_SPILL_NONE, impCurStmtDI); + // one is constant and we know its something we can handle, so pop both peeked values - op1 = gtNewLclvNode(impBoxTemp, TYP_REF); + op1 = cnsNode; + op2 = otherNode; + } + else + { + assert(ni == NI_System_Math_Min); - // Record that this is a "box" node and keep track of the matching parts. - op1 = new (this, GT_BOX) GenTreeBox(TYP_REF, op1, asgStmt, copyStmt); + // minsd, minss return op2 if both inputs are 0 of either sign + // we require -0 to be lesser than +0, so we can't handle if + // the known constant is -0. This is because if the unknown value + // is +0, we'd need the cns to be op2. But if the unknown value + // is NaN, we'd need the cns to be op1 instead. - // If it is a value class, mark the "box" node. We can use this information - // to optimise several cases: - // "box(x) == null" --> false - // "(box(x)).CallAnInterfaceMethod(...)" --> "(&x).CallAValueTypeMethod" - // "(box(x)).CallAnObjectMethod(...)" --> "(&x).CallAValueTypeMethod" + if (cnsNode->IsFloatNegativeZero()) + { + break; + } - op1->gtFlags |= GTF_BOX_VALUE; - assert(op1->IsBoxedValue()); - assert(asg->gtOper == GT_ASG); - } - else - { - // Don't optimize, just call the helper and be done with it. - JITDUMP(" helper call because: %s\n", canExpandInline ? "optimizing for size" : "nullable"); - assert(operCls != nullptr); + // Given the checks, op1 can safely be the cns and op2 the other node - // Ensure that the value class is restored - op2 = impTokenToHandle(pResolvedToken, nullptr, true /* mustRestoreHandle */); - if (op2 == nullptr) - { - // We must be backing out of an inline. - assert(compDonotInline()); - return; - } + ni = (callType == TYP_DOUBLE) ? NI_SSE2_Min : NI_SSE_Min; - op1 = gtNewHelperCallNode(boxHelper, TYP_REF, op2, impGetStructAddr(exprToBox, operCls, CHECK_SPILL_ALL, true)); - } + // one is constant and we know its something we can handle, so pop both peeked values - /* Push the result back on the stack, */ - /* even if clsHnd is a value class we want the TI_REF */ - typeInfo tiRetVal = typeInfo(TI_REF, info.compCompHnd->getTypeForBox(pResolvedToken->hClass)); - impPushOnStack(op1, tiRetVal); -} + op1 = cnsNode; + op2 = otherNode; + } -//------------------------------------------------------------------------ -// impImportNewObjArray: Build and import `new` of multi-dimensional array -// -// Arguments: -// pResolvedToken - The CORINFO_RESOLVED_TOKEN that has been initialized -// by a call to CEEInfo::resolveToken(). -// pCallInfo - The CORINFO_CALL_INFO that has been initialized -// by a call to CEEInfo::getCallInfo(). -// -// Assumptions: -// The multi-dimensional array constructor arguments (array dimensions) are -// pushed on the IL stack on entry to this method. -// -// Notes: -// Multi-dimensional array constructors are imported as calls to a JIT -// helper, not as regular calls. -// -void Compiler::impImportNewObjArray(CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_CALL_INFO* pCallInfo) -{ - GenTree* classHandle = impParentClassTokenToHandle(pResolvedToken); - if (classHandle == nullptr) - { // compDonotInline() - return; - } + assert(op1->IsCnsFltOrDbl() && !op2->IsCnsFltOrDbl()); - assert(pCallInfo->sig.numArgs); + impPopStack(); + impPopStack(); - GenTree* node; + GenTreeVecCon* vecCon = gtNewVconNode(TYP_SIMD16, callJitType); - // Reuse the temp used to pass the array dimensions to avoid bloating - // the stack frame in case there are multiple calls to multi-dim array - // constructors within a single method. - if (lvaNewObjArrayArgs == BAD_VAR_NUM) - { - lvaNewObjArrayArgs = lvaGrabTemp(false DEBUGARG("NewObjArrayArgs")); - lvaTable[lvaNewObjArrayArgs].lvType = TYP_BLK; - lvaTable[lvaNewObjArrayArgs].lvExactSize = 0; - } + if (callJitType == CORINFO_TYPE_FLOAT) + { + vecCon->gtSimd16Val.f32[0] = (float)op1->AsDblCon()->gtDconVal; + } + else + { + vecCon->gtSimd16Val.f64[0] = op1->AsDblCon()->gtDconVal; + } - // Increase size of lvaNewObjArrayArgs to be the largest size needed to hold 'numArgs' integers - // for our call to CORINFO_HELP_NEW_MDARR. - lvaTable[lvaNewObjArrayArgs].lvExactSize = - max(lvaTable[lvaNewObjArrayArgs].lvExactSize, pCallInfo->sig.numArgs * sizeof(INT32)); + op1 = vecCon; + op2 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, NI_Vector128_CreateScalarUnsafe, callJitType, 16); - // The side-effects may include allocation of more multi-dimensional arrays. Spill all side-effects - // to ensure that the shared lvaNewObjArrayArgs local variable is only ever used to pass arguments - // to one allocation at a time. - impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("impImportNewObjArray")); + retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, ni, callJitType, 16); + retNode = gtNewSimdHWIntrinsicNode(callType, retNode, NI_Vector128_ToScalar, callJitType, 16); - // - // The arguments of the CORINFO_HELP_NEW_MDARR helper are: - // - Array class handle - // - Number of dimension arguments - // - Pointer to block of int32 dimensions: address of lvaNewObjArrayArgs temp. - // + break; + } +#endif - node = gtNewLclVarAddrNode(lvaNewObjArrayArgs); + case NI_System_Math_Pow: + case NI_System_Math_Round: + case NI_System_Math_Sin: + case NI_System_Math_Sinh: + case NI_System_Math_Sqrt: + case NI_System_Math_Tan: + case NI_System_Math_Tanh: + case NI_System_Math_Truncate: + { + retNode = impMathIntrinsic(method, sig, callType, ni, tailCall); + break; + } - // Pop dimension arguments from the stack one at a time and store it - // into lvaNewObjArrayArgs temp. - for (int i = pCallInfo->sig.numArgs - 1; i >= 0; i--) - { - GenTree* arg = impImplicitIorI4Cast(impPopStack().val, TYP_INT); - GenTree* dest = gtNewLclFldNode(lvaNewObjArrayArgs, TYP_INT, sizeof(INT32) * i); - node = gtNewOperNode(GT_COMMA, node->TypeGet(), gtNewAssignNode(dest, arg), node); - } + case NI_System_Array_Clone: + case NI_System_Collections_Generic_Comparer_get_Default: + case NI_System_Collections_Generic_EqualityComparer_get_Default: + case NI_System_Object_MemberwiseClone: + case NI_System_Threading_Thread_get_CurrentThread: + { + // Flag for later handling. + isSpecial = true; + break; + } - node = - gtNewHelperCallNode(CORINFO_HELP_NEW_MDARR, TYP_REF, classHandle, gtNewIconNode(pCallInfo->sig.numArgs), node); + case NI_System_Object_GetType: + { + JITDUMP("\n impIntrinsic: call to Object.GetType\n"); + GenTree* op1 = impStackTop(0).val; - node->AsCall()->compileTimeHelperArgumentHandle = (CORINFO_GENERIC_HANDLE)pResolvedToken->hClass; + // If we're calling GetType on a boxed value, just get the type directly. + if (op1->IsBoxedValue()) + { + JITDUMP("Attempting to optimize box(...).getType() to direct type construction\n"); - // Remember that this function contains 'new' of a MD array. - optMethodFlags |= OMF_HAS_MDNEWARRAY; + // Try and clean up the box. Obtain the handle we + // were going to pass to the newobj. + GenTree* boxTypeHandle = gtTryRemoveBoxUpstreamEffects(op1, BR_REMOVE_AND_NARROW_WANT_TYPE_HANDLE); - impPushOnStack(node, typeInfo(TI_REF, pResolvedToken->hClass)); -} + if (boxTypeHandle != nullptr) + { + // Note we don't need to play the TYP_STRUCT games here like + // do for LDTOKEN since the return value of this operator is Type, + // not RuntimeTypeHandle. + impPopStack(); + GenTree* runtimeType = + gtNewHelperCallNode(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE, TYP_REF, boxTypeHandle); + retNode = runtimeType; + } + } -//------------------------------------------------------------------------ -// impInitClass: Build a node to initialize the class before accessing the -// field if necessary -// -// Arguments: -// pResolvedToken - The CORINFO_RESOLVED_TOKEN that has been initialized -// by a call to CEEInfo::resolveToken(). -// -// Return Value: If needed, a pointer to the node that will perform the class -// initializtion. Otherwise, nullptr. -// + // If we have a constrained callvirt with a "box this" transform + // we know we have a value class and hence an exact type. + // + // If so, instead of boxing and then extracting the type, just + // construct the type directly. + if ((retNode == nullptr) && (pConstrainedResolvedToken != nullptr) && + (constraintCallThisTransform == CORINFO_BOX_THIS)) + { + // Ensure this is one of the simple box cases (in particular, rule out nullables). + const CorInfoHelpFunc boxHelper = info.compCompHnd->getBoxHelper(pConstrainedResolvedToken->hClass); + const bool isSafeToOptimize = (boxHelper == CORINFO_HELP_BOX); -GenTree* Compiler::impInitClass(CORINFO_RESOLVED_TOKEN* pResolvedToken) -{ - CorInfoInitClassResult initClassResult = - info.compCompHnd->initClass(pResolvedToken->hField, info.compMethodHnd, impTokenLookupContextHandle); + if (isSafeToOptimize) + { + JITDUMP("Optimizing constrained box-this obj.getType() to direct type construction\n"); + impPopStack(); + GenTree* typeHandleOp = + impTokenToHandle(pConstrainedResolvedToken, nullptr, true /* mustRestoreHandle */); + if (typeHandleOp == nullptr) + { + assert(compDonotInline()); + return nullptr; + } + GenTree* runtimeType = + gtNewHelperCallNode(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE, TYP_REF, typeHandleOp); + retNode = runtimeType; + } + } - if ((initClassResult & CORINFO_INITCLASS_USE_HELPER) == 0) - { - return nullptr; - } - bool runtimeLookup; +#ifdef DEBUG + if (retNode != nullptr) + { + JITDUMP("Optimized result for call to GetType is\n"); + if (verbose) + { + gtDispTree(retNode); + } + } +#endif - GenTree* node = impParentClassTokenToHandle(pResolvedToken, &runtimeLookup); + // Else expand as an intrinsic, unless the call is constrained, + // in which case we defer expansion to allow impImportCall do the + // special constraint processing. + if ((retNode == nullptr) && (pConstrainedResolvedToken == nullptr)) + { + JITDUMP("Expanding as special intrinsic\n"); + impPopStack(); + op1 = new (this, GT_INTRINSIC) GenTreeIntrinsic(genActualType(callType), op1, ni, method); - if (node == nullptr) - { - assert(compDonotInline()); - return nullptr; - } + // Set the CALL flag to indicate that the operator is implemented by a call. + // Set also the EXCEPTION flag because the native implementation of + // NI_System_Object_GetType intrinsic can throw NullReferenceException. + op1->gtFlags |= (GTF_CALL | GTF_EXCEPT); + retNode = op1; + // Might be further optimizable, so arrange to leave a mark behind + isSpecial = true; + } - if (runtimeLookup) - { - node = gtNewHelperCallNode(CORINFO_HELP_INITCLASS, TYP_VOID, node); - } - else - { - // Call the shared non gc static helper, as its the fastest - node = fgGetSharedCCtor(pResolvedToken->hClass); - } + if (retNode == nullptr) + { + JITDUMP("Leaving as normal call\n"); + // Might be further optimizable, so arrange to leave a mark behind + isSpecial = true; + } - return node; -} + break; + } -//------------------------------------------------------------------------ -// impImportStaticReadOnlyField: Tries to import 'static readonly' field -// as a constant if the host type is statically initialized. -// -// Arguments: -// field - 'static readonly' field -// ownerCls - class handle of the type the given field defined in -// -// Return Value: -// The tree representing the constant value of the statically initialized -// readonly tree. -// -GenTree* Compiler::impImportStaticReadOnlyField(CORINFO_FIELD_HANDLE field, CORINFO_CLASS_HANDLE ownerCls) -{ - if (!opts.OptimizationEnabled()) - { - return nullptr; - } + case NI_System_Array_GetLength: + case NI_System_Array_GetLowerBound: + case NI_System_Array_GetUpperBound: + { + // System.Array.GetLength(Int32) method: + // public int GetLength(int dimension) + // System.Array.GetLowerBound(Int32) method: + // public int GetLowerBound(int dimension) + // System.Array.GetUpperBound(Int32) method: + // public int GetUpperBound(int dimension) + // + // Only implement these as intrinsics for multi-dimensional arrays. + // Only handle constant dimension arguments. - JITDUMP("\nChecking if we can import 'static readonly' as a jit-time constant... ") + GenTree* gtDim = impStackTop().val; + GenTree* gtArr = impStackTop(1).val; - CORINFO_CLASS_HANDLE fieldClsHnd; - var_types fieldType = JITtype2varType(info.compCompHnd->getFieldType(field, &fieldClsHnd, ownerCls)); + if (gtDim->IsIntegralConst()) + { + bool isExact = false; + bool isNonNull = false; + CORINFO_CLASS_HANDLE arrCls = gtGetClassHandle(gtArr, &isExact, &isNonNull); + if (arrCls != NO_CLASS_HANDLE) + { + unsigned rank = info.compCompHnd->getArrayRank(arrCls); + if ((rank > 1) && !info.compCompHnd->isSDArray(arrCls)) + { + // `rank` is guaranteed to be <=32 (see MAX_RANK in vm\array.h). Any constant argument + // is `int` sized. + INT64 dimValue = gtDim->AsIntConCommon()->IntegralValue(); + assert((unsigned int)dimValue == dimValue); + unsigned dim = (unsigned int)dimValue; + if (dim < rank) + { + // This is now known to be a multi-dimension array with a constant dimension + // that is in range; we can expand it as an intrinsic. + + impPopStack().val; // Pop the dim and array object; we already have a pointer to them. + impPopStack().val; + + // Make sure there are no global effects in the array (such as it being a function + // call), so we can mark the generated indirection with GTF_IND_INVARIANT. In the + // GetUpperBound case we need the cloned object, since we refer to the array + // object twice. In the other cases, we don't need to clone. + GenTree* gtArrClone = nullptr; + if (((gtArr->gtFlags & GTF_GLOB_EFFECT) != 0) || (ni == NI_System_Array_GetUpperBound)) + { + gtArr = impCloneExpr(gtArr, >ArrClone, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, + nullptr DEBUGARG("MD intrinsics array")); + } - const int bufferSize = sizeof(uint64_t); - uint8_t buffer[bufferSize] = {0}; - if (varTypeIsIntegral(fieldType) || varTypeIsFloating(fieldType) || (fieldType == TYP_REF)) - { - assert(bufferSize >= genTypeSize(fieldType)); - if (info.compCompHnd->getReadonlyStaticFieldValue(field, buffer, genTypeSize(fieldType))) - { - GenTree* cnsValue = impImportCnsTreeFromBuffer(buffer, fieldType); - if (cnsValue != nullptr) - { - JITDUMP("... success! The value is:\n"); - DISPTREE(cnsValue); - return cnsValue; + switch (ni) + { + case NI_System_Array_GetLength: + { + // Generate *(array + offset-to-length-array + sizeof(int) * dim) + unsigned offs = eeGetMDArrayLengthOffset(rank, dim); + GenTree* gtOffs = gtNewIconNode(offs, TYP_I_IMPL); + GenTree* gtAddr = gtNewOperNode(GT_ADD, TYP_BYREF, gtArr, gtOffs); + retNode = gtNewIndir(TYP_INT, gtAddr); + retNode->gtFlags |= GTF_IND_INVARIANT; + break; + } + case NI_System_Array_GetLowerBound: + { + // Generate *(array + offset-to-bounds-array + sizeof(int) * dim) + unsigned offs = eeGetMDArrayLowerBoundOffset(rank, dim); + GenTree* gtOffs = gtNewIconNode(offs, TYP_I_IMPL); + GenTree* gtAddr = gtNewOperNode(GT_ADD, TYP_BYREF, gtArr, gtOffs); + retNode = gtNewIndir(TYP_INT, gtAddr); + retNode->gtFlags |= GTF_IND_INVARIANT; + break; + } + case NI_System_Array_GetUpperBound: + { + assert(gtArrClone != nullptr); + + // Generate: + // *(array + offset-to-length-array + sizeof(int) * dim) + + // *(array + offset-to-bounds-array + sizeof(int) * dim) - 1 + unsigned offs = eeGetMDArrayLowerBoundOffset(rank, dim); + GenTree* gtOffs = gtNewIconNode(offs, TYP_I_IMPL); + GenTree* gtAddr = gtNewOperNode(GT_ADD, TYP_BYREF, gtArr, gtOffs); + GenTree* gtLowerBound = gtNewIndir(TYP_INT, gtAddr); + gtLowerBound->gtFlags |= GTF_IND_INVARIANT; + + offs = eeGetMDArrayLengthOffset(rank, dim); + gtOffs = gtNewIconNode(offs, TYP_I_IMPL); + gtAddr = gtNewOperNode(GT_ADD, TYP_BYREF, gtArrClone, gtOffs); + GenTree* gtLength = gtNewIndir(TYP_INT, gtAddr); + gtLength->gtFlags |= GTF_IND_INVARIANT; + + GenTree* gtSum = gtNewOperNode(GT_ADD, TYP_INT, gtLowerBound, gtLength); + GenTree* gtOne = gtNewIconNode(1, TYP_INT); + retNode = gtNewOperNode(GT_SUB, TYP_INT, gtSum, gtOne); + break; + } + default: + unreached(); + } + } + } + } + } + break; } - } - } - else if (fieldType == TYP_STRUCT) - { - unsigned totalSize = info.compCompHnd->getClassSize(fieldClsHnd); - unsigned fieldsCnt = info.compCompHnd->getClassNumInstanceFields(fieldClsHnd); - // For large structs we only want to handle "initialized with zero" case - // e.g. Guid.Empty and decimal.Zero static readonly fields. - if ((totalSize > TARGET_POINTER_SIZE) || (fieldsCnt != 1)) - { - JITDUMP("checking if we can do anything for a large struct ..."); - const int MaxStructSize = 64; - if ((totalSize == 0) || (totalSize > MaxStructSize)) + case NI_System_Buffers_Binary_BinaryPrimitives_ReverseEndianness: { - // Limit to 64 bytes for better throughput - JITDUMP("struct is larger than 64 bytes - bail out."); - return nullptr; + assert(sig->numArgs == 1); + + // We expect the return type of the ReverseEndianness routine to match the type of the + // one and only argument to the method. We use a special instruction for 16-bit + // BSWAPs since on x86 processors this is implemented as ROR <16-bit reg>, 8. Additionally, + // we only emit 64-bit BSWAP instructions on 64-bit archs; if we're asked to perform a + // 64-bit byte swap on a 32-bit arch, we'll fall to the default case in the switch block below. + + switch (sig->retType) + { + case CorInfoType::CORINFO_TYPE_SHORT: + case CorInfoType::CORINFO_TYPE_USHORT: + retNode = gtNewCastNode(TYP_INT, gtNewOperNode(GT_BSWAP16, TYP_INT, impPopStack().val), false, + callType); + break; + + case CorInfoType::CORINFO_TYPE_INT: + case CorInfoType::CORINFO_TYPE_UINT: +#ifdef TARGET_64BIT + case CorInfoType::CORINFO_TYPE_LONG: + case CorInfoType::CORINFO_TYPE_ULONG: +#endif // TARGET_64BIT + retNode = gtNewOperNode(GT_BSWAP, callType, impPopStack().val); + break; + + default: + // This default case gets hit on 32-bit archs when a call to a 64-bit overload + // of ReverseEndianness is encountered. In that case we'll let JIT treat this as a standard + // method call, where the implementation decomposes the operation into two 32-bit + // bswap routines. If the input to the 64-bit function is a constant, then we rely + // on inlining + constant folding of 32-bit bswaps to effectively constant fold + // the 64-bit call site. + break; + } + + break; } - uint8_t buffer[MaxStructSize] = {0}; - if (info.compCompHnd->getReadonlyStaticFieldValue(field, buffer, totalSize)) + // Fold PopCount for constant input + case NI_System_Numerics_BitOperations_PopCount: { - for (unsigned i = 0; i < totalSize; i++) + assert(sig->numArgs == 1); + if (impStackTop().val->IsIntegralConst()) { - if (buffer[i] != 0) + typeInfo argType = verParseArgSigToTypeInfo(sig, sig->args).NormaliseForStack(); + INT64 cns = impPopStack().val->AsIntConCommon()->IntegralValue(); + if (argType.IsType(TI_LONG)) + { + retNode = gtNewIconNode(genCountBits(cns), callType); + } + else { - // Value is not all zeroes - bail out. - // Although, We might eventually support that too. - JITDUMP("value is not all zeros - bail out."); - return nullptr; + assert(argType.IsType(TI_INT)); + retNode = gtNewIconNode(genCountBits(static_cast(cns)), callType); } } + break; + } - JITDUMP("success! Optimizing to ASG(struct, 0)."); - unsigned structTempNum = lvaGrabTemp(true DEBUGARG("folding static ro fld empty struct")); - lvaSetStruct(structTempNum, fieldClsHnd, false); + case NI_System_GC_KeepAlive: + { + retNode = impKeepAliveIntrinsic(impPopStack().val); + break; + } - // realType is either struct or SIMD - var_types realType = lvaGetRealType(structTempNum); - GenTreeLclVar* structLcl = gtNewLclvNode(structTempNum, realType); - impAppendTree(gtNewBlkOpNode(structLcl, gtNewIconNode(0)), CHECK_SPILL_NONE, impCurStmtDI); + case NI_System_BitConverter_DoubleToInt64Bits: + { + GenTree* op1 = impStackTop().val; + assert(varTypeIsFloating(op1)); - return gtNewLclvNode(structTempNum, realType); - } + if (op1->IsCnsFltOrDbl()) + { + impPopStack(); - JITDUMP("getReadonlyStaticFieldValue returned false - bail out."); - return nullptr; - } + double f64Cns = op1->AsDblCon()->gtDconVal; + retNode = gtNewLconNode(*reinterpret_cast(&f64Cns)); + } +#if TARGET_64BIT + else + { + // TODO-Cleanup: We should support this on 32-bit but it requires decomposition work + impPopStack(); - // Only single-field structs are supported here to avoid potential regressions where - // Metadata-driven struct promotion leads to regressions. + if (op1->TypeGet() != TYP_DOUBLE) + { + op1 = gtNewCastNode(TYP_DOUBLE, op1, false, TYP_DOUBLE); + } + retNode = gtNewBitCastNode(TYP_LONG, op1); + } +#endif + break; + } - CORINFO_FIELD_HANDLE innerField = info.compCompHnd->getFieldInClass(fieldClsHnd, 0); - CORINFO_CLASS_HANDLE innerFieldClsHnd; - var_types fieldVarType = - JITtype2varType(info.compCompHnd->getFieldType(innerField, &innerFieldClsHnd, fieldClsHnd)); + case NI_System_BitConverter_Int32BitsToSingle: + { + GenTree* op1 = impPopStack().val; + assert(varTypeIsInt(op1)); - // Technically, we can support frozen gc refs here and maybe floating point in future - if (!varTypeIsIntegral(fieldVarType)) - { - JITDUMP("struct has non-primitive fields - bail out."); - return nullptr; - } + if (op1->IsIntegralConst()) + { + int32_t i32Cns = (int32_t)op1->AsIntConCommon()->IconValue(); + retNode = gtNewDconNode(*reinterpret_cast(&i32Cns), TYP_FLOAT); + } + else + { + retNode = gtNewBitCastNode(TYP_FLOAT, op1); + } + break; + } - unsigned fldOffset = info.compCompHnd->getFieldOffset(innerField); + case NI_System_BitConverter_Int64BitsToDouble: + { + GenTree* op1 = impStackTop().val; + assert(varTypeIsLong(op1)); - if ((fldOffset != 0) || (totalSize != genTypeSize(fieldVarType)) || (totalSize == 0)) - { - // The field is expected to be of the exact size as the struct with 0 offset - JITDUMP("struct has complex layout - bail out."); - return nullptr; - } + if (op1->IsIntegralConst()) + { + impPopStack(); - const int bufferSize = TARGET_POINTER_SIZE; - uint8_t buffer[bufferSize] = {0}; + int64_t i64Cns = op1->AsIntConCommon()->LngValue(); + retNode = gtNewDconNode(*reinterpret_cast(&i64Cns)); + } +#if TARGET_64BIT + else + { + // TODO-Cleanup: We should support this on 32-bit but it requires decomposition work + impPopStack(); - if ((totalSize > bufferSize) || !info.compCompHnd->getReadonlyStaticFieldValue(field, buffer, totalSize)) - { - return nullptr; - } + retNode = gtNewBitCastNode(TYP_DOUBLE, op1); + } +#endif + break; + } - unsigned structTempNum = lvaGrabTemp(true DEBUGARG("folding static ro fld struct")); - lvaSetStruct(structTempNum, fieldClsHnd, false); + case NI_System_BitConverter_SingleToInt32Bits: + { + GenTree* op1 = impPopStack().val; + assert(varTypeIsFloating(op1)); - GenTree* constValTree = impImportCnsTreeFromBuffer(buffer, fieldVarType); - assert(constValTree != nullptr); + if (op1->IsCnsFltOrDbl()) + { + float f32Cns = (float)op1->AsDblCon()->gtDconVal; + retNode = gtNewIconNode(*reinterpret_cast(&f32Cns)); + } + else + { + if (op1->TypeGet() != TYP_FLOAT) + { + op1 = gtNewCastNode(TYP_FLOAT, op1, false, TYP_FLOAT); + } + retNode = gtNewBitCastNode(TYP_INT, op1); + } + break; + } - GenTreeLclFld* fieldTree = gtNewLclFldNode(structTempNum, fieldVarType, fldOffset); - GenTree* fieldAsgTree = gtNewAssignNode(fieldTree, constValTree); - impAppendTree(fieldAsgTree, CHECK_SPILL_NONE, impCurStmtDI); + default: + break; + } + } - JITDUMP("Folding 'static readonly %s' field to an ASG(LCL, CNS) node\n", eeGetClassName(fieldClsHnd)); + if (mustExpand && (retNode == nullptr)) + { + assert(!"Unhandled must expand intrinsic, throwing PlatformNotSupportedException"); + return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand); + } - return impCreateLocalNode(structTempNum DEBUGARG(0)); + // Optionally report if this intrinsic is special + // (that is, potentially re-optimizable during morph). + if (isSpecialIntrinsic != nullptr) + { + *isSpecialIntrinsic = isSpecial; } - return nullptr; + + return retNode; } -//------------------------------------------------------------------------ -// impImportCnsTreeFromBuffer: read value of the given type from the -// given buffer to create a tree representing that constant. -// -// Arguments: -// buffer - array of bytes representing the value -// valueType - type of the value -// -// Return Value: -// The tree representing the constant from the given buffer -// -GenTree* Compiler::impImportCnsTreeFromBuffer(uint8_t* buffer, var_types valueType) +GenTree* Compiler::impSRCSUnsafeIntrinsic(NamedIntrinsic intrinsic, + CORINFO_CLASS_HANDLE clsHnd, + CORINFO_METHOD_HANDLE method, + CORINFO_SIG_INFO* sig) { - GenTree* tree = nullptr; - switch (valueType) - { -// Use memcpy to read from the buffer and create an Icon/Dcon tree -#define CreateTreeFromBuffer(type, treeFactory) \ - type v##type; \ - memcpy(&v##type, buffer, sizeof(type)); \ - tree = treeFactory(v##type); + assert(sig->sigInst.classInstCount == 0); - case TYP_BOOL: + switch (intrinsic) + { + case NI_SRCS_UNSAFE_Add: { - CreateTreeFromBuffer(bool, gtNewIconNode); - break; + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // ldarg.1 + // sizeof !!T + // conv.i + // mul + // add + // ret + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + impBashVarAddrsToI(op1, op2); + + op2 = impImplicitIorI4Cast(op2, TYP_I_IMPL); + + unsigned classSize = info.compCompHnd->getClassSize(sig->sigInst.methInst[0]); + + if (classSize != 1) + { + GenTree* size = gtNewIconNode(classSize, TYP_I_IMPL); + op2 = gtNewOperNode(GT_MUL, TYP_I_IMPL, op2, size); + } + + var_types type = impGetByRefResultType(GT_ADD, /* uns */ false, &op1, &op2); + return gtNewOperNode(GT_ADD, type, op1, op2); } - case TYP_BYTE: + + case NI_SRCS_UNSAFE_AddByteOffset: { - CreateTreeFromBuffer(int8_t, gtNewIconNode); - break; - } - case TYP_UBYTE: + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // ldarg.1 + // add + // ret + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + impBashVarAddrsToI(op1, op2); + + var_types type = impGetByRefResultType(GT_ADD, /* uns */ false, &op1, &op2); + return gtNewOperNode(GT_ADD, type, op1, op2); + } + + case NI_SRCS_UNSAFE_AreSame: { - CreateTreeFromBuffer(uint8_t, gtNewIconNode); - break; + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // ldarg.1 + // ceq + // ret + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + + GenTree* tmp = gtNewOperNode(GT_EQ, TYP_INT, op1, op2); + return gtFoldExpr(tmp); } - case TYP_SHORT: + + case NI_SRCS_UNSAFE_As: { - CreateTreeFromBuffer(int16_t, gtNewIconNode); - break; + assert((sig->sigInst.methInstCount == 1) || (sig->sigInst.methInstCount == 2)); + + // ldarg.0 + // ret + + return impPopStack().val; } - case TYP_USHORT: + + case NI_SRCS_UNSAFE_AsPointer: { - CreateTreeFromBuffer(uint16_t, gtNewIconNode); - break; + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // conv.u + // ret + + GenTree* op1 = impPopStack().val; + impBashVarAddrsToI(op1); + + return gtNewCastNode(TYP_I_IMPL, op1, /* uns */ false, TYP_I_IMPL); } - case TYP_UINT: - case TYP_INT: + + case NI_SRCS_UNSAFE_AsRef: { - CreateTreeFromBuffer(int32_t, gtNewIconNode); - break; + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // ret + + return impPopStack().val; } - case TYP_LONG: - case TYP_ULONG: + + case NI_SRCS_UNSAFE_ByteOffset: { - CreateTreeFromBuffer(int64_t, gtNewLconNode); - break; + assert(sig->sigInst.methInstCount == 1); + + // ldarg.1 + // ldarg.0 + // sub + // ret + + impSpillSideEffect(true, verCurrentState.esStackDepth - + 2 DEBUGARG("Spilling op1 side effects for Unsafe.ByteOffset")); + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + impBashVarAddrsToI(op1, op2); + + var_types type = impGetByRefResultType(GT_SUB, /* uns */ false, &op2, &op1); + return gtNewOperNode(GT_SUB, type, op2, op1); } - case TYP_FLOAT: + + case NI_SRCS_UNSAFE_Copy: { - CreateTreeFromBuffer(float, gtNewDconNode); - break; + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // ldarg.1 + // ldobj !!T + // stobj !!T + // ret + + return nullptr; } - case TYP_DOUBLE: + + case NI_SRCS_UNSAFE_CopyBlock: { - CreateTreeFromBuffer(double, gtNewDconNode); - break; + assert(sig->sigInst.methInstCount == 0); + + // ldarg.0 + // ldarg.1 + // ldarg.2 + // cpblk + // ret + + return nullptr; } - case TYP_REF: + + case NI_SRCS_UNSAFE_CopyBlockUnaligned: { - size_t ptr; - memcpy(&ptr, buffer, sizeof(ssize_t)); + assert(sig->sigInst.methInstCount == 0); - if (ptr == 0) - { - tree = gtNewNull(); - } - else - { - setMethodHasFrozenObjects(); - tree = gtNewIconEmbHndNode((void*)ptr, nullptr, GTF_ICON_OBJ_HDL, nullptr); - tree->gtType = TYP_REF; - INDEBUG(tree->AsIntCon()->gtTargetHandle = ptr); - } - break; + // ldarg.0 + // ldarg.1 + // ldarg.2 + // unaligned. 0x1 + // cpblk + // ret + + return nullptr; } - default: + + case NI_SRCS_UNSAFE_InitBlock: + { + assert(sig->sigInst.methInstCount == 0); + + // ldarg.0 + // ldarg.1 + // ldarg.2 + // initblk + // ret + return nullptr; - } + } - assert(tree != nullptr); - tree->gtType = genActualType(valueType); - return tree; -} + case NI_SRCS_UNSAFE_InitBlockUnaligned: + { + assert(sig->sigInst.methInstCount == 0); -GenTree* Compiler::impImportStaticFieldAccess(CORINFO_RESOLVED_TOKEN* pResolvedToken, - CORINFO_ACCESS_FLAGS access, - CORINFO_FIELD_INFO* pFieldInfo, - var_types lclTyp) -{ - // Ordinary static fields never overlap. RVA statics, however, can overlap (if they're - // mapped to the same ".data" declaration). That said, such mappings only appear to be - // possible with ILASM, and in ILASM-produced (ILONLY) images, RVA statics are always - // read-only (using "stsfld" on them is UB). In mixed-mode assemblies, RVA statics can - // be mutable, but the only current producer of such images, the C++/CLI compiler, does - // not appear to support mapping different fields to the same address. So we will say - // that "mutable overlapping RVA statics" are UB as well. + // ldarg.0 + // ldarg.1 + // ldarg.2 + // unaligned. 0x1 + // initblk + // ret - // For statics that are not "boxed", the initial address tree will contain the field sequence. - // For those that are, we will attach it later, when adding the indirection for the box, since - // that tree will represent the true address. - bool isBoxedStatic = (pFieldInfo->fieldFlags & CORINFO_FLG_FIELD_STATIC_IN_HEAP) != 0; - bool isSharedStatic = (pFieldInfo->fieldAccessor == CORINFO_FIELD_STATIC_GENERICS_STATIC_HELPER) || - (pFieldInfo->fieldAccessor == CORINFO_FIELD_STATIC_READYTORUN_HELPER); - FieldSeq::FieldKind fieldKind = - isSharedStatic ? FieldSeq::FieldKind::SharedStatic : FieldSeq::FieldKind::SimpleStatic; + return nullptr; + } - bool hasConstAddr = (pFieldInfo->fieldAccessor == CORINFO_FIELD_STATIC_ADDRESS) || - (pFieldInfo->fieldAccessor == CORINFO_FIELD_STATIC_RVA_ADDRESS); + case NI_SRCS_UNSAFE_IsAddressGreaterThan: + { + assert(sig->sigInst.methInstCount == 1); - FieldSeq* innerFldSeq; - FieldSeq* outerFldSeq; - if (isBoxedStatic) - { - innerFldSeq = nullptr; - outerFldSeq = GetFieldSeqStore()->Create(pResolvedToken->hField, TARGET_POINTER_SIZE, fieldKind); - } - else - { - ssize_t offset; - if (hasConstAddr) + // ldarg.0 + // ldarg.1 + // cgt.un + // ret + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + + GenTree* tmp = gtNewOperNode(GT_GT, TYP_INT, op1, op2); + tmp->gtFlags |= GTF_UNSIGNED; + return gtFoldExpr(tmp); + } + + case NI_SRCS_UNSAFE_IsAddressLessThan: { - // Change SimpleStatic to SimpleStaticKnownAddress - assert(fieldKind == FieldSeq::FieldKind::SimpleStatic); - fieldKind = FieldSeq::FieldKind::SimpleStaticKnownAddress; + assert(sig->sigInst.methInstCount == 1); - assert(pFieldInfo->fieldLookup.accessType == IAT_VALUE); - offset = reinterpret_cast(pFieldInfo->fieldLookup.addr); + // ldarg.0 + // ldarg.1 + // clt.un + // ret + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + + GenTree* tmp = gtNewOperNode(GT_LT, TYP_INT, op1, op2); + tmp->gtFlags |= GTF_UNSIGNED; + return gtFoldExpr(tmp); } - else + + case NI_SRCS_UNSAFE_IsNullRef: { - offset = pFieldInfo->offset; + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // ldc.i4.0 + // conv.u + // ceq + // ret + + GenTree* op1 = impPopStack().val; + GenTree* cns = gtNewIconNode(0, TYP_BYREF); + GenTree* tmp = gtNewOperNode(GT_EQ, TYP_INT, op1, cns); + return gtFoldExpr(tmp); } - innerFldSeq = GetFieldSeqStore()->Create(pResolvedToken->hField, offset, fieldKind); - outerFldSeq = nullptr; - } + case NI_SRCS_UNSAFE_NullRef: + { + assert(sig->sigInst.methInstCount == 1); - bool isStaticReadOnlyInitedRef = false; - GenTree* op1; - switch (pFieldInfo->fieldAccessor) - { - case CORINFO_FIELD_STATIC_GENERICS_STATIC_HELPER: + // ldc.i4.0 + // conv.u + // ret + + return gtNewIconNode(0, TYP_BYREF); + } + + case NI_SRCS_UNSAFE_Read: { - assert(!compIsForInlining()); + assert(sig->sigInst.methInstCount == 1); - // We first call a special helper to get the statics base pointer - op1 = impParentClassTokenToHandle(pResolvedToken); + // ldarg.0 + // ldobj !!T + // ret - // compIsForInlining() is false so we should not get NULL here - assert(op1 != nullptr); + return nullptr; + } - var_types type = TYP_BYREF; + case NI_SRCS_UNSAFE_ReadUnaligned: + { + assert(sig->sigInst.methInstCount == 1); - switch (pFieldInfo->helper) - { - case CORINFO_HELP_GETGENERICS_NONGCTHREADSTATIC_BASE: - type = TYP_I_IMPL; - break; - case CORINFO_HELP_GETGENERICS_GCSTATIC_BASE: - case CORINFO_HELP_GETGENERICS_NONGCSTATIC_BASE: - case CORINFO_HELP_GETGENERICS_GCTHREADSTATIC_BASE: - break; - default: - assert(!"unknown generic statics helper"); - break; - } + // ldarg.0 + // unaligned. 0x1 + // ldobj !!T + // ret - op1 = gtNewHelperCallNode(pFieldInfo->helper, type, op1); - op1 = gtNewOperNode(GT_ADD, type, op1, gtNewIconNode(pFieldInfo->offset, innerFldSeq)); + return nullptr; } - break; - case CORINFO_FIELD_STATIC_SHARED_STATIC_HELPER: + case NI_SRCS_UNSAFE_SizeOf: { -#ifdef FEATURE_READYTORUN - if (opts.IsReadyToRun()) - { - GenTreeFlags callFlags = GTF_EMPTY; + assert(sig->sigInst.methInstCount == 1); - if (info.compCompHnd->getClassAttribs(pResolvedToken->hClass) & CORINFO_FLG_BEFOREFIELDINIT) - { - callFlags |= GTF_CALL_HOISTABLE; - } + // sizeof !!T + // ret - op1 = gtNewHelperCallNode(pFieldInfo->helper, TYP_BYREF); - if (pResolvedToken->hClass == info.compClassHnd && m_preferredInitCctor == CORINFO_HELP_UNDEF && - (pFieldInfo->helper == CORINFO_HELP_READYTORUN_GCSTATIC_BASE || - pFieldInfo->helper == CORINFO_HELP_READYTORUN_NONGCSTATIC_BASE)) - { - m_preferredInitCctor = pFieldInfo->helper; - } - op1->gtFlags |= callFlags; + unsigned classSize = info.compCompHnd->getClassSize(sig->sigInst.methInst[0]); + return gtNewIconNode(classSize, TYP_INT); + } - op1->AsCall()->setEntryPoint(pFieldInfo->fieldLookup); + case NI_SRCS_UNSAFE_SkipInit: + { + assert(sig->sigInst.methInstCount == 1); + + // ret + + GenTree* op1 = impPopStack().val; + + if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0) + { + return gtUnusedValNode(op1); } else -#endif { - op1 = fgGetStaticsCCtorHelper(pResolvedToken->hClass, pFieldInfo->helper); + return gtNewNothingNode(); } - - op1 = gtNewOperNode(GT_ADD, op1->TypeGet(), op1, gtNewIconNode(pFieldInfo->offset, innerFldSeq)); - break; } - case CORINFO_FIELD_STATIC_READYTORUN_HELPER: + case NI_SRCS_UNSAFE_Subtract: { -#ifdef FEATURE_READYTORUN - assert(opts.IsReadyToRun()); - assert(!compIsForInlining()); - CORINFO_LOOKUP_KIND kind; - info.compCompHnd->getLocationOfThisType(info.compMethodHnd, &kind); - assert(kind.needsRuntimeLookup); + assert(sig->sigInst.methInstCount == 1); - GenTree* ctxTree = getRuntimeContextTree(kind.runtimeLookupKind); + // ldarg.0 + // ldarg.1 + // sizeof !!T + // conv.i + // mul + // sub + // ret - GenTreeFlags callFlags = GTF_EMPTY; + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + impBashVarAddrsToI(op1, op2); - if (info.compCompHnd->getClassAttribs(pResolvedToken->hClass) & CORINFO_FLG_BEFOREFIELDINIT) + op2 = impImplicitIorI4Cast(op2, TYP_I_IMPL); + + unsigned classSize = info.compCompHnd->getClassSize(sig->sigInst.methInst[0]); + + if (classSize != 1) { - callFlags |= GTF_CALL_HOISTABLE; + GenTree* size = gtNewIconNode(classSize, TYP_I_IMPL); + op2 = gtNewOperNode(GT_MUL, TYP_I_IMPL, op2, size); } - var_types type = TYP_BYREF; - op1 = gtNewHelperCallNode(CORINFO_HELP_READYTORUN_GENERIC_STATIC_BASE, type, ctxTree); - op1->gtFlags |= callFlags; - op1->AsCall()->setEntryPoint(pFieldInfo->fieldLookup); - op1 = gtNewOperNode(GT_ADD, type, op1, gtNewIconNode(pFieldInfo->offset, innerFldSeq)); -#else - unreached(); -#endif // FEATURE_READYTORUN + var_types type = impGetByRefResultType(GT_SUB, /* uns */ false, &op1, &op2); + return gtNewOperNode(GT_SUB, type, op1, op2); } - break; - default: + case NI_SRCS_UNSAFE_SubtractByteOffset: { -// TODO-CQ: enable this optimization for 32 bit targets. -#ifdef TARGET_64BIT - if (!isBoxedStatic && (lclTyp == TYP_REF) && ((access & CORINFO_ACCESS_ADDRESS) == 0)) - { - bool isSpeculative = true; - if ((info.compCompHnd->getStaticFieldCurrentClass(pResolvedToken->hField, &isSpeculative) != - NO_CLASS_HANDLE)) - { - isStaticReadOnlyInitedRef = !isSpeculative; - } - } -#endif // TARGET_64BIT + assert(sig->sigInst.methInstCount == 1); - assert(hasConstAddr); - assert(pFieldInfo->fieldLookup.addr != nullptr); - assert(pFieldInfo->fieldLookup.accessType == IAT_VALUE); - size_t fldAddr = reinterpret_cast(pFieldInfo->fieldLookup.addr); - GenTreeFlags handleKind; - if (isBoxedStatic) - { - handleKind = GTF_ICON_STATIC_BOX_PTR; - } - else if (isStaticReadOnlyInitedRef) - { - handleKind = GTF_ICON_CONST_PTR; - } - else - { - handleKind = GTF_ICON_STATIC_HDL; - } - op1 = gtNewIconHandleNode(fldAddr, handleKind, innerFldSeq); - INDEBUG(op1->AsIntCon()->gtTargetHandle = reinterpret_cast(pResolvedToken->hField)); - if (pFieldInfo->fieldFlags & CORINFO_FLG_FIELD_INITCLASS) - { - op1->gtFlags |= GTF_ICON_INITCLASS; - } - break; + // ldarg.0 + // ldarg.1 + // sub + // ret + + GenTree* op2 = impPopStack().val; + GenTree* op1 = impPopStack().val; + impBashVarAddrsToI(op1, op2); + + var_types type = impGetByRefResultType(GT_SUB, /* uns */ false, &op1, &op2); + return gtNewOperNode(GT_SUB, type, op1, op2); } - } - if (isBoxedStatic) - { - op1 = gtNewIndir(TYP_REF, op1, GTF_IND_INVARIANT | GTF_IND_NONFAULTING | GTF_IND_NONNULL); - op1 = gtNewOperNode(GT_ADD, TYP_BYREF, op1, gtNewIconNode(TARGET_POINTER_SIZE, outerFldSeq)); - } + case NI_SRCS_UNSAFE_Unbox: + { + assert(sig->sigInst.methInstCount == 1); - if (!(access & CORINFO_ACCESS_ADDRESS)) - { - if (varTypeIsStruct(lclTyp)) + // ldarg.0 + // unbox !!T + // ret + + return nullptr; + } + + case NI_SRCS_UNSAFE_Write: { - // Constructor adds GTF_GLOB_REF. Note that this is *not* GTF_EXCEPT. - op1 = gtNewObjNode(pFieldInfo->structType, op1); + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // ldarg.1 + // stobj !!T + // ret + + return nullptr; } - else + + case NI_SRCS_UNSAFE_WriteUnaligned: { - op1 = gtNewOperNode(GT_IND, lclTyp, op1); - op1->gtFlags |= GTF_GLOB_REF; + assert(sig->sigInst.methInstCount == 1); + + // ldarg.0 + // ldarg.1 + // unaligned. 0x01 + // stobj !!T + // ret + + return nullptr; } - if (isStaticReadOnlyInitedRef) + + default: { - op1->gtFlags |= (GTF_IND_INVARIANT | GTF_IND_NONFAULTING | GTF_IND_NONNULL); + unreached(); } } - - return op1; } -// In general try to call this before most of the verification work. Most people expect the access -// exceptions before the verification exceptions. If you do this after, that usually doesn't happen. Turns -// out if you can't access something we also think that you're unverifiable for other reasons. -void Compiler::impHandleAccessAllowed(CorInfoIsAccessAllowedResult result, CORINFO_HELPER_DESC* helperCall) +GenTree* Compiler::impTypeIsAssignable(GenTree* typeTo, GenTree* typeFrom) { - if (result != CORINFO_ACCESS_ALLOWED) + // Optimize patterns like: + // + // typeof(TTo).IsAssignableFrom(typeof(TTFrom)) + // valueTypeVar.GetType().IsAssignableFrom(typeof(TTFrom)) + // typeof(TTFrom).IsAssignableTo(typeof(TTo)) + // typeof(TTFrom).IsAssignableTo(valueTypeVar.GetType()) + // + // to true/false + + if (typeTo->IsCall() && typeFrom->IsCall()) { - impHandleAccessAllowedInternal(result, helperCall); - } -} + // make sure both arguments are `typeof()` + CORINFO_METHOD_HANDLE hTypeof = eeFindHelper(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE); + if ((typeTo->AsCall()->gtCallMethHnd == hTypeof) && (typeFrom->AsCall()->gtCallMethHnd == hTypeof)) + { + assert((typeTo->AsCall()->gtArgs.CountArgs() == 1) && (typeFrom->AsCall()->gtArgs.CountArgs() == 1)); + CORINFO_CLASS_HANDLE hClassTo = + gtGetHelperArgClassHandle(typeTo->AsCall()->gtArgs.GetArgByIndex(0)->GetEarlyNode()); + CORINFO_CLASS_HANDLE hClassFrom = + gtGetHelperArgClassHandle(typeFrom->AsCall()->gtArgs.GetArgByIndex(0)->GetEarlyNode()); -void Compiler::impHandleAccessAllowedInternal(CorInfoIsAccessAllowedResult result, CORINFO_HELPER_DESC* helperCall) -{ - switch (result) - { - case CORINFO_ACCESS_ALLOWED: - break; - case CORINFO_ACCESS_ILLEGAL: - // if we're verifying, then we need to reject the illegal access to ensure that we don't think the - // method is verifiable. Otherwise, delay the exception to runtime. - if (compIsForImportOnly()) + if (hClassTo == NO_CLASS_HANDLE || hClassFrom == NO_CLASS_HANDLE) { - info.compCompHnd->ThrowExceptionForHelper(helperCall); + return nullptr; } - else + + TypeCompareState castResult = info.compCompHnd->compareTypesForCast(hClassFrom, hClassTo); + if (castResult == TypeCompareState::May) { - impInsertHelperCall(helperCall); + // requires runtime check + // e.g. __Canon, COMObjects, Nullable + return nullptr; } - break; - } -} -void Compiler::impInsertHelperCall(CORINFO_HELPER_DESC* helperInfo) -{ - assert(helperInfo->helperNum != CORINFO_HELP_UNDEF); + GenTreeIntCon* retNode = gtNewIconNode((castResult == TypeCompareState::Must) ? 1 : 0); + impPopStack(); // drop both CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE calls + impPopStack(); - /* TODO-Review: - * Mark as CSE'able, and hoistable. Consider marking hoistable unless you're in the inlinee. - * Also, consider sticking this in the first basic block. - */ - GenTreeCall* callout = gtNewHelperCallNode(helperInfo->helperNum, TYP_VOID); - // Add the arguments - for (unsigned i = helperInfo->numArgs; i > 0; --i) - { - const CORINFO_HELPER_ARG& helperArg = helperInfo->args[i - 1]; - GenTree* currentArg = nullptr; - switch (helperArg.argType) - { - case CORINFO_HELPER_ARG_TYPE_Field: - info.compCompHnd->classMustBeLoadedBeforeCodeIsRun( - info.compCompHnd->getFieldClass(helperArg.fieldHandle)); - currentArg = gtNewIconEmbFldHndNode(helperArg.fieldHandle); - break; - case CORINFO_HELPER_ARG_TYPE_Method: - info.compCompHnd->methodMustBeLoadedBeforeCodeIsRun(helperArg.methodHandle); - currentArg = gtNewIconEmbMethHndNode(helperArg.methodHandle); - break; - case CORINFO_HELPER_ARG_TYPE_Class: - info.compCompHnd->classMustBeLoadedBeforeCodeIsRun(helperArg.classHandle); - currentArg = gtNewIconEmbClsHndNode(helperArg.classHandle); - break; - case CORINFO_HELPER_ARG_TYPE_Module: - currentArg = gtNewIconEmbScpHndNode(helperArg.moduleHandle); - break; - case CORINFO_HELPER_ARG_TYPE_Const: - currentArg = gtNewIconNode(helperArg.constant); - break; - default: - NO_WAY("Illegal helper arg type"); + return retNode; } - callout->gtArgs.PushFront(this, NewCallArg::Primitive(currentArg)); } - impAppendTree(callout, CHECK_SPILL_NONE, impCurStmtDI); + return nullptr; } -/******************************************************************************** - * - * Returns true if the current opcode and and the opcodes following it correspond - * to a supported tail call IL pattern. - * - */ -bool Compiler::impIsTailCallILPattern( - bool tailPrefixed, OPCODE curOpcode, const BYTE* codeAddrOfNextOpcode, const BYTE* codeEnd, bool isRecursive) +GenTree* Compiler::impMathIntrinsic(CORINFO_METHOD_HANDLE method, + CORINFO_SIG_INFO* sig, + var_types callType, + NamedIntrinsic intrinsicName, + bool tailCall) { - // Bail out if the current opcode is not a call. - if (!impOpcodeIsCallOpcode(curOpcode)) - { - return false; - } + GenTree* op1; + GenTree* op2; -#if !FEATURE_TAILCALL_OPT_SHARED_RETURN - // If shared ret tail opt is not enabled, we will enable - // it for recursive methods. - if (isRecursive) + assert(callType != TYP_STRUCT); + assert(IsMathIntrinsic(intrinsicName)); + + op1 = nullptr; + +#if !defined(TARGET_X86) + // Intrinsics that are not implemented directly by target instructions will + // be re-materialized as users calls in rationalizer. For prefixed tail calls, + // don't do this optimization, because + // a) For back compatibility reasons on desktop .NET Framework 4.6 / 4.6.1 + // b) It will be non-trivial task or too late to re-materialize a surviving + // tail prefixed GT_INTRINSIC as tail call in rationalizer. + if (!IsIntrinsicImplementedByUserCall(intrinsicName) || !tailCall) +#else + // On x86 RyuJIT, importing intrinsics that are implemented as user calls can cause incorrect calculation + // of the depth of the stack if these intrinsics are used as arguments to another call. This causes bad + // code generation for certain EH constructs. + if (!IsIntrinsicImplementedByUserCall(intrinsicName)) #endif { - // we can actually handle if the ret is in a fallthrough block, as long as that is the only part of the - // sequence. Make sure we don't go past the end of the IL however. - codeEnd = min(codeEnd + 1, info.compCode + info.compILCodeSize); - } + CORINFO_CLASS_HANDLE tmpClass; + CORINFO_ARG_LIST_HANDLE arg; + var_types op1Type; + var_types op2Type; - // Bail out if there is no next opcode after call - if (codeAddrOfNextOpcode >= codeEnd) - { - return false; - } + switch (sig->numArgs) + { + case 1: + op1 = impPopStack().val; - OPCODE nextOpcode = (OPCODE)getU1LittleEndian(codeAddrOfNextOpcode); + arg = sig->args; + op1Type = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg, &tmpClass))); - return (nextOpcode == CEE_RET); -} + if (op1->TypeGet() != genActualType(op1Type)) + { + assert(varTypeIsFloating(op1)); + op1 = gtNewCastNode(callType, op1, false, callType); + } -/***************************************************************************** - * - * Determine whether the call could be converted to an implicit tail call - * - */ -bool Compiler::impIsImplicitTailCallCandidate( - OPCODE opcode, const BYTE* codeAddrOfNextOpcode, const BYTE* codeEnd, int prefixFlags, bool isRecursive) -{ + op1 = new (this, GT_INTRINSIC) GenTreeIntrinsic(genActualType(callType), op1, intrinsicName, method); + break; -#if FEATURE_TAILCALL_OPT - if (!opts.compTailCallOpt) - { - return false; - } + case 2: + op2 = impPopStack().val; + op1 = impPopStack().val; - if (opts.OptimizationDisabled()) - { - return false; - } + arg = sig->args; + op1Type = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg, &tmpClass))); - // must not be tail prefixed - if (prefixFlags & PREFIX_TAILCALL_EXPLICIT) - { - return false; - } + if (op1->TypeGet() != genActualType(op1Type)) + { + assert(varTypeIsFloating(op1)); + op1 = gtNewCastNode(callType, op1, false, callType); + } -#if !FEATURE_TAILCALL_OPT_SHARED_RETURN - // the block containing call is marked as BBJ_RETURN - // We allow shared ret tail call optimization on recursive calls even under - // !FEATURE_TAILCALL_OPT_SHARED_RETURN. - if (!isRecursive && (compCurBB->bbJumpKind != BBJ_RETURN)) - return false; -#endif // !FEATURE_TAILCALL_OPT_SHARED_RETURN + arg = info.compCompHnd->getArgNext(arg); + op2Type = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg, &tmpClass))); - // must be call+ret or call+pop+ret - if (!impIsTailCallILPattern(false, opcode, codeAddrOfNextOpcode, codeEnd, isRecursive)) - { - return false; + if (op2->TypeGet() != genActualType(op2Type)) + { + assert(varTypeIsFloating(op2)); + op2 = gtNewCastNode(callType, op2, false, callType); + } + + op1 = + new (this, GT_INTRINSIC) GenTreeIntrinsic(genActualType(callType), op1, op2, intrinsicName, method); + break; + + default: + NO_WAY("Unsupported number of args for Math Intrinsic"); + } + + if (IsIntrinsicImplementedByUserCall(intrinsicName)) + { + op1->gtFlags |= GTF_CALL; + } } - return true; -#else - return false; -#endif // FEATURE_TAILCALL_OPT + return op1; } -/***************************************************************************** - For struct return values, re-type the operand in the case where the ABI - does not use a struct return buffer - */ - //------------------------------------------------------------------------ -// impFixupStructReturnType: Adjust a struct value being returned. -// -// In the multi-reg case, we we force IR to be one of the following: -// GT_RETURN(LCL_VAR) or GT_RETURN(CALL). If op is anything other than -// a lclvar or call, it is assigned to a temp, which is then returned. -// In the non-multireg case, the two special helpers with "fake" return -// buffers are handled ("GETFIELDSTRUCT" and "UNBOX_NULLABLE"). +// lookupNamedIntrinsic: map method to jit named intrinsic value // // Arguments: -// op - the return value +// method -- method handle for method // // Return Value: -// The (possibly modified) value to return. +// Id for the named intrinsic, or Illegal if none. +// +// Notes: +// method should have CORINFO_FLG_INTRINSIC set in its attributes, +// otherwise it is not a named jit intrinsic. // -GenTree* Compiler::impFixupStructReturnType(GenTree* op) +NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) { - assert(varTypeIsStruct(info.compRetType)); - assert(info.compRetBuffArg == BAD_VAR_NUM); + const char* className = nullptr; + const char* namespaceName = nullptr; + const char* enclosingClassName = nullptr; + const char* methodName = + info.compCompHnd->getMethodNameFromMetadata(method, &className, &namespaceName, &enclosingClassName); - JITDUMP("\nimpFixupStructReturnType: retyping\n"); - DISPTREE(op); + JITDUMP("Named Intrinsic "); - if (op->IsCall() && op->AsCall()->TreatAsShouldHaveRetBufArg(this)) + if (namespaceName != nullptr) { - // This must be one of those 'special' helpers that don't really have a return buffer, but instead - // use it as a way to keep the trees cleaner with fewer address-taken temps. Well now we have to - // materialize the return buffer as an address-taken temp. Then we can return the temp. - // - unsigned tmpNum = lvaGrabTemp(true DEBUGARG("pseudo return buffer")); - - // No need to spill anything as we're about to return. - impAssignTempGen(tmpNum, op, info.compMethodInfo->args.retTypeClass, CHECK_SPILL_NONE); - - op = gtNewLclvNode(tmpNum, info.compRetType); - JITDUMP("\nimpFixupStructReturnType: created a pseudo-return buffer for a special helper\n"); - DISPTREE(op); - - return op; + JITDUMP("%s.", namespaceName); } - - if (compMethodReturnsMultiRegRetType() || op->IsMultiRegNode()) + if (enclosingClassName != nullptr) { - // We can use any local with multiple registers (it will be forced to memory on mismatch), - // except for implicit byrefs (they may turn into indirections). - if (op->OperIs(GT_LCL_VAR) && !lvaIsImplicitByRefLocal(op->AsLclVar()->GetLclNum())) - { - // Note that this is a multi-reg return. - unsigned lclNum = op->AsLclVarCommon()->GetLclNum(); - lvaTable[lclNum].lvIsMultiRegRet = true; - - // TODO-1stClassStructs: Handle constant propagation and CSE-ing of multireg returns. - op->gtFlags |= GTF_DONT_CSE; - - return op; - } - - // In contrast, we can only use multi-reg calls directly if they have the exact same ABI. - // Calling convention equality is a conservative approximation for that check. - if (op->IsCall() && (op->AsCall()->GetUnmanagedCallConv() == info.compCallConv) -#if defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) - // TODO-Review: this seems unnecessary. Return ABI doesn't change under varargs. - && !op->AsCall()->IsVarargs() -#endif // defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) - ) - { - return op; - } - - if (op->IsCall()) - { - // We cannot tail call because control needs to return to fixup the calling convention - // for result return. - op->AsCall()->gtCallMoreFlags &= ~GTF_CALL_M_TAILCALL; - op->AsCall()->gtCallMoreFlags &= ~GTF_CALL_M_EXPLICIT_TAILCALL; - } - - // The backend does not support other struct-producing nodes (e. g. OBJs) as sources of multi-reg returns. - // It also does not support assembling a multi-reg node into one register (for RETURN nodes at least). - return impAssignMultiRegTypeToVar(op, info.compMethodInfo->args.retTypeClass DEBUGARG(info.compCallConv)); + JITDUMP("%s.", enclosingClassName); } - - // Not a multi-reg return or value, we can simply use it directly. - return op; -} - -/***************************************************************************** - CEE_LEAVE may be jumping out of a protected block, viz, a catch or a - finally-protected try. We find the finally blocks protecting the current - offset (in order) by walking over the complete exception table and - finding enclosing clauses. This assumes that the table is sorted. - This will create a series of BBJ_CALLFINALLY -> BBJ_CALLFINALLY ... -> BBJ_ALWAYS. - - If we are leaving a catch handler, we need to attach the - CPX_ENDCATCHes to the correct BBJ_CALLFINALLY blocks. - - After this function, the BBJ_LEAVE block has been converted to a different type. - */ - -#if !defined(FEATURE_EH_FUNCLETS) - -void Compiler::impImportLeave(BasicBlock* block) -{ -#ifdef DEBUG - if (verbose) + if (className != nullptr) { - printf("\nBefore import CEE_LEAVE:\n"); - fgDispBasicBlocks(); - fgDispHandlerTab(); + JITDUMP("%s.", className); + } + if (methodName != nullptr) + { + JITDUMP("%s", methodName); } -#endif // DEBUG - - bool invalidatePreds = false; // If we create new blocks, invalidate the predecessor lists (if created) - unsigned blkAddr = block->bbCodeOffs; - BasicBlock* leaveTarget = block->bbJumpDest; - unsigned jmpAddr = leaveTarget->bbCodeOffs; - - // LEAVE clears the stack, spill side effects, and set stack to 0 - impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("impImportLeave")); - verCurrentState.esStackDepth = 0; + if ((namespaceName == nullptr) || (className == nullptr) || (methodName == nullptr)) + { + // Check if we are dealing with an MD array's known runtime method + CorInfoArrayIntrinsic arrayFuncIndex = info.compCompHnd->getArrayIntrinsicID(method); + switch (arrayFuncIndex) + { + case CorInfoArrayIntrinsic::GET: + JITDUMP("ARRAY_FUNC_GET: Recognized\n"); + return NI_Array_Get; + case CorInfoArrayIntrinsic::SET: + JITDUMP("ARRAY_FUNC_SET: Recognized\n"); + return NI_Array_Set; + case CorInfoArrayIntrinsic::ADDRESS: + JITDUMP("ARRAY_FUNC_ADDRESS: Recognized\n"); + return NI_Array_Address; + default: + break; + } - assert(block->bbJumpKind == BBJ_LEAVE); - assert(fgBBs == (BasicBlock**)0xCDCD || fgLookupBB(jmpAddr) != NULL); // should be a BB boundary + JITDUMP(": Not recognized, not enough metadata\n"); + return NI_Illegal; + } - BasicBlock* step = DUMMY_INIT(NULL); - unsigned encFinallies = 0; // Number of enclosing finallies. - GenTree* endCatches = NULL; - Statement* endLFinStmt = NULL; // The statement tree to indicate the end of locally-invoked finally. + JITDUMP(": "); - unsigned XTnum; - EHblkDsc* HBtab; + NamedIntrinsic result = NI_Illegal; - for (XTnum = 0, HBtab = compHndBBtab; XTnum < compHndBBtabCount; XTnum++, HBtab++) + if (strcmp(namespaceName, "System") == 0) { - // Grab the handler offsets - - IL_OFFSET tryBeg = HBtab->ebdTryBegOffs(); - IL_OFFSET tryEnd = HBtab->ebdTryEndOffs(); - IL_OFFSET hndBeg = HBtab->ebdHndBegOffs(); - IL_OFFSET hndEnd = HBtab->ebdHndEndOffs(); - - /* Is this a catch-handler we are CEE_LEAVEing out of? - * If so, we need to call CORINFO_HELP_ENDCATCH. - */ - - if (jitIsBetween(blkAddr, hndBeg, hndEnd) && !jitIsBetween(jmpAddr, hndBeg, hndEnd)) + if ((strcmp(className, "Enum") == 0) && (strcmp(methodName, "HasFlag") == 0)) { - // Can't CEE_LEAVE out of a finally/fault handler - if (HBtab->HasFinallyOrFaultHandler()) - BADCODE("leave out of fault/finally block"); - - // Create the call to CORINFO_HELP_ENDCATCH - GenTree* endCatch = gtNewHelperCallNode(CORINFO_HELP_ENDCATCH, TYP_VOID); - - // Make a list of all the currently pending endCatches - if (endCatches) - endCatches = gtNewOperNode(GT_COMMA, TYP_VOID, endCatches, endCatch); - else - endCatches = endCatch; - -#ifdef DEBUG - if (verbose) + result = NI_System_Enum_HasFlag; + } + else if (strcmp(className, "Activator") == 0) + { + if (strcmp(methodName, "AllocatorOf") == 0) { - printf("impImportLeave - " FMT_BB " jumping out of catch handler EH#%u, adding call to " - "CORINFO_HELP_ENDCATCH\n", - block->bbNum, XTnum); + result = NI_System_Activator_AllocatorOf; + } + else if (strcmp(methodName, "DefaultConstructorOf") == 0) + { + result = NI_System_Activator_DefaultConstructorOf; } -#endif } - else if (HBtab->HasFinallyHandler() && jitIsBetween(blkAddr, tryBeg, tryEnd) && - !jitIsBetween(jmpAddr, tryBeg, tryEnd)) + else if (strcmp(className, "BitConverter") == 0) { - /* This is a finally-protected try we are jumping out of */ - - /* If there are any pending endCatches, and we have already - jumped out of a finally-protected try, then the endCatches - have to be put in a block in an outer try for async - exceptions to work correctly. - Else, just use append to the original block */ - - BasicBlock* callBlock; - - assert(!encFinallies == - !endLFinStmt); // if we have finallies, we better have an endLFin tree, and vice-versa - - if (encFinallies == 0) + if (methodName[0] == 'D') { - assert(step == DUMMY_INIT(NULL)); - callBlock = block; - callBlock->bbJumpKind = BBJ_CALLFINALLY; // convert the BBJ_LEAVE to BBJ_CALLFINALLY - - if (endCatches) - impAppendTree(endCatches, CHECK_SPILL_NONE, impCurStmtDI); - -#ifdef DEBUG - if (verbose) + if (strcmp(methodName, "DoubleToInt64Bits") == 0) { - printf("impImportLeave - jumping out of a finally-protected try, convert block to BBJ_CALLFINALLY " - "block %s\n", - callBlock->dspToString()); + result = NI_System_BitConverter_DoubleToInt64Bits; + } + else if (strcmp(methodName, "DoubleToUInt64Bits") == 0) + { + result = NI_System_BitConverter_DoubleToInt64Bits; } -#endif } - else + else if (methodName[0] == 'I') { - assert(step != DUMMY_INIT(NULL)); - - /* Calling the finally block */ - callBlock = fgNewBBinRegion(BBJ_CALLFINALLY, XTnum + 1, 0, step); - assert(step->bbJumpKind == BBJ_ALWAYS); - step->bbJumpDest = callBlock; // the previous call to a finally returns to this call (to the next - // finally in the chain) - step->bbJumpDest->bbRefs++; - - /* The new block will inherit this block's weight */ - callBlock->inheritWeight(block); - -#ifdef DEBUG - if (verbose) + if (strcmp(methodName, "Int32BitsToSingle") == 0) { - printf("impImportLeave - jumping out of a finally-protected try, new BBJ_CALLFINALLY block %s\n", - callBlock->dspToString()); + result = NI_System_BitConverter_Int32BitsToSingle; } -#endif - - Statement* lastStmt; - - if (endCatches) + else if (strcmp(methodName, "Int64BitsToDouble") == 0) { - lastStmt = gtNewStmt(endCatches); - endLFinStmt->SetNextStmt(lastStmt); - lastStmt->SetPrevStmt(endLFinStmt); + result = NI_System_BitConverter_Int64BitsToDouble; } - else + } + else if (methodName[0] == 'S') + { + if (strcmp(methodName, "SingleToInt32Bits") == 0) { - lastStmt = endLFinStmt; + result = NI_System_BitConverter_SingleToInt32Bits; + } + else if (strcmp(methodName, "SingleToUInt32Bits") == 0) + { + result = NI_System_BitConverter_SingleToInt32Bits; } - - // note that this sets BBF_IMPORTED on the block - impEndTreeList(callBlock, endLFinStmt, lastStmt); } - - step = fgNewBBafter(BBJ_ALWAYS, callBlock, true); - /* The new block will inherit this block's weight */ - step->inheritWeight(block); - step->bbFlags |= BBF_IMPORTED | BBF_KEEP_BBJ_ALWAYS; - -#ifdef DEBUG - if (verbose) + else if (methodName[0] == 'U') { - printf("impImportLeave - jumping out of a finally-protected try, created step (BBJ_ALWAYS) block %s\n", - step->dspToString()); + if (strcmp(methodName, "UInt32BitsToSingle") == 0) + { + result = NI_System_BitConverter_Int32BitsToSingle; + } + else if (strcmp(methodName, "UInt64BitsToDouble") == 0) + { + result = NI_System_BitConverter_Int64BitsToDouble; + } } -#endif - - unsigned finallyNesting = compHndBBtab[XTnum].ebdHandlerNestingLevel; - assert(finallyNesting <= compHndBBtabCount); - - callBlock->bbJumpDest = HBtab->ebdHndBeg; // This callBlock will call the "finally" handler. - GenTree* endLFin = new (this, GT_END_LFIN) GenTreeVal(GT_END_LFIN, TYP_VOID, finallyNesting); - endLFinStmt = gtNewStmt(endLFin); - endCatches = NULL; - - encFinallies++; - - invalidatePreds = true; } - } - - /* Append any remaining endCatches, if any */ - - assert(!encFinallies == !endLFinStmt); - - if (encFinallies == 0) - { - assert(step == DUMMY_INIT(NULL)); - block->bbJumpKind = BBJ_ALWAYS; // convert the BBJ_LEAVE to a BBJ_ALWAYS - - if (endCatches) - impAppendTree(endCatches, CHECK_SPILL_NONE, impCurStmtDI); - -#ifdef DEBUG - if (verbose) + else if (strcmp(className, "Math") == 0 || strcmp(className, "MathF") == 0) { - printf("impImportLeave - no enclosing finally-protected try blocks; convert CEE_LEAVE block to BBJ_ALWAYS " - "block %s\n", - block->dspToString()); + if (strcmp(methodName, "Abs") == 0) + { + result = NI_System_Math_Abs; + } + else if (strcmp(methodName, "Acos") == 0) + { + result = NI_System_Math_Acos; + } + else if (strcmp(methodName, "Acosh") == 0) + { + result = NI_System_Math_Acosh; + } + else if (strcmp(methodName, "Asin") == 0) + { + result = NI_System_Math_Asin; + } + else if (strcmp(methodName, "Asinh") == 0) + { + result = NI_System_Math_Asinh; + } + else if (strcmp(methodName, "Atan") == 0) + { + result = NI_System_Math_Atan; + } + else if (strcmp(methodName, "Atanh") == 0) + { + result = NI_System_Math_Atanh; + } + else if (strcmp(methodName, "Atan2") == 0) + { + result = NI_System_Math_Atan2; + } + else if (strcmp(methodName, "Cbrt") == 0) + { + result = NI_System_Math_Cbrt; + } + else if (strcmp(methodName, "Ceiling") == 0) + { + result = NI_System_Math_Ceiling; + } + else if (strcmp(methodName, "Cos") == 0) + { + result = NI_System_Math_Cos; + } + else if (strcmp(methodName, "Cosh") == 0) + { + result = NI_System_Math_Cosh; + } + else if (strcmp(methodName, "Exp") == 0) + { + result = NI_System_Math_Exp; + } + else if (strcmp(methodName, "Floor") == 0) + { + result = NI_System_Math_Floor; + } + else if (strcmp(methodName, "FMod") == 0) + { + result = NI_System_Math_FMod; + } + else if (strcmp(methodName, "FusedMultiplyAdd") == 0) + { + result = NI_System_Math_FusedMultiplyAdd; + } + else if (strcmp(methodName, "ILogB") == 0) + { + result = NI_System_Math_ILogB; + } + else if (strcmp(methodName, "Log") == 0) + { + result = NI_System_Math_Log; + } + else if (strcmp(methodName, "Log2") == 0) + { + result = NI_System_Math_Log2; + } + else if (strcmp(methodName, "Log10") == 0) + { + result = NI_System_Math_Log10; + } + else if (strcmp(methodName, "Max") == 0) + { + result = NI_System_Math_Max; + } + else if (strcmp(methodName, "Min") == 0) + { + result = NI_System_Math_Min; + } + else if (strcmp(methodName, "Pow") == 0) + { + result = NI_System_Math_Pow; + } + else if (strcmp(methodName, "Round") == 0) + { + result = NI_System_Math_Round; + } + else if (strcmp(methodName, "Sin") == 0) + { + result = NI_System_Math_Sin; + } + else if (strcmp(methodName, "Sinh") == 0) + { + result = NI_System_Math_Sinh; + } + else if (strcmp(methodName, "Sqrt") == 0) + { + result = NI_System_Math_Sqrt; + } + else if (strcmp(methodName, "Tan") == 0) + { + result = NI_System_Math_Tan; + } + else if (strcmp(methodName, "Tanh") == 0) + { + result = NI_System_Math_Tanh; + } + else if (strcmp(methodName, "Truncate") == 0) + { + result = NI_System_Math_Truncate; + } } -#endif - } - else - { - // If leaveTarget is the start of another try block, we want to make sure that - // we do not insert finalStep into that try block. Hence, we find the enclosing - // try block. - unsigned tryIndex = bbFindInnermostCommonTryRegion(step, leaveTarget); - - // Insert a new BB either in the try region indicated by tryIndex or - // the handler region indicated by leaveTarget->bbHndIndex, - // depending on which is the inner region. - BasicBlock* finalStep = fgNewBBinRegion(BBJ_ALWAYS, tryIndex, leaveTarget->bbHndIndex, step); - finalStep->bbFlags |= BBF_KEEP_BBJ_ALWAYS; - step->bbJumpDest = finalStep; - - /* The new block will inherit this block's weight */ - finalStep->inheritWeight(block); - -#ifdef DEBUG - if (verbose) + else if (strcmp(className, "GC") == 0) { - printf("impImportLeave - finalStep block required (encFinallies(%d) > 0), new block %s\n", encFinallies, - finalStep->dspToString()); + if (strcmp(methodName, "KeepAlive") == 0) + { + result = NI_System_GC_KeepAlive; + } } -#endif - - Statement* lastStmt; - - if (endCatches) + else if (strcmp(className, "Array") == 0) { - lastStmt = gtNewStmt(endCatches); - endLFinStmt->SetNextStmt(lastStmt); - lastStmt->SetPrevStmt(endLFinStmt); + if (strcmp(methodName, "Clone") == 0) + { + result = NI_System_Array_Clone; + } + else if (strcmp(methodName, "GetLength") == 0) + { + result = NI_System_Array_GetLength; + } + else if (strcmp(methodName, "GetLowerBound") == 0) + { + result = NI_System_Array_GetLowerBound; + } + else if (strcmp(methodName, "GetUpperBound") == 0) + { + result = NI_System_Array_GetUpperBound; + } } - else + else if (strcmp(className, "Object") == 0) { - lastStmt = endLFinStmt; + if (strcmp(methodName, "MemberwiseClone") == 0) + { + result = NI_System_Object_MemberwiseClone; + } + else if (strcmp(methodName, "GetType") == 0) + { + result = NI_System_Object_GetType; + } } - - impEndTreeList(finalStep, endLFinStmt, lastStmt); - - finalStep->bbJumpDest = leaveTarget; // this is the ultimate destination of the LEAVE - - // Queue up the jump target for importing - - impImportBlockPending(leaveTarget); - - invalidatePreds = true; - } - - if (invalidatePreds && fgComputePredsDone) - { - JITDUMP("\n**** impImportLeave - Removing preds after creating new blocks\n"); - fgRemovePreds(); - } - -#ifdef DEBUG - fgVerifyHandlerTab(); - - if (verbose) + else if (strcmp(className, "RuntimeTypeHandle") == 0) + { + if (strcmp(methodName, "GetValueInternal") == 0) + { + result = NI_System_RuntimeTypeHandle_GetValueInternal; + } + } + else if (strcmp(className, "Type") == 0) + { + if (strcmp(methodName, "get_IsValueType") == 0) + { + result = NI_System_Type_get_IsValueType; + } + else if (strcmp(methodName, "get_IsByRefLike") == 0) + { + result = NI_System_Type_get_IsByRefLike; + } + else if (strcmp(methodName, "IsAssignableFrom") == 0) + { + result = NI_System_Type_IsAssignableFrom; + } + else if (strcmp(methodName, "IsAssignableTo") == 0) + { + result = NI_System_Type_IsAssignableTo; + } + else if (strcmp(methodName, "op_Equality") == 0) + { + result = NI_System_Type_op_Equality; + } + else if (strcmp(methodName, "op_Inequality") == 0) + { + result = NI_System_Type_op_Inequality; + } + else if (strcmp(methodName, "GetTypeFromHandle") == 0) + { + result = NI_System_Type_GetTypeFromHandle; + } + } + else if (strcmp(className, "String") == 0) + { + if (strcmp(methodName, "Equals") == 0) + { + result = NI_System_String_Equals; + } + else if (strcmp(methodName, "get_Chars") == 0) + { + result = NI_System_String_get_Chars; + } + else if (strcmp(methodName, "get_Length") == 0) + { + result = NI_System_String_get_Length; + } + else if (strcmp(methodName, "op_Implicit") == 0) + { + result = NI_System_String_op_Implicit; + } + else if (strcmp(methodName, "StartsWith") == 0) + { + result = NI_System_String_StartsWith; + } + } + else if (strcmp(className, "MemoryExtensions") == 0) + { + if (strcmp(methodName, "AsSpan") == 0) + { + result = NI_System_MemoryExtensions_AsSpan; + } + if (strcmp(methodName, "SequenceEqual") == 0) + { + result = NI_System_MemoryExtensions_SequenceEqual; + } + else if (strcmp(methodName, "Equals") == 0) + { + result = NI_System_MemoryExtensions_Equals; + } + else if (strcmp(methodName, "StartsWith") == 0) + { + result = NI_System_MemoryExtensions_StartsWith; + } + } + else if (strcmp(className, "Span`1") == 0) + { + if (strcmp(methodName, "get_Item") == 0) + { + result = NI_System_Span_get_Item; + } + } + else if (strcmp(className, "ReadOnlySpan`1") == 0) + { + if (strcmp(methodName, "get_Item") == 0) + { + result = NI_System_ReadOnlySpan_get_Item; + } + } + else if (strcmp(className, "EETypePtr") == 0) + { + if (strcmp(methodName, "EETypePtrOf") == 0) + { + result = NI_System_EETypePtr_EETypePtrOf; + } + } + } + else if (strcmp(namespaceName, "Internal.Runtime") == 0) { - printf("\nAfter import CEE_LEAVE:\n"); - fgDispBasicBlocks(); - fgDispHandlerTab(); + if (strcmp(className, "MethodTable") == 0) + { + if (strcmp(methodName, "Of") == 0) + { + result = NI_Internal_Runtime_MethodTable_Of; + } + } } -#endif // DEBUG -} - -#else // FEATURE_EH_FUNCLETS - -void Compiler::impImportLeave(BasicBlock* block) -{ -#ifdef DEBUG - if (verbose) + else if (strcmp(namespaceName, "System.Threading") == 0) { - printf("\nBefore import CEE_LEAVE in " FMT_BB " (targeting " FMT_BB "):\n", block->bbNum, - block->bbJumpDest->bbNum); - fgDispBasicBlocks(); - fgDispHandlerTab(); + if (strcmp(className, "Thread") == 0) + { + if (strcmp(methodName, "get_CurrentThread") == 0) + { + result = NI_System_Threading_Thread_get_CurrentThread; + } + else if (strcmp(methodName, "get_ManagedThreadId") == 0) + { + result = NI_System_Threading_Thread_get_ManagedThreadId; + } + } + else if (strcmp(className, "Interlocked") == 0) + { +#ifndef TARGET_ARM64 + // TODO-CQ: Implement for XArch (https://github.com/dotnet/runtime/issues/32239). + if (strcmp(methodName, "And") == 0) + { + result = NI_System_Threading_Interlocked_And; + } + else if (strcmp(methodName, "Or") == 0) + { + result = NI_System_Threading_Interlocked_Or; + } +#endif + if (strcmp(methodName, "CompareExchange") == 0) + { + result = NI_System_Threading_Interlocked_CompareExchange; + } + else if (strcmp(methodName, "Exchange") == 0) + { + result = NI_System_Threading_Interlocked_Exchange; + } + else if (strcmp(methodName, "ExchangeAdd") == 0) + { + result = NI_System_Threading_Interlocked_ExchangeAdd; + } + else if (strcmp(methodName, "MemoryBarrier") == 0) + { + result = NI_System_Threading_Interlocked_MemoryBarrier; + } + else if (strcmp(methodName, "ReadMemoryBarrier") == 0) + { + result = NI_System_Threading_Interlocked_ReadMemoryBarrier; + } + } } -#endif // DEBUG - - bool invalidatePreds = false; // If we create new blocks, invalidate the predecessor lists (if created) - unsigned blkAddr = block->bbCodeOffs; - BasicBlock* leaveTarget = block->bbJumpDest; - unsigned jmpAddr = leaveTarget->bbCodeOffs; - - // LEAVE clears the stack, spill side effects, and set stack to 0 - - impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("impImportLeave")); - verCurrentState.esStackDepth = 0; - - assert(block->bbJumpKind == BBJ_LEAVE); - assert(fgBBs == (BasicBlock**)0xCDCD || fgLookupBB(jmpAddr) != nullptr); // should be a BB boundary +#if defined(TARGET_XARCH) || defined(TARGET_ARM64) + else if (strcmp(namespaceName, "System.Buffers.Binary") == 0) + { + if ((strcmp(className, "BinaryPrimitives") == 0) && (strcmp(methodName, "ReverseEndianness") == 0)) + { + result = NI_System_Buffers_Binary_BinaryPrimitives_ReverseEndianness; + } + } +#endif // defined(TARGET_XARCH) || defined(TARGET_ARM64) + else if (strcmp(namespaceName, "System.Collections.Generic") == 0) + { + if ((strcmp(className, "EqualityComparer`1") == 0) && (strcmp(methodName, "get_Default") == 0)) + { + result = NI_System_Collections_Generic_EqualityComparer_get_Default; + } + else if ((strcmp(className, "Comparer`1") == 0) && (strcmp(methodName, "get_Default") == 0)) + { + result = NI_System_Collections_Generic_Comparer_get_Default; + } + } + else if ((strcmp(namespaceName, "System.Numerics") == 0) && (strcmp(className, "BitOperations") == 0)) + { + if (strcmp(methodName, "PopCount") == 0) + { + result = NI_System_Numerics_BitOperations_PopCount; + } + } +#ifdef FEATURE_HW_INTRINSICS + else if (strcmp(namespaceName, "System.Numerics") == 0) + { + CORINFO_SIG_INFO sig; + info.compCompHnd->getMethodSig(method, &sig); - BasicBlock* step = nullptr; + int sizeOfVectorT = getSIMDVectorRegisterByteLength(); - enum StepType + result = SimdAsHWIntrinsicInfo::lookupId(&sig, className, methodName, enclosingClassName, sizeOfVectorT); + } +#endif // FEATURE_HW_INTRINSICS + else if (strcmp(namespaceName, "System.Runtime.CompilerServices") == 0) { - // No step type; step == NULL. - ST_None, + if (strcmp(className, "Unsafe") == 0) + { + if (strcmp(methodName, "Add") == 0) + { + result = NI_SRCS_UNSAFE_Add; + } + else if (strcmp(methodName, "AddByteOffset") == 0) + { + result = NI_SRCS_UNSAFE_AddByteOffset; + } + else if (strcmp(methodName, "AreSame") == 0) + { + result = NI_SRCS_UNSAFE_AreSame; + } + else if (strcmp(methodName, "As") == 0) + { + result = NI_SRCS_UNSAFE_As; + } + else if (strcmp(methodName, "AsPointer") == 0) + { + result = NI_SRCS_UNSAFE_AsPointer; + } + else if (strcmp(methodName, "AsRef") == 0) + { + result = NI_SRCS_UNSAFE_AsRef; + } + else if (strcmp(methodName, "ByteOffset") == 0) + { + result = NI_SRCS_UNSAFE_ByteOffset; + } + else if (strcmp(methodName, "Copy") == 0) + { + result = NI_SRCS_UNSAFE_Copy; + } + else if (strcmp(methodName, "CopyBlock") == 0) + { + result = NI_SRCS_UNSAFE_CopyBlock; + } + else if (strcmp(methodName, "CopyBlockUnaligned") == 0) + { + result = NI_SRCS_UNSAFE_CopyBlockUnaligned; + } + else if (strcmp(methodName, "InitBlock") == 0) + { + result = NI_SRCS_UNSAFE_InitBlock; + } + else if (strcmp(methodName, "InitBlockUnaligned") == 0) + { + result = NI_SRCS_UNSAFE_InitBlockUnaligned; + } + else if (strcmp(methodName, "IsAddressGreaterThan") == 0) + { + result = NI_SRCS_UNSAFE_IsAddressGreaterThan; + } + else if (strcmp(methodName, "IsAddressLessThan") == 0) + { + result = NI_SRCS_UNSAFE_IsAddressLessThan; + } + else if (strcmp(methodName, "IsNullRef") == 0) + { + result = NI_SRCS_UNSAFE_IsNullRef; + } + else if (strcmp(methodName, "NullRef") == 0) + { + result = NI_SRCS_UNSAFE_NullRef; + } + else if (strcmp(methodName, "Read") == 0) + { + result = NI_SRCS_UNSAFE_Read; + } + else if (strcmp(methodName, "ReadUnaligned") == 0) + { + result = NI_SRCS_UNSAFE_ReadUnaligned; + } + else if (strcmp(methodName, "SizeOf") == 0) + { + result = NI_SRCS_UNSAFE_SizeOf; + } + else if (strcmp(methodName, "SkipInit") == 0) + { + result = NI_SRCS_UNSAFE_SkipInit; + } + else if (strcmp(methodName, "Subtract") == 0) + { + result = NI_SRCS_UNSAFE_Subtract; + } + else if (strcmp(methodName, "SubtractByteOffset") == 0) + { + result = NI_SRCS_UNSAFE_SubtractByteOffset; + } + else if (strcmp(methodName, "Unbox") == 0) + { + result = NI_SRCS_UNSAFE_Unbox; + } + else if (strcmp(methodName, "Write") == 0) + { + result = NI_SRCS_UNSAFE_Write; + } + else if (strcmp(methodName, "WriteUnaligned") == 0) + { + result = NI_SRCS_UNSAFE_WriteUnaligned; + } + } + else if (strcmp(className, "RuntimeHelpers") == 0) + { + if (strcmp(methodName, "CreateSpan") == 0) + { + result = NI_System_Runtime_CompilerServices_RuntimeHelpers_CreateSpan; + } + else if (strcmp(methodName, "InitializeArray") == 0) + { + result = NI_System_Runtime_CompilerServices_RuntimeHelpers_InitializeArray; + } + else if (strcmp(methodName, "IsKnownConstant") == 0) + { + result = NI_System_Runtime_CompilerServices_RuntimeHelpers_IsKnownConstant; + } + } + } + else if (strcmp(namespaceName, "System.Runtime.InteropServices") == 0) + { + if (strcmp(className, "MemoryMarshal") == 0) + { + if (strcmp(methodName, "GetArrayDataReference") == 0) + { + result = NI_System_Runtime_InteropService_MemoryMarshal_GetArrayDataReference; + } + } + } + else if (strncmp(namespaceName, "System.Runtime.Intrinsics", 25) == 0) + { + // We go down this path even when FEATURE_HW_INTRINSICS isn't enabled + // so we can specially handle IsSupported and recursive calls. - // Is the step block the BBJ_ALWAYS block of a BBJ_CALLFINALLY/BBJ_ALWAYS pair? - // That is, is step->bbJumpDest where a finally will return to? - ST_FinallyReturn, + // This is required to appropriately handle the intrinsics on platforms + // which don't support them. On such a platform methods like Vector64.Create + // will be seen as `Intrinsic` and `mustExpand` due to having a code path + // which is recursive. When such a path is hit we expect it to be handled by + // the importer and we fire an assert if it wasn't and in previous versions + // of the JIT would fail fast. This was changed to throw a PNSE instead but + // we still assert as most intrinsics should have been recognized/handled. - // The step block is a catch return. - ST_Catch, + // In order to avoid the assert, we specially handle the IsSupported checks + // (to better allow dead-code optimizations) and we explicitly throw a PNSE + // as we know that is the desired behavior for the HWIntrinsics when not + // supported. For cases like Vector64.Create, this is fine because it will + // be behind a relevant IsSupported check and will never be hit and the + // software fallback will be executed instead. - // The step block is in a "try", created as the target for a finally return or the target for a catch return. - ST_Try - }; - StepType stepType = ST_None; + CLANG_FORMAT_COMMENT_ANCHOR; - unsigned XTnum; - EHblkDsc* HBtab; +#ifdef FEATURE_HW_INTRINSICS + namespaceName += 25; + const char* platformNamespaceName; - for (XTnum = 0, HBtab = compHndBBtab; XTnum < compHndBBtabCount; XTnum++, HBtab++) - { - // Grab the handler offsets +#if defined(TARGET_XARCH) + platformNamespaceName = ".X86"; +#elif defined(TARGET_ARM64) + platformNamespaceName = ".Arm"; +#else +#error Unsupported platform +#endif - IL_OFFSET tryBeg = HBtab->ebdTryBegOffs(); - IL_OFFSET tryEnd = HBtab->ebdTryEndOffs(); - IL_OFFSET hndBeg = HBtab->ebdHndBegOffs(); - IL_OFFSET hndEnd = HBtab->ebdHndEndOffs(); + if ((namespaceName[0] == '\0') || (strcmp(namespaceName, platformNamespaceName) == 0)) + { + CORINFO_SIG_INFO sig; + info.compCompHnd->getMethodSig(method, &sig); - /* Is this a catch-handler we are CEE_LEAVEing out of? - */ + result = HWIntrinsicInfo::lookupId(this, &sig, className, methodName, enclosingClassName); + } +#endif // FEATURE_HW_INTRINSICS - if (jitIsBetween(blkAddr, hndBeg, hndEnd) && !jitIsBetween(jmpAddr, hndBeg, hndEnd)) + if (result == NI_Illegal) { - // Can't CEE_LEAVE out of a finally/fault handler - if (HBtab->HasFinallyOrFaultHandler()) + if ((strcmp(methodName, "get_IsSupported") == 0) || (strcmp(methodName, "get_IsHardwareAccelerated") == 0)) { - BADCODE("leave out of fault/finally block"); - } - - /* We are jumping out of a catch */ + // This allows the relevant code paths to be dropped as dead code even + // on platforms where FEATURE_HW_INTRINSICS is not supported. - if (step == nullptr) + result = NI_IsSupported_False; + } + else if (gtIsRecursiveCall(method)) { - step = block; - step->bbJumpKind = BBJ_EHCATCHRET; // convert the BBJ_LEAVE to BBJ_EHCATCHRET - stepType = ST_Catch; + // For the framework itself, any recursive intrinsics will either be + // only supported on a single platform or will be guarded by a relevant + // IsSupported check so the throw PNSE will be valid or dropped. -#ifdef DEBUG - if (verbose) - { - printf("impImportLeave - jumping out of a catch (EH#%u), convert block " FMT_BB - " to BBJ_EHCATCHRET " - "block\n", - XTnum, step->bbNum); - } -#endif + result = NI_Throw_PlatformNotSupportedException; } - else + } + } + else if (strcmp(namespaceName, "System.StubHelpers") == 0) + { + if (strcmp(className, "StubHelpers") == 0) + { + if (strcmp(methodName, "GetStubContext") == 0) { - BasicBlock* exitBlock; - - /* Create a new catch exit block in the catch region for the existing step block to jump to in this - * scope */ - exitBlock = fgNewBBinRegion(BBJ_EHCATCHRET, 0, XTnum + 1, step); - - assert(step->KindIs(BBJ_ALWAYS, BBJ_EHCATCHRET)); - step->bbJumpDest = exitBlock; // the previous step (maybe a call to a nested finally, or a nested catch - // exit) returns to this block - step->bbJumpDest->bbRefs++; - -#if defined(TARGET_ARM) - if (stepType == ST_FinallyReturn) - { - assert(step->bbJumpKind == BBJ_ALWAYS); - // Mark the target of a finally return - step->bbJumpDest->bbFlags |= BBF_FINALLY_TARGET; - } -#endif // defined(TARGET_ARM) - - /* The new block will inherit this block's weight */ - exitBlock->inheritWeight(block); - exitBlock->bbFlags |= BBF_IMPORTED; + result = NI_System_StubHelpers_GetStubContext; + } + else if (strcmp(methodName, "NextCallReturnAddress") == 0) + { + result = NI_System_StubHelpers_NextCallReturnAddress; + } + } + } - /* This exit block is the new step */ - step = exitBlock; - stepType = ST_Catch; + if (result == NI_Illegal) + { + JITDUMP("Not recognized\n"); + } + else if (result == NI_IsSupported_False) + { + JITDUMP("Unsupported - return false"); + } + else if (result == NI_Throw_PlatformNotSupportedException) + { + JITDUMP("Unsupported - throw PlatformNotSupportedException"); + } + else + { + JITDUMP("Recognized\n"); + } + return result; +} - invalidatePreds = true; +//------------------------------------------------------------------------ +// impUnsupportedNamedIntrinsic: Throws an exception for an unsupported named intrinsic +// +// Arguments: +// helper - JIT helper ID for the exception to be thrown +// method - method handle of the intrinsic function. +// sig - signature of the intrinsic call +// mustExpand - true if the intrinsic must return a GenTree*; otherwise, false +// +// Return Value: +// a gtNewMustThrowException if mustExpand is true; otherwise, nullptr +// +GenTree* Compiler::impUnsupportedNamedIntrinsic(unsigned helper, + CORINFO_METHOD_HANDLE method, + CORINFO_SIG_INFO* sig, + bool mustExpand) +{ + // We've hit some error case and may need to return a node for the given error. + // + // When `mustExpand=false`, we are attempting to inline the intrinsic directly into another method. In this + // scenario, we need to return `nullptr` so that a GT_CALL to the intrinsic is emitted instead. This is to + // ensure that everything continues to behave correctly when optimizations are enabled (e.g. things like the + // inliner may expect the node we return to have a certain signature, and the `MustThrowException` node won't + // match that). + // + // When `mustExpand=true`, we are in a GT_CALL to the intrinsic and are attempting to JIT it. This will generally + // be in response to an indirect call (e.g. done via reflection) or in response to an earlier attempt returning + // `nullptr` (under `mustExpand=false`). In that scenario, we are safe to return the `MustThrowException` node. -#ifdef DEBUG - if (verbose) - { - printf("impImportLeave - jumping out of a catch (EH#%u), new BBJ_EHCATCHRET block " FMT_BB "\n", - XTnum, exitBlock->bbNum); - } -#endif - } - } - else if (HBtab->HasFinallyHandler() && jitIsBetween(blkAddr, tryBeg, tryEnd) && - !jitIsBetween(jmpAddr, tryBeg, tryEnd)) + if (mustExpand) + { + for (unsigned i = 0; i < sig->numArgs; i++) { - /* We are jumping out of a finally-protected try */ + impPopStack(); + } - BasicBlock* callBlock; + return gtNewMustThrowException(helper, JITtype2varType(sig->retType), sig->retTypeClass); + } + else + { + return nullptr; + } +} - if (step == nullptr) - { -#if FEATURE_EH_CALLFINALLY_THUNKS +//------------------------------------------------------------------------ +// impArrayAccessIntrinsic: try to replace a multi-dimensional array intrinsics with IR nodes. +// +// Arguments: +// clsHnd - handle for the intrinsic method's class +// sig - signature of the intrinsic method +// memberRef - the token for the intrinsic method +// readonlyCall - true if call has a readonly prefix +// intrinsicName - the intrinsic to expand: one of NI_Array_Address, NI_Array_Get, NI_Array_Set +// +// Return Value: +// The intrinsic expansion, or nullptr if the expansion was not done (and a function call should be made instead). +// +GenTree* Compiler::impArrayAccessIntrinsic( + CORINFO_CLASS_HANDLE clsHnd, CORINFO_SIG_INFO* sig, int memberRef, bool readonlyCall, NamedIntrinsic intrinsicName) +{ + assert((intrinsicName == NI_Array_Address) || (intrinsicName == NI_Array_Get) || (intrinsicName == NI_Array_Set)); - // Put the call to the finally in the enclosing region. - unsigned callFinallyTryIndex = - (HBtab->ebdEnclosingTryIndex == EHblkDsc::NO_ENCLOSING_INDEX) ? 0 : HBtab->ebdEnclosingTryIndex + 1; - unsigned callFinallyHndIndex = - (HBtab->ebdEnclosingHndIndex == EHblkDsc::NO_ENCLOSING_INDEX) ? 0 : HBtab->ebdEnclosingHndIndex + 1; - callBlock = fgNewBBinRegion(BBJ_CALLFINALLY, callFinallyTryIndex, callFinallyHndIndex, block); + // If we are generating SMALL_CODE, we don't want to use intrinsics, as it generates fatter code. + if (compCodeOpt() == SMALL_CODE) + { + JITDUMP("impArrayAccessIntrinsic: rejecting array intrinsic due to SMALL_CODE\n"); + return nullptr; + } - // Convert the BBJ_LEAVE to BBJ_ALWAYS, jumping to the new BBJ_CALLFINALLY. This is because - // the new BBJ_CALLFINALLY is in a different EH region, thus it can't just replace the BBJ_LEAVE, - // which might be in the middle of the "try". In most cases, the BBJ_ALWAYS will jump to the - // next block, and flow optimizations will remove it. - block->bbJumpKind = BBJ_ALWAYS; - block->bbJumpDest = callBlock; - block->bbJumpDest->bbRefs++; + unsigned rank = (intrinsicName == NI_Array_Set) ? (sig->numArgs - 1) : sig->numArgs; - /* The new block will inherit this block's weight */ - callBlock->inheritWeight(block); - callBlock->bbFlags |= BBF_IMPORTED; + // Handle a maximum rank of GT_ARR_MAX_RANK (3). This is an implementation choice (larger ranks are expected + // to be rare) and could be increased. + if (rank > GT_ARR_MAX_RANK) + { + JITDUMP("impArrayAccessIntrinsic: rejecting array intrinsic because rank (%d) > GT_ARR_MAX_RANK (%d)\n", rank, + GT_ARR_MAX_RANK); + return nullptr; + } -#ifdef DEBUG - if (verbose) - { - printf("impImportLeave - jumping out of a finally-protected try (EH#%u), convert block " FMT_BB - " to " - "BBJ_ALWAYS, add BBJ_CALLFINALLY block " FMT_BB "\n", - XTnum, block->bbNum, callBlock->bbNum); - } -#endif + // The rank 1 case is special because it has to handle two array formats. We will simply not do that case. + if (rank <= 1) + { + JITDUMP("impArrayAccessIntrinsic: rejecting array intrinsic because rank (%d) <= 1\n", rank); + return nullptr; + } -#else // !FEATURE_EH_CALLFINALLY_THUNKS + CORINFO_CLASS_HANDLE arrElemClsHnd = nullptr; + var_types elemType = JITtype2varType(info.compCompHnd->getChildType(clsHnd, &arrElemClsHnd)); - callBlock = block; - callBlock->bbJumpKind = BBJ_CALLFINALLY; // convert the BBJ_LEAVE to BBJ_CALLFINALLY + // For the ref case, we will only be able to inline if the types match + // (verifier checks for this, we don't care for the nonverified case and the + // type is final (so we don't need to do the cast)) + if ((intrinsicName != NI_Array_Get) && !readonlyCall && varTypeIsGC(elemType)) + { + // Get the call site signature + CORINFO_SIG_INFO LocalSig; + eeGetCallSiteSig(memberRef, info.compScopeHnd, impTokenLookupContextHandle, &LocalSig); + assert(LocalSig.hasThis()); -#ifdef DEBUG - if (verbose) - { - printf("impImportLeave - jumping out of a finally-protected try (EH#%u), convert block " FMT_BB - " to " - "BBJ_CALLFINALLY block\n", - XTnum, callBlock->bbNum); - } -#endif + CORINFO_CLASS_HANDLE actualElemClsHnd; -#endif // !FEATURE_EH_CALLFINALLY_THUNKS - } - else + if (intrinsicName == NI_Array_Set) + { + // Fetch the last argument, the one that indicates the type we are setting. + CORINFO_ARG_LIST_HANDLE argType = LocalSig.args; + for (unsigned r = 0; r < rank; r++) { - // Calling the finally block. We already have a step block that is either the call-to-finally from a - // more nested try/finally (thus we are jumping out of multiple nested 'try' blocks, each protected by - // a 'finally'), or the step block is the return from a catch. - // - // Due to ThreadAbortException, we can't have the catch return target the call-to-finally block - // directly. Note that if a 'catch' ends without resetting the ThreadAbortException, the VM will - // automatically re-raise the exception, using the return address of the catch (that is, the target - // block of the BBJ_EHCATCHRET) as the re-raise address. If this address is in a finally, the VM will - // refuse to do the re-raise, and the ThreadAbortException will get eaten (and lost). On AMD64/ARM64, - // we put the call-to-finally thunk in a special "cloned finally" EH region that does look like a - // finally clause to the VM. Thus, on these platforms, we can't have BBJ_EHCATCHRET target a - // BBJ_CALLFINALLY directly. (Note that on ARM32, we don't mark the thunk specially -- it lives directly - // within the 'try' region protected by the finally, since we generate code in such a way that execution - // never returns to the call-to-finally call, and the finally-protected 'try' region doesn't appear on - // stack walks.) + argType = info.compCompHnd->getArgNext(argType); + } - assert(step->KindIs(BBJ_ALWAYS, BBJ_EHCATCHRET)); + typeInfo argInfo = verParseArgSigToTypeInfo(&LocalSig, argType); + actualElemClsHnd = argInfo.GetClassHandle(); + } + else + { + assert(intrinsicName == NI_Array_Address); -#if FEATURE_EH_CALLFINALLY_THUNKS - if (step->bbJumpKind == BBJ_EHCATCHRET) - { - // Need to create another step block in the 'try' region that will actually branch to the - // call-to-finally thunk. - BasicBlock* step2 = fgNewBBinRegion(BBJ_ALWAYS, XTnum + 1, 0, step); - step->bbJumpDest = step2; - step->bbJumpDest->bbRefs++; - step2->inheritWeight(block); - step2->bbFlags |= (block->bbFlags & BBF_RUN_RARELY) | BBF_IMPORTED; + // Fetch the return type + typeInfo retInfo = verMakeTypeInfo(LocalSig.retType, LocalSig.retTypeClass); + assert(retInfo.IsByRef()); + actualElemClsHnd = retInfo.GetClassHandle(); + } -#ifdef DEBUG - if (verbose) - { - printf("impImportLeave - jumping out of a finally-protected try (EH#%u), step block is " - "BBJ_EHCATCHRET (" FMT_BB "), new BBJ_ALWAYS step-step block " FMT_BB "\n", - XTnum, step->bbNum, step2->bbNum); - } -#endif + // if it's not final, we can't do the optimization + if (!(info.compCompHnd->getClassAttribs(actualElemClsHnd) & CORINFO_FLG_FINAL)) + { + JITDUMP("impArrayAccessIntrinsic: rejecting array intrinsic because actualElemClsHnd (%p) is not final\n", + dspPtr(actualElemClsHnd)); + return nullptr; + } + } - step = step2; - assert(stepType == ST_Catch); // Leave it as catch type for now. - } -#endif // FEATURE_EH_CALLFINALLY_THUNKS + unsigned arrayElemSize; + if (elemType == TYP_STRUCT) + { + assert(arrElemClsHnd); + arrayElemSize = info.compCompHnd->getClassSize(arrElemClsHnd); + } + else + { + arrayElemSize = genTypeSize(elemType); + } -#if FEATURE_EH_CALLFINALLY_THUNKS - unsigned callFinallyTryIndex = - (HBtab->ebdEnclosingTryIndex == EHblkDsc::NO_ENCLOSING_INDEX) ? 0 : HBtab->ebdEnclosingTryIndex + 1; - unsigned callFinallyHndIndex = - (HBtab->ebdEnclosingHndIndex == EHblkDsc::NO_ENCLOSING_INDEX) ? 0 : HBtab->ebdEnclosingHndIndex + 1; -#else // !FEATURE_EH_CALLFINALLY_THUNKS - unsigned callFinallyTryIndex = XTnum + 1; - unsigned callFinallyHndIndex = 0; // don't care -#endif // !FEATURE_EH_CALLFINALLY_THUNKS + if ((unsigned char)arrayElemSize != arrayElemSize) + { + // arrayElemSize would be truncated as an unsigned char. + // This means the array element is too large. Don't do the optimization. + JITDUMP("impArrayAccessIntrinsic: rejecting array intrinsic because arrayElemSize (%d) is too large\n", + arrayElemSize); + return nullptr; + } - callBlock = fgNewBBinRegion(BBJ_CALLFINALLY, callFinallyTryIndex, callFinallyHndIndex, step); - step->bbJumpDest = callBlock; // the previous call to a finally returns to this call (to the next - // finally in the chain) - step->bbJumpDest->bbRefs++; + GenTree* val = nullptr; -#if defined(TARGET_ARM) - if (stepType == ST_FinallyReturn) - { - assert(step->bbJumpKind == BBJ_ALWAYS); - // Mark the target of a finally return - step->bbJumpDest->bbFlags |= BBF_FINALLY_TARGET; - } -#endif // defined(TARGET_ARM) + if (intrinsicName == NI_Array_Set) + { + // Assignment of a struct is more work, and there are more gets than sets. + // TODO-CQ: support SET (`a[i,j,k] = s`) for struct element arrays. + if (elemType == TYP_STRUCT) + { + JITDUMP("impArrayAccessIntrinsic: rejecting SET array intrinsic because elemType is TYP_STRUCT" + " (implementation limitation)\n", + arrayElemSize); + return nullptr; + } - /* The new block will inherit this block's weight */ - callBlock->inheritWeight(block); - callBlock->bbFlags |= BBF_IMPORTED; + val = impPopStack().val; + assert((genActualType(elemType) == genActualType(val->gtType)) || + (elemType == TYP_FLOAT && val->gtType == TYP_DOUBLE) || + (elemType == TYP_INT && val->gtType == TYP_BYREF) || + (elemType == TYP_DOUBLE && val->gtType == TYP_FLOAT)); + } -#ifdef DEBUG - if (verbose) - { - printf("impImportLeave - jumping out of a finally-protected try (EH#%u), new BBJ_CALLFINALLY " - "block " FMT_BB "\n", - XTnum, callBlock->bbNum); - } -#endif - } + // Here, we're committed to expanding the intrinsic and creating a GT_ARR_ELEM node. + optMethodFlags |= OMF_HAS_MDARRAYREF; + compCurBB->bbFlags |= BBF_HAS_MDARRAYREF; - step = fgNewBBafter(BBJ_ALWAYS, callBlock, true); - stepType = ST_FinallyReturn; + noway_assert((unsigned char)GT_ARR_MAX_RANK == GT_ARR_MAX_RANK); - /* The new block will inherit this block's weight */ - step->inheritWeight(block); - step->bbFlags |= BBF_IMPORTED | BBF_KEEP_BBJ_ALWAYS; + GenTree* inds[GT_ARR_MAX_RANK]; + for (unsigned k = rank; k > 0; k--) + { + // The indices should be converted to `int` type, as they would be if the intrinsic was not expanded. + GenTree* argVal = impPopStack().val; + if (impInlineRoot()->opts.compJitEarlyExpandMDArrays) + { + // This is only enabled when early MD expansion is set because it causes small + // asm diffs (only in some test cases) otherwise. The GT_ARR_ELEM lowering code "accidentally" does + // this cast, but the new code requires it to be explicit. + argVal = impImplicitIorI4Cast(argVal, TYP_INT); + } + inds[k - 1] = argVal; + } -#ifdef DEBUG - if (verbose) - { - printf("impImportLeave - jumping out of a finally-protected try (EH#%u), created step (BBJ_ALWAYS) " - "block " FMT_BB "\n", - XTnum, step->bbNum); - } -#endif + GenTree* arr = impPopStack().val; + assert(arr->gtType == TYP_REF); - callBlock->bbJumpDest = HBtab->ebdHndBeg; // This callBlock will call the "finally" handler. + GenTree* arrElem = new (this, GT_ARR_ELEM) GenTreeArrElem(TYP_BYREF, arr, static_cast(rank), + static_cast(arrayElemSize), &inds[0]); - invalidatePreds = true; + if (intrinsicName != NI_Array_Address) + { + if (varTypeIsStruct(elemType)) + { + arrElem = gtNewObjNode(sig->retTypeClass, arrElem); } - else if (HBtab->HasCatchHandler() && jitIsBetween(blkAddr, tryBeg, tryEnd) && - !jitIsBetween(jmpAddr, tryBeg, tryEnd)) + else { - // We are jumping out of a catch-protected try. - // - // If we are returning from a call to a finally, then we must have a step block within a try - // that is protected by a catch. This is so when unwinding from that finally (e.g., if code within the - // finally raises an exception), the VM will find this step block, notice that it is in a protected region, - // and invoke the appropriate catch. - // - // We also need to handle a special case with the handling of ThreadAbortException. If a try/catch - // catches a ThreadAbortException (which might be because it catches a parent, e.g. System.Exception), - // and the catch doesn't call System.Threading.Thread::ResetAbort(), then when the catch returns to the VM, - // the VM will automatically re-raise the ThreadAbortException. When it does this, it uses the target - // address of the catch return as the new exception address. That is, the re-raised exception appears to - // occur at the catch return address. If this exception return address skips an enclosing try/catch that - // catches ThreadAbortException, then the enclosing try/catch will not catch the exception, as it should. - // For example: - // - // try { - // try { - // // something here raises ThreadAbortException - // LEAVE LABEL_1; // no need to stop at LABEL_2 - // } catch (Exception) { - // // This catches ThreadAbortException, but doesn't call System.Threading.Thread::ResetAbort(), so - // // ThreadAbortException is re-raised by the VM at the address specified by the LEAVE opcode. - // // This is bad, since it means the outer try/catch won't get a chance to catch the re-raised - // // ThreadAbortException. So, instead, create step block LABEL_2 and LEAVE to that. We only - // // need to do this transformation if the current EH block is a try/catch that catches - // // ThreadAbortException (or one of its parents), however we might not be able to find that - // // information, so currently we do it for all catch types. - // LEAVE LABEL_1; // Convert this to LEAVE LABEL2; - // } - // LABEL_2: LEAVE LABEL_1; // inserted by this step creation code - // } catch (ThreadAbortException) { - // } - // LABEL_1: - // - // Note that this pattern isn't theoretical: it occurs in ASP.NET, in IL code generated by the Roslyn C# - // compiler. + arrElem = gtNewOperNode(GT_IND, elemType, arrElem); + } + } - if ((stepType == ST_FinallyReturn) || (stepType == ST_Catch)) - { - BasicBlock* catchStep; + if (intrinsicName == NI_Array_Set) + { + assert(val != nullptr); + return gtNewAssignNode(arrElem, val); + } + else + { + return arrElem; + } +} - assert(step); +//------------------------------------------------------------------------ +// impKeepAliveIntrinsic: Import the GC.KeepAlive intrinsic call +// +// Imports the intrinsic as a GT_KEEPALIVE node, and, as an optimization, +// if the object to keep alive is a GT_BOX, removes its side effects and +// uses the address of a local (copied from the box's source if needed) +// as the operand for GT_KEEPALIVE. For the BOX optimization, if the class +// of the box has no GC fields, a GT_NOP is returned. +// +// Arguments: +// objToKeepAlive - the intrinisic call's argument +// +// Return Value: +// The imported GT_KEEPALIVE or GT_NOP - see description. +// +GenTree* Compiler::impKeepAliveIntrinsic(GenTree* objToKeepAlive) +{ + assert(objToKeepAlive->TypeIs(TYP_REF)); - if (stepType == ST_FinallyReturn) - { - assert(step->bbJumpKind == BBJ_ALWAYS); - } - else - { - assert(stepType == ST_Catch); - assert(step->bbJumpKind == BBJ_EHCATCHRET); - } + if (opts.OptimizationEnabled() && objToKeepAlive->IsBoxedValue()) + { + CORINFO_CLASS_HANDLE boxedClass = lvaGetDesc(objToKeepAlive->AsBox()->BoxOp()->AsLclVar())->lvClassHnd; + ClassLayout* layout = typGetObjLayout(boxedClass); - /* Create a new exit block in the try region for the existing step block to jump to in this scope */ - catchStep = fgNewBBinRegion(BBJ_ALWAYS, XTnum + 1, 0, step); - step->bbJumpDest = catchStep; - step->bbJumpDest->bbRefs++; + if (!layout->HasGCPtr()) + { + gtTryRemoveBoxUpstreamEffects(objToKeepAlive, BR_REMOVE_AND_NARROW); + JITDUMP("\nBOX class has no GC fields, KEEPALIVE is a NOP"); -#if defined(TARGET_ARM) - if (stepType == ST_FinallyReturn) - { - // Mark the target of a finally return - step->bbJumpDest->bbFlags |= BBF_FINALLY_TARGET; - } -#endif // defined(TARGET_ARM) + return gtNewNothingNode(); + } - /* The new block will inherit this block's weight */ - catchStep->inheritWeight(block); - catchStep->bbFlags |= BBF_IMPORTED; + GenTree* boxSrc = gtTryRemoveBoxUpstreamEffects(objToKeepAlive, BR_REMOVE_BUT_NOT_NARROW); + if (boxSrc != nullptr) + { + unsigned boxTempNum; + if (boxSrc->OperIs(GT_LCL_VAR)) + { + boxTempNum = boxSrc->AsLclVarCommon()->GetLclNum(); + } + else + { + boxTempNum = lvaGrabTemp(true DEBUGARG("Temp for the box source")); + GenTree* boxTempAsg = gtNewTempAssign(boxTempNum, boxSrc); + Statement* boxAsgStmt = objToKeepAlive->AsBox()->gtCopyStmtWhenInlinedBoxValue; + boxAsgStmt->SetRootNode(boxTempAsg); + } -#ifdef DEBUG - if (verbose) - { - if (stepType == ST_FinallyReturn) - { - printf("impImportLeave - return from finally jumping out of a catch-protected try (EH#%u), new " - "BBJ_ALWAYS block " FMT_BB "\n", - XTnum, catchStep->bbNum); - } - else - { - assert(stepType == ST_Catch); - printf("impImportLeave - return from catch jumping out of a catch-protected try (EH#%u), new " - "BBJ_ALWAYS block " FMT_BB "\n", - XTnum, catchStep->bbNum); - } - } -#endif // DEBUG + JITDUMP("\nImporting KEEPALIVE(BOX) as KEEPALIVE(ADDR(LCL_VAR V%02u))", boxTempNum); - /* This block is the new step */ - step = catchStep; - stepType = ST_Try; + GenTree* boxTemp = gtNewLclvNode(boxTempNum, boxSrc->TypeGet()); + GenTree* boxTempAddr = gtNewOperNode(GT_ADDR, TYP_BYREF, boxTemp); - invalidatePreds = true; - } + return gtNewKeepAliveNode(boxTempAddr); } } - if (step == nullptr) + return gtNewKeepAliveNode(objToKeepAlive); +} + +bool Compiler::verMergeEntryStates(BasicBlock* block, bool* changed) +{ + unsigned i; + + // do some basic checks first + if (block->bbStackDepthOnEntry() != verCurrentState.esStackDepth) { - block->bbJumpKind = BBJ_ALWAYS; // convert the BBJ_LEAVE to a BBJ_ALWAYS + return false; + } -#ifdef DEBUG - if (verbose) + if (verCurrentState.esStackDepth > 0) + { + // merge stack types + StackEntry* parentStack = block->bbStackOnEntry(); + StackEntry* childStack = verCurrentState.esStack; + + for (i = 0; i < verCurrentState.esStackDepth; i++, parentStack++, childStack++) { - printf("impImportLeave - no enclosing finally-protected try blocks or catch handlers; convert CEE_LEAVE " - "block " FMT_BB " to BBJ_ALWAYS\n", - block->bbNum); + if (tiMergeToCommonParent(&parentStack->seTypeInfo, &childStack->seTypeInfo, changed) == false) + { + return false; + } } -#endif } - else + + // merge initialization status of this ptr + + if (verTrackObjCtorInitState) { - step->bbJumpDest = leaveTarget; // this is the ultimate destination of the LEAVE + // If we're tracking the CtorInitState, then it must not be unknown in the current state. + assert(verCurrentState.thisInitialized != TIS_Bottom); -#if defined(TARGET_ARM) - if (stepType == ST_FinallyReturn) + // If the successor block's thisInit state is unknown, copy it from the current state. + if (block->bbThisOnEntry() == TIS_Bottom) { - assert(step->bbJumpKind == BBJ_ALWAYS); - // Mark the target of a finally return - step->bbJumpDest->bbFlags |= BBF_FINALLY_TARGET; + *changed = true; + verSetThisInit(block, verCurrentState.thisInitialized); } -#endif // defined(TARGET_ARM) - -#ifdef DEBUG - if (verbose) + else if (verCurrentState.thisInitialized != block->bbThisOnEntry()) { - printf("impImportLeave - final destination of step blocks set to " FMT_BB "\n", leaveTarget->bbNum); - } -#endif - - // Queue up the jump target for importing + if (block->bbThisOnEntry() != TIS_Top) + { + *changed = true; + verSetThisInit(block, TIS_Top); - impImportBlockPending(leaveTarget); + if (block->bbFlags & BBF_FAILED_VERIFICATION) + { + // The block is bad. Control can flow through the block to any handler that catches the + // verification exception, but the importer ignores bad blocks and therefore won't model + // this flow in the normal way. To complete the merge into the bad block, the new state + // needs to be manually pushed to the handlers that may be reached after the verification + // exception occurs. + // + // Usually, the new state was already propagated to the relevant handlers while processing + // the predecessors of the bad block. The exception is when the bad block is at the start + // of a try region, meaning it is protected by additional handlers that do not protect its + // predecessors. + // + if (block->hasTryIndex() && ((block->bbFlags & BBF_TRY_BEG) != 0)) + { + // Push TIS_Top to the handlers that protect the bad block. Note that this can cause + // recursive calls back into this code path (if successors of the current bad block are + // also bad blocks). + // + ThisInitState origTIS = verCurrentState.thisInitialized; + verCurrentState.thisInitialized = TIS_Top; + impVerifyEHBlock(block, true); + verCurrentState.thisInitialized = origTIS; + } + } + } + } } - - if (invalidatePreds && fgComputePredsDone) + else { - JITDUMP("\n**** impImportLeave - Removing preds after creating new blocks\n"); - fgRemovePreds(); + assert(verCurrentState.thisInitialized == TIS_Bottom && block->bbThisOnEntry() == TIS_Bottom); } -#ifdef DEBUG - fgVerifyHandlerTab(); - - if (verbose) - { - printf("\nAfter import CEE_LEAVE:\n"); - fgDispBasicBlocks(); - fgDispHandlerTab(); - } -#endif // DEBUG + return true; } -#endif // FEATURE_EH_FUNCLETS - -/*****************************************************************************/ -// This is called when reimporting a leave block. It resets the JumpKind, -// JumpDest, and bbNext to the original values +/***************************************************************************** + * 'logMsg' is true if a log message needs to be logged. false if the caller has + * already logged it (presumably in a more detailed fashion than done here) + */ -void Compiler::impResetLeaveBlock(BasicBlock* block, unsigned jmpAddr) +void Compiler::verConvertBBToThrowVerificationException(BasicBlock* block DEBUGARG(bool logMsg)) { -#if defined(FEATURE_EH_FUNCLETS) - // With EH Funclets, while importing leave opcode we create another block ending with BBJ_ALWAYS (call it B1) - // and the block containing leave (say B0) is marked as BBJ_CALLFINALLY. Say for some reason we reimport B0, - // it is reset (in this routine) by marking as ending with BBJ_LEAVE and further down when B0 is reimported, we - // create another BBJ_ALWAYS (call it B2). In this process B1 gets orphaned and any blocks to which B1 is the - // only predecessor are also considered orphans and attempted to be deleted. - // - // try { - // .... - // try - // { - // .... - // leave OUTSIDE; // B0 is the block containing this leave, following this would be B1 - // } finally { } - // } finally { } - // OUTSIDE: - // - // In the above nested try-finally example, we create a step block (call it Bstep) which in branches to a block - // where a finally would branch to (and such block is marked as finally target). Block B1 branches to step block. - // Because of re-import of B0, Bstep is also orphaned. Since Bstep is a finally target it cannot be removed. To - // work around this we will duplicate B0 (call it B0Dup) before resetting. B0Dup is marked as BBJ_CALLFINALLY and - // only serves to pair up with B1 (BBJ_ALWAYS) that got orphaned. Now during orphan block deletion B0Dup and B1 - // will be treated as pair and handled correctly. - if (block->bbJumpKind == BBJ_CALLFINALLY) - { - BasicBlock* dupBlock = bbNewBasicBlock(block->bbJumpKind); - dupBlock->bbFlags = block->bbFlags; - dupBlock->bbJumpDest = block->bbJumpDest; - dupBlock->copyEHRegion(block); - dupBlock->bbCatchTyp = block->bbCatchTyp; + block->bbJumpKind = BBJ_THROW; + block->bbFlags |= BBF_FAILED_VERIFICATION; + block->bbFlags &= ~BBF_IMPORTED; - // Mark this block as - // a) not referenced by any other block to make sure that it gets deleted - // b) weight zero - // c) prevent from being imported - // d) as internal - // e) as rarely run - dupBlock->bbRefs = 0; - dupBlock->bbWeight = BB_ZERO_WEIGHT; - dupBlock->bbFlags |= BBF_IMPORTED | BBF_INTERNAL | BBF_RUN_RARELY; + impCurStmtOffsSet(block->bbCodeOffs); - // Insert the block right after the block which is getting reset so that BBJ_CALLFINALLY and BBJ_ALWAYS - // will be next to each other. - fgInsertBBafter(block, dupBlock); + // Clear the statement list as it exists so far; we're only going to have a verification exception. + impStmtList = impLastStmt = nullptr; #ifdef DEBUG + if (logMsg) + { + JITLOG((LL_ERROR, "Verification failure: while compiling %s near IL offset %x..%xh \n", info.compFullName, + block->bbCodeOffs, block->bbCodeOffsEnd)); if (verbose) { - printf("New Basic Block " FMT_BB " duplicate of " FMT_BB " created.\n", dupBlock->bbNum, block->bbNum); + printf("\n\nVerification failure: %s near IL %xh \n", info.compFullName, block->bbCodeOffs); } -#endif } -#endif // FEATURE_EH_FUNCLETS - - block->bbJumpKind = BBJ_LEAVE; - fgInitBBLookup(); - block->bbJumpDest = fgLookupBB(jmpAddr); - // We will leave the BBJ_ALWAYS block we introduced. When it's reimported - // the BBJ_ALWAYS block will be unreachable, and will be removed after. The - // reason we don't want to remove the block at this point is that if we call - // fgInitBBLookup() again we will do it wrong as the BBJ_ALWAYS block won't be - // added and the linked list length will be different than fgBBcount. -} + if (JitConfig.DebugBreakOnVerificationFailure()) + { + DebugBreak(); + } +#endif -/*****************************************************************************/ -// Get the first non-prefix opcode. Used for verification of valid combinations -// of prefixes and actual opcodes. + impBeginTreeList(); -OPCODE Compiler::impGetNonPrefixOpcode(const BYTE* codeAddr, const BYTE* codeEndp) -{ - while (codeAddr < codeEndp) + // if the stack is non-empty evaluate all the side-effects + if (verCurrentState.esStackDepth > 0) { - OPCODE opcode = (OPCODE)getU1LittleEndian(codeAddr); - codeAddr += sizeof(__int8); + impEvalSideEffects(); + } + assert(verCurrentState.esStackDepth == 0); - if (opcode == CEE_PREFIX1) - { - if (codeAddr >= codeEndp) + GenTree* op1 = gtNewHelperCallNode(CORINFO_HELP_VERIFICATION, TYP_VOID, gtNewIconNode(block->bbCodeOffs)); + // verCurrentState.esStackDepth = 0; + impAppendTree(op1, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); + + // The inliner is not able to handle methods that require throw block, so + // make sure this methods never gets inlined. + info.compCompHnd->setMethodAttribs(info.compMethodHnd, CORINFO_FLG_BAD_INLINEE); +} + +/***************************************************************************** + * + */ +void Compiler::verHandleVerificationFailure(BasicBlock* block DEBUGARG(bool logMsg)) +{ + verResetCurrentState(block, &verCurrentState); + verConvertBBToThrowVerificationException(block DEBUGARG(logMsg)); + +#ifdef DEBUG + impNoteLastILoffs(); // Remember at which BC offset the tree was finished +#endif // DEBUG +} + +/******************************************************************************/ +typeInfo Compiler::verMakeTypeInfo(CorInfoType ciType, CORINFO_CLASS_HANDLE clsHnd) +{ + assert(ciType < CORINFO_TYPE_COUNT); + + typeInfo tiResult; + switch (ciType) + { + case CORINFO_TYPE_STRING: + case CORINFO_TYPE_CLASS: + tiResult = verMakeTypeInfo(clsHnd); + if (!tiResult.IsType(TI_REF)) + { // type must be consistent with element type + return typeInfo(); + } + break; + +#ifdef TARGET_64BIT + case CORINFO_TYPE_NATIVEINT: + case CORINFO_TYPE_NATIVEUINT: + if (clsHnd) { - break; + // If we have more precise information, use it + return verMakeTypeInfo(clsHnd); } - opcode = (OPCODE)(getU1LittleEndian(codeAddr) + 256); - codeAddr += sizeof(__int8); - } + else + { + return typeInfo::nativeInt(); + } + break; +#endif // TARGET_64BIT - switch (opcode) + case CORINFO_TYPE_VALUECLASS: + case CORINFO_TYPE_REFANY: + tiResult = verMakeTypeInfo(clsHnd); + // type must be constant with element type; + if (!tiResult.IsValueClass()) + { + return typeInfo(); + } + break; + case CORINFO_TYPE_VAR: + return verMakeTypeInfo(clsHnd); + + case CORINFO_TYPE_PTR: // for now, pointers are treated as an error + case CORINFO_TYPE_VOID: + return typeInfo(); + break; + + case CORINFO_TYPE_BYREF: { - case CEE_UNALIGNED: - case CEE_VOLATILE: - case CEE_TAILCALL: - case CEE_CONSTRAINED: - case CEE_READONLY: - break; - default: - return opcode; + CORINFO_CLASS_HANDLE childClassHandle; + CorInfoType childType = info.compCompHnd->getChildType(clsHnd, &childClassHandle); + return ByRef(verMakeTypeInfo(childType, childClassHandle)); } + break; - codeAddr += opcodeSizes[opcode]; + default: + if (clsHnd) + { // If we have more precise information, use it + return typeInfo(TI_STRUCT, clsHnd); + } + else + { + return typeInfo(JITtype2tiType(ciType)); + } } - - return CEE_ILLEGAL; + return tiResult; } -/*****************************************************************************/ -// Checks whether the opcode is a valid opcode for volatile. and unaligned. prefixes +/******************************************************************************/ -void Compiler::impValidateMemoryAccessOpcode(const BYTE* codeAddr, const BYTE* codeEndp, bool volatilePrefix) +typeInfo Compiler::verMakeTypeInfo(CORINFO_CLASS_HANDLE clsHnd, bool bashStructToRef /* = false */) { - OPCODE opcode = impGetNonPrefixOpcode(codeAddr, codeEndp); - - if (!( - // Opcode of all ldind and stdind happen to be in continuous, except stind.i. - ((CEE_LDIND_I1 <= opcode) && (opcode <= CEE_STIND_R8)) || (opcode == CEE_STIND_I) || - (opcode == CEE_LDFLD) || (opcode == CEE_STFLD) || (opcode == CEE_LDOBJ) || (opcode == CEE_STOBJ) || - (opcode == CEE_INITBLK) || (opcode == CEE_CPBLK) || - // volatile. prefix is allowed with the ldsfld and stsfld - (volatilePrefix && ((opcode == CEE_LDSFLD) || (opcode == CEE_STSFLD))))) + if (clsHnd == nullptr) { - BADCODE("Invalid opcode for unaligned. or volatile. prefix"); + return typeInfo(); } -} -/***************************************************************************** - * Determine the result type of an arithmetic operation - * On 64-bit inserts upcasts when native int is mixed with int32 - */ -var_types Compiler::impGetByRefResultType(genTreeOps oper, bool fUnsigned, GenTree** pOp1, GenTree** pOp2) -{ - var_types type = TYP_UNDEF; - GenTree* op1 = *pOp1; - GenTree* op2 = *pOp2; + // Byrefs should only occur in method and local signatures, which are accessed + // using ICorClassInfo and ICorClassInfo.getChildType. + // So findClass() and getClassAttribs() should not be called for byrefs - // Arithmetic operations are generally only allowed with - // primitive types, but certain operations are allowed - // with byrefs + if (JITtype2varType(info.compCompHnd->asCorInfoType(clsHnd)) == TYP_BYREF) + { + assert(!"Did findClass() return a Byref?"); + return typeInfo(); + } - if ((oper == GT_SUB) && (genActualType(op1->TypeGet()) == TYP_BYREF || genActualType(op2->TypeGet()) == TYP_BYREF)) + unsigned attribs = info.compCompHnd->getClassAttribs(clsHnd); + + if (attribs & CORINFO_FLG_VALUECLASS) { - if ((genActualType(op1->TypeGet()) == TYP_BYREF) && (genActualType(op2->TypeGet()) == TYP_BYREF)) + CorInfoType t = info.compCompHnd->getTypeForPrimitiveValueClass(clsHnd); + + // Meta-data validation should ensure that CORINF_TYPE_BYREF should + // not occur here, so we may want to change this to an assert instead. + if (t == CORINFO_TYPE_VOID || t == CORINFO_TYPE_BYREF || t == CORINFO_TYPE_PTR) { - // byref1-byref2 => gives a native int - type = TYP_I_IMPL; + return typeInfo(); } - else if (genActualTypeIsIntOrI(op1->TypeGet()) && (genActualType(op2->TypeGet()) == TYP_BYREF)) - { - // [native] int - byref => gives a native int - - // - // The reason is that it is possible, in managed C++, - // to have a tree like this: - // - // - - // / \. - // / \. - // / \. - // / \. - // const(h) int addr byref - // - // VSW 318822 - // - // So here we decide to make the resulting type to be a native int. - CLANG_FORMAT_COMMENT_ANCHOR; #ifdef TARGET_64BIT - if (genActualType(op1->TypeGet()) != TYP_I_IMPL) - { - // insert an explicit upcast - op1 = *pOp1 = gtNewCastNode(TYP_I_IMPL, op1, fUnsigned, fUnsigned ? TYP_U_IMPL : TYP_I_IMPL); - } + if (t == CORINFO_TYPE_NATIVEINT || t == CORINFO_TYPE_NATIVEUINT) + { + return typeInfo::nativeInt(); + } #endif // TARGET_64BIT - type = TYP_I_IMPL; + if (t != CORINFO_TYPE_UNDEF) + { + return (typeInfo(JITtype2tiType(t))); + } + else if (bashStructToRef) + { + return (typeInfo(TI_REF, clsHnd)); } else { - // byref - [native] int => gives a byref - assert(genActualType(op1->TypeGet()) == TYP_BYREF && genActualTypeIsIntOrI(op2->TypeGet())); - -#ifdef TARGET_64BIT - if ((genActualType(op2->TypeGet()) != TYP_I_IMPL)) - { - // insert an explicit upcast - op2 = *pOp2 = gtNewCastNode(TYP_I_IMPL, op2, fUnsigned, fUnsigned ? TYP_U_IMPL : TYP_I_IMPL); - } -#endif // TARGET_64BIT - - type = TYP_BYREF; + return (typeInfo(TI_STRUCT, clsHnd)); } } - else if ((oper == GT_ADD) && - (genActualType(op1->TypeGet()) == TYP_BYREF || genActualType(op2->TypeGet()) == TYP_BYREF)) + else if (attribs & CORINFO_FLG_GENERIC_TYPE_VARIABLE) { - // byref + [native] int => gives a byref - // (or) - // [native] int + byref => gives a byref + // See comment in _typeInfo.h for why we do it this way. + return (typeInfo(TI_REF, clsHnd, true)); + } + else + { + return (typeInfo(TI_REF, clsHnd)); + } +} - // only one can be a byref : byref op byref not allowed - assert(genActualType(op1->TypeGet()) != TYP_BYREF || genActualType(op2->TypeGet()) != TYP_BYREF); - assert(genActualTypeIsIntOrI(op1->TypeGet()) || genActualTypeIsIntOrI(op2->TypeGet())); +/******************************************************************************/ +bool Compiler::verIsSDArray(const typeInfo& ti) +{ + if (ti.IsNullObjRef()) + { // nulls are SD arrays + return true; + } -#ifdef TARGET_64BIT - if (genActualType(op2->TypeGet()) == TYP_BYREF) - { - if (genActualType(op1->TypeGet()) != TYP_I_IMPL) - { - // insert an explicit upcast - op1 = *pOp1 = gtNewCastNode(TYP_I_IMPL, op1, fUnsigned, fUnsigned ? TYP_U_IMPL : TYP_I_IMPL); - } - } - else if (genActualType(op2->TypeGet()) != TYP_I_IMPL) - { - // insert an explicit upcast - op2 = *pOp2 = gtNewCastNode(TYP_I_IMPL, op2, fUnsigned, fUnsigned ? TYP_U_IMPL : TYP_I_IMPL); - } -#endif // TARGET_64BIT + if (!ti.IsType(TI_REF)) + { + return false; + } - type = TYP_BYREF; + if (!info.compCompHnd->isSDArray(ti.GetClassHandleForObjRef())) + { + return false; } -#ifdef TARGET_64BIT - else if (genActualType(op1->TypeGet()) == TYP_I_IMPL || genActualType(op2->TypeGet()) == TYP_I_IMPL) + return true; +} + +/******************************************************************************/ +/* Given 'arrayObjectType' which is an array type, fetch the element type. */ +/* Returns an error type if anything goes wrong */ + +typeInfo Compiler::verGetArrayElemType(const typeInfo& arrayObjectType) +{ + assert(!arrayObjectType.IsNullObjRef()); // you need to check for null explicitly since that is a success case + + if (!verIsSDArray(arrayObjectType)) { - assert(!varTypeIsFloating(op1->gtType) && !varTypeIsFloating(op2->gtType)); + return typeInfo(); + } + + CORINFO_CLASS_HANDLE childClassHandle = nullptr; + CorInfoType ciType = info.compCompHnd->getChildType(arrayObjectType.GetClassHandleForObjRef(), &childClassHandle); + + return verMakeTypeInfo(ciType, childClassHandle); +} + +/***************************************************************************** + */ +typeInfo Compiler::verParseArgSigToTypeInfo(CORINFO_SIG_INFO* sig, CORINFO_ARG_LIST_HANDLE args) +{ + CORINFO_CLASS_HANDLE classHandle; + CorInfoType ciType = strip(info.compCompHnd->getArgType(sig, args, &classHandle)); + + var_types type = JITtype2varType(ciType); + if (varTypeIsGC(type)) + { + // For efficiency, getArgType only returns something in classHandle for + // value types. For other types that have addition type info, you + // have to call back explicitly + classHandle = info.compCompHnd->getArgClass(sig, args); + if (!classHandle) + { + NO_WAY("Could not figure out Class specified in argument or local signature"); + } + } + + return verMakeTypeInfo(ciType, classHandle); +} + +bool Compiler::verIsByRefLike(const typeInfo& ti) +{ + if (ti.IsByRef()) + { + return true; + } + if (!ti.IsType(TI_STRUCT)) + { + return false; + } + return info.compCompHnd->getClassAttribs(ti.GetClassHandleForValueClass()) & CORINFO_FLG_BYREF_LIKE; +} + +bool Compiler::verIsSafeToReturnByRef(const typeInfo& ti) +{ + if (ti.IsPermanentHomeByRef()) + { + return true; + } + else + { + return false; + } +} + +bool Compiler::verIsBoxable(const typeInfo& ti) +{ + return (ti.IsPrimitiveType() || ti.IsObjRef() // includes boxed generic type variables + || ti.IsUnboxedGenericTypeVar() || + (ti.IsType(TI_STRUCT) && + // exclude byreflike structs + !(info.compCompHnd->getClassAttribs(ti.GetClassHandleForValueClass()) & CORINFO_FLG_BYREF_LIKE))); +} + +// Is it a boxed value type? +bool Compiler::verIsBoxedValueType(const typeInfo& ti) +{ + if (ti.GetType() == TI_REF) + { + CORINFO_CLASS_HANDLE clsHnd = ti.GetClassHandleForObjRef(); + return !!eeIsValueClass(clsHnd); + } + else + { + return false; + } +} + +/***************************************************************************** + * + * Check if a TailCall is legal. + */ + +bool Compiler::verCheckTailCallConstraint( + OPCODE opcode, + CORINFO_RESOLVED_TOKEN* pResolvedToken, + CORINFO_RESOLVED_TOKEN* pConstrainedResolvedToken, // Is this a "constrained." call on a type parameter? + bool speculative // If true, won't throw if verificatoin fails. Instead it will + // return false to the caller. + // If false, it will throw. + ) +{ + DWORD mflags; + CORINFO_SIG_INFO sig; + unsigned int popCount = 0; // we can't pop the stack since impImportCall needs it, so + // this counter is used to keep track of how many items have been + // virtually popped + + CORINFO_METHOD_HANDLE methodHnd = nullptr; + CORINFO_CLASS_HANDLE methodClassHnd = nullptr; + unsigned methodClassFlgs = 0; + + assert(impOpcodeIsCallOpcode(opcode)); + + if (compIsForInlining()) + { + return false; + } + + // for calli, VerifyOrReturn that this is not a virtual method + if (opcode == CEE_CALLI) + { + /* Get the call sig */ + eeGetSig(pResolvedToken->token, pResolvedToken->tokenScope, pResolvedToken->tokenContext, &sig); + + // We don't know the target method, so we have to infer the flags, or + // assume the worst-case. + mflags = (sig.callConv & CORINFO_CALLCONV_HASTHIS) ? 0 : CORINFO_FLG_STATIC; + } + else + { + methodHnd = pResolvedToken->hMethod; + + mflags = info.compCompHnd->getMethodAttribs(methodHnd); + + // When verifying generic code we pair the method handle with its + // owning class to get the exact method signature. + methodClassHnd = pResolvedToken->hClass; + assert(methodClassHnd); + + eeGetMethodSig(methodHnd, &sig, methodClassHnd); + + // opcode specific check + methodClassFlgs = info.compCompHnd->getClassAttribs(methodClassHnd); + } + + // We must have got the methodClassHnd if opcode is not CEE_CALLI + assert((methodHnd != nullptr && methodClassHnd != nullptr) || opcode == CEE_CALLI); + + if ((sig.callConv & CORINFO_CALLCONV_MASK) == CORINFO_CALLCONV_VARARG) + { + eeGetCallSiteSig(pResolvedToken->token, pResolvedToken->tokenScope, pResolvedToken->tokenContext, &sig); + } + + // check compatibility of the arguments + unsigned int argCount; + argCount = sig.numArgs; + CORINFO_ARG_LIST_HANDLE args; + args = sig.args; + while (argCount--) + { + typeInfo tiDeclared = verParseArgSigToTypeInfo(&sig, args).NormaliseForStack(); + + // check that the argument is not a byref for tailcalls + VerifyOrReturnSpeculative(!verIsByRefLike(tiDeclared), "tailcall on byrefs", speculative); + + // For unsafe code, we might have parameters containing pointer to the stack location. + // Disallow the tailcall for this kind. + CORINFO_CLASS_HANDLE classHandle; + CorInfoType ciType = strip(info.compCompHnd->getArgType(&sig, args, &classHandle)); + VerifyOrReturnSpeculative(ciType != CORINFO_TYPE_PTR, "tailcall on CORINFO_TYPE_PTR", speculative); + + args = info.compCompHnd->getArgNext(args); + } + + // update popCount + popCount += sig.numArgs; + + // check for 'this' which is on non-static methods, not called via NEWOBJ + if (!(mflags & CORINFO_FLG_STATIC)) + { + // Always update the popCount. + // This is crucial for the stack calculation to be correct. + typeInfo tiThis = impStackTop(popCount).seTypeInfo; + popCount++; + + if (opcode == CEE_CALLI) + { + // For CALLI, we don't know the methodClassHnd. Therefore, let's check the "this" object + // on the stack. + if (tiThis.IsValueClass()) + { + tiThis.MakeByRef(); + } + VerifyOrReturnSpeculative(!verIsByRefLike(tiThis), "byref in tailcall", speculative); + } + else + { + // Check type compatibility of the this argument + typeInfo tiDeclaredThis = verMakeTypeInfo(methodClassHnd); + if (tiDeclaredThis.IsValueClass()) + { + tiDeclaredThis.MakeByRef(); + } + + VerifyOrReturnSpeculative(!verIsByRefLike(tiDeclaredThis), "byref in tailcall", speculative); + } + } + + // Tail calls on constrained calls should be illegal too: + // when instantiated at a value type, a constrained call may pass the address of a stack allocated value + VerifyOrReturnSpeculative(!pConstrainedResolvedToken, "byref in constrained tailcall", speculative); + + // Get the exact view of the signature for an array method + if (sig.retType != CORINFO_TYPE_VOID) + { + if (methodClassFlgs & CORINFO_FLG_ARRAY) + { + assert(opcode != CEE_CALLI); + eeGetCallSiteSig(pResolvedToken->token, pResolvedToken->tokenScope, pResolvedToken->tokenContext, &sig); + } + } + + typeInfo tiCalleeRetType = verMakeTypeInfo(sig.retType, sig.retTypeClass); + typeInfo tiCallerRetType = + verMakeTypeInfo(info.compMethodInfo->args.retType, info.compMethodInfo->args.retTypeClass); + + // void return type gets morphed into the error type, so we have to treat them specially here + if (sig.retType == CORINFO_TYPE_VOID) + { + VerifyOrReturnSpeculative(info.compMethodInfo->args.retType == CORINFO_TYPE_VOID, "tailcall return mismatch", + speculative); + } + else + { + VerifyOrReturnSpeculative(tiCompatibleWith(NormaliseForStack(tiCalleeRetType), + NormaliseForStack(tiCallerRetType), true), + "tailcall return mismatch", speculative); + } + + // for tailcall, stack must be empty + VerifyOrReturnSpeculative(verCurrentState.esStackDepth == popCount, "stack non-empty on tailcall", speculative); + + return true; // Yes, tailcall is legal +} + +/***************************************************************************** + * + * Checks the IL verification rules for the call + */ + +void Compiler::verVerifyCall(OPCODE opcode, + CORINFO_RESOLVED_TOKEN* pResolvedToken, + CORINFO_RESOLVED_TOKEN* pConstrainedResolvedToken, + bool tailCall, + bool readonlyCall, + const BYTE* delegateCreateStart, + const BYTE* codeAddr, + CORINFO_CALL_INFO* callInfo DEBUGARG(const char* methodName)) +{ + DWORD mflags; + CORINFO_SIG_INFO* sig = nullptr; + unsigned int popCount = 0; // we can't pop the stack since impImportCall needs it, so + // this counter is used to keep track of how many items have been + // virtually popped + + // for calli, VerifyOrReturn that this is not a virtual method + if (opcode == CEE_CALLI) + { + Verify(false, "Calli not verifiable"); + return; + } + + // It would be nice to cache the rest of it, but eeFindMethod is the big ticket item. + mflags = callInfo->verMethodFlags; + + sig = &callInfo->verSig; + + if ((sig->callConv & CORINFO_CALLCONV_MASK) == CORINFO_CALLCONV_VARARG) + { + eeGetCallSiteSig(pResolvedToken->token, pResolvedToken->tokenScope, pResolvedToken->tokenContext, sig); + } + + // opcode specific check + unsigned methodClassFlgs = callInfo->classFlags; + switch (opcode) + { + case CEE_CALLVIRT: + // cannot do callvirt on valuetypes + VerifyOrReturn(!(methodClassFlgs & CORINFO_FLG_VALUECLASS), "callVirt on value class"); + VerifyOrReturn(sig->hasThis(), "CallVirt on static method"); + break; + + case CEE_NEWOBJ: + { + assert(!tailCall); // Importer should not allow this + VerifyOrReturn((mflags & CORINFO_FLG_CONSTRUCTOR) && !(mflags & CORINFO_FLG_STATIC), + "newobj must be on instance"); + + if (methodClassFlgs & CORINFO_FLG_DELEGATE) + { + VerifyOrReturn(sig->numArgs == 2, "wrong number args to delegate ctor"); + typeInfo tiDeclaredObj = verParseArgSigToTypeInfo(sig, sig->args).NormaliseForStack(); + typeInfo tiDeclaredFtn = + verParseArgSigToTypeInfo(sig, info.compCompHnd->getArgNext(sig->args)).NormaliseForStack(); + VerifyOrReturn(tiDeclaredFtn.IsNativeIntType(), "ftn arg needs to be a native int type"); + + assert(popCount == 0); + typeInfo tiActualObj = impStackTop(1).seTypeInfo; + typeInfo tiActualFtn = impStackTop(0).seTypeInfo; + + VerifyOrReturn(tiActualFtn.IsMethod(), "delegate needs method as first arg"); + VerifyOrReturn(tiCompatibleWith(tiActualObj, tiDeclaredObj, true), "delegate object type mismatch"); + VerifyOrReturn(tiActualObj.IsNullObjRef() || tiActualObj.IsType(TI_REF), + "delegate object type mismatch"); + + CORINFO_CLASS_HANDLE objTypeHandle = + tiActualObj.IsNullObjRef() ? nullptr : tiActualObj.GetClassHandleForObjRef(); + + // the method signature must be compatible with the delegate's invoke method + + // check that for virtual functions, the type of the object used to get the + // ftn ptr is the same as the type of the object passed to the delegate ctor. + // since this is a bit of work to determine in general, we pattern match stylized + // code sequences + + // the delegate creation code check, which used to be done later, is now done here + // so we can read delegateMethodRef directly from + // from the preceding LDFTN or CEE_LDVIRTFN instruction sequence; + // we then use it in our call to isCompatibleDelegate(). + + mdMemberRef delegateMethodRef = mdMemberRefNil; + VerifyOrReturn(verCheckDelegateCreation(delegateCreateStart, codeAddr, delegateMethodRef), + "must create delegates with certain IL"); + + CORINFO_RESOLVED_TOKEN delegateResolvedToken; + delegateResolvedToken.tokenContext = impTokenLookupContextHandle; + delegateResolvedToken.tokenScope = info.compScopeHnd; + delegateResolvedToken.token = delegateMethodRef; + delegateResolvedToken.tokenType = CORINFO_TOKENKIND_Method; + info.compCompHnd->resolveToken(&delegateResolvedToken); + + CORINFO_CALL_INFO delegateCallInfo; + eeGetCallInfo(&delegateResolvedToken, nullptr /* constraint typeRef */, CORINFO_CALLINFO_SECURITYCHECKS, + &delegateCallInfo); + + bool isOpenDelegate = false; + VerifyOrReturn(info.compCompHnd->isCompatibleDelegate(objTypeHandle, delegateResolvedToken.hClass, + tiActualFtn.GetMethod(), pResolvedToken->hClass, + &isOpenDelegate), + "function incompatible with delegate"); + + // check the constraints on the target method + VerifyOrReturn(info.compCompHnd->satisfiesClassConstraints(delegateResolvedToken.hClass), + "delegate target has unsatisfied class constraints"); + VerifyOrReturn(info.compCompHnd->satisfiesMethodConstraints(delegateResolvedToken.hClass, + tiActualFtn.GetMethod()), + "delegate target has unsatisfied method constraints"); + + // See ECMA spec section 1.8.1.5.2 (Delegating via instance dispatch) + // for additional verification rules for delegates + CORINFO_METHOD_HANDLE actualMethodHandle = tiActualFtn.GetMethod(); + DWORD actualMethodAttribs = info.compCompHnd->getMethodAttribs(actualMethodHandle); + if (impIsLDFTN_TOKEN(delegateCreateStart, codeAddr)) + { + + if ((actualMethodAttribs & CORINFO_FLG_VIRTUAL) && ((actualMethodAttribs & CORINFO_FLG_FINAL) == 0)) + { + VerifyOrReturn((tiActualObj.IsThisPtr() && lvaIsOriginalThisReadOnly()) || + verIsBoxedValueType(tiActualObj), + "The 'this' parameter to the call must be either the calling method's " + "'this' parameter or " + "a boxed value type."); + } + } + + if (actualMethodAttribs & CORINFO_FLG_PROTECTED) + { + bool targetIsStatic = actualMethodAttribs & CORINFO_FLG_STATIC; + + Verify(targetIsStatic || !isOpenDelegate, + "Unverifiable creation of an open instance delegate for a protected member."); + + CORINFO_CLASS_HANDLE instanceClassHnd = (tiActualObj.IsNullObjRef() || targetIsStatic) + ? info.compClassHnd + : tiActualObj.GetClassHandleForObjRef(); + + // In the case of protected methods, it is a requirement that the 'this' + // pointer be a subclass of the current context. Perform this check. + Verify(info.compCompHnd->canAccessFamily(info.compMethodHnd, instanceClassHnd), + "Accessing protected method through wrong type."); + } + goto DONE_ARGS; + } + } + // fall thru to default checks + FALLTHROUGH; + default: + VerifyOrReturn(!(mflags & CORINFO_FLG_ABSTRACT), "method abstract"); + } + VerifyOrReturn(!((mflags & CORINFO_FLG_CONSTRUCTOR) && (methodClassFlgs & CORINFO_FLG_DELEGATE)), + "can only newobj a delegate constructor"); + + // check compatibility of the arguments + unsigned int argCount; + argCount = sig->numArgs; + CORINFO_ARG_LIST_HANDLE args; + args = sig->args; + while (argCount--) + { + typeInfo tiActual = impStackTop(popCount + argCount).seTypeInfo; + + typeInfo tiDeclared = verParseArgSigToTypeInfo(sig, args).NormaliseForStack(); + VerifyOrReturn(tiCompatibleWith(tiActual, tiDeclared, true), "type mismatch"); + + args = info.compCompHnd->getArgNext(args); + } + +DONE_ARGS: + + // update popCount + popCount += sig->numArgs; + + // check for 'this' which are is non-static methods, not called via NEWOBJ + CORINFO_CLASS_HANDLE instanceClassHnd = info.compClassHnd; + if (!(mflags & CORINFO_FLG_STATIC) && (opcode != CEE_NEWOBJ)) + { + typeInfo tiThis = impStackTop(popCount).seTypeInfo; + popCount++; + + // If it is null, we assume we can access it (since it will AV shortly) + // If it is anything but a reference class, there is no hierarchy, so + // again, we don't need the precise instance class to compute 'protected' access + if (tiThis.IsType(TI_REF)) + { + instanceClassHnd = tiThis.GetClassHandleForObjRef(); + } + + // Check type compatibility of the this argument + typeInfo tiDeclaredThis = verMakeTypeInfo(pResolvedToken->hClass); + if (tiDeclaredThis.IsValueClass()) + { + tiDeclaredThis.MakeByRef(); + } + + // If this is a call to the base class .ctor, set thisPtr Init for + // this block. + if (mflags & CORINFO_FLG_CONSTRUCTOR) + { + if (verTrackObjCtorInitState && tiThis.IsThisPtr() && + verIsCallToInitThisPtr(info.compClassHnd, pResolvedToken->hClass)) + { + assert(verCurrentState.thisInitialized != + TIS_Bottom); // This should never be the case just from the logic of the verifier. + VerifyOrReturn(verCurrentState.thisInitialized == TIS_Uninit, + "Call to base class constructor when 'this' is possibly initialized"); + // Otherwise, 'this' is now initialized. + verCurrentState.thisInitialized = TIS_Init; + tiThis.SetInitialisedObjRef(); + } + else + { + // We allow direct calls to value type constructors + // NB: we have to check that the contents of tiThis is a value type, otherwise we could use a + // constrained callvirt to illegally re-enter a .ctor on a value of reference type. + VerifyOrReturn(tiThis.IsByRef() && DereferenceByRef(tiThis).IsValueClass(), + "Bad call to a constructor"); + } + } + + if (pConstrainedResolvedToken != nullptr) + { + VerifyOrReturn(tiThis.IsByRef(), "non-byref this type in constrained call"); + + typeInfo tiConstraint = verMakeTypeInfo(pConstrainedResolvedToken->hClass); + + // We just dereference this and test for equality + tiThis.DereferenceByRef(); + VerifyOrReturn(typeInfo::AreEquivalent(tiThis, tiConstraint), + "this type mismatch with constrained type operand"); + + // Now pretend the this type is the boxed constrained type, for the sake of subsequent checks + tiThis = typeInfo(TI_REF, pConstrainedResolvedToken->hClass); + } + + // To support direct calls on readonly byrefs, just pretend tiDeclaredThis is readonly too + if (tiDeclaredThis.IsByRef() && tiThis.IsReadonlyByRef()) + { + tiDeclaredThis.SetIsReadonlyByRef(); + } + + VerifyOrReturn(tiCompatibleWith(tiThis, tiDeclaredThis, true), "this type mismatch"); + + if (tiThis.IsByRef()) + { + // Find the actual type where the method exists (as opposed to what is declared + // in the metadata). This is to prevent passing a byref as the "this" argument + // while calling methods like System.ValueType.GetHashCode() which expect boxed objects. + + CORINFO_CLASS_HANDLE actualClassHnd = info.compCompHnd->getMethodClass(pResolvedToken->hMethod); + VerifyOrReturn(eeIsValueClass(actualClassHnd), + "Call to base type of valuetype (which is never a valuetype)"); + } + + // Rules for non-virtual call to a non-final virtual method: + + // Define: + // The "this" pointer is considered to be "possibly written" if + // 1. Its address have been taken (LDARGA 0) anywhere in the method. + // (or) + // 2. It has been stored to (STARG.0) anywhere in the method. + + // A non-virtual call to a non-final virtual method is only allowed if + // 1. The this pointer passed to the callee is an instance of a boxed value type. + // (or) + // 2. The this pointer passed to the callee is the current method's this pointer. + // (and) The current method's this pointer is not "possibly written". + + // Thus the rule is that if you assign to this ANYWHERE you can't make "base" calls to + // virtual methods. (Luckily this does affect .ctors, since they are not virtual). + // This is stronger that is strictly needed, but implementing a laxer rule is significantly + // hard and more error prone. + + if (opcode == CEE_CALL && (mflags & CORINFO_FLG_VIRTUAL) && ((mflags & CORINFO_FLG_FINAL) == 0)) + { + VerifyOrReturn((tiThis.IsThisPtr() && lvaIsOriginalThisReadOnly()) || verIsBoxedValueType(tiThis), + "The 'this' parameter to the call must be either the calling method's 'this' parameter or " + "a boxed value type."); + } + } + + // check any constraints on the callee's class and type parameters + VerifyOrReturn(info.compCompHnd->satisfiesClassConstraints(pResolvedToken->hClass), + "method has unsatisfied class constraints"); + VerifyOrReturn(info.compCompHnd->satisfiesMethodConstraints(pResolvedToken->hClass, pResolvedToken->hMethod), + "method has unsatisfied method constraints"); + + if (mflags & CORINFO_FLG_PROTECTED) + { + VerifyOrReturn(info.compCompHnd->canAccessFamily(info.compMethodHnd, instanceClassHnd), + "Can't access protected method"); + } + + // Get the exact view of the signature for an array method + if (sig->retType != CORINFO_TYPE_VOID) + { + eeGetMethodSig(pResolvedToken->hMethod, sig, pResolvedToken->hClass); + } + + // "readonly." prefixed calls only allowed for the Address operation on arrays. + // The methods supported by array types are under the control of the EE + // so we can trust that only the Address operation returns a byref. + if (readonlyCall) + { + typeInfo tiCalleeRetType = verMakeTypeInfo(sig->retType, sig->retTypeClass); + VerifyOrReturn((methodClassFlgs & CORINFO_FLG_ARRAY) && tiCalleeRetType.IsByRef(), + "unexpected use of readonly prefix"); + } + + // Verify the tailcall + if (tailCall) + { + verCheckTailCallConstraint(opcode, pResolvedToken, pConstrainedResolvedToken, false); + } +} + +/***************************************************************************** + * Checks that a delegate creation is done using the following pattern: + * dup + * ldvirtftn targetMemberRef + * OR + * ldftn targetMemberRef + * + * 'delegateCreateStart' points at the last dup or ldftn in this basic block (null if + * not in this basic block) + * + * targetMemberRef is read from the code sequence. + * targetMemberRef is validated iff verificationNeeded. + */ + +bool Compiler::verCheckDelegateCreation(const BYTE* delegateCreateStart, + const BYTE* codeAddr, + mdMemberRef& targetMemberRef) +{ + if (impIsLDFTN_TOKEN(delegateCreateStart, codeAddr)) + { + targetMemberRef = getU4LittleEndian(&delegateCreateStart[2]); + return true; + } + else if (impIsDUP_LDVIRTFTN_TOKEN(delegateCreateStart, codeAddr)) + { + targetMemberRef = getU4LittleEndian(&delegateCreateStart[3]); + return true; + } + + return false; +} + +typeInfo Compiler::verVerifySTIND(const typeInfo& tiTo, const typeInfo& value, const typeInfo& instrType) +{ + Verify(!tiTo.IsReadonlyByRef(), "write to readonly byref"); + typeInfo ptrVal = verVerifyLDIND(tiTo, instrType); + typeInfo normPtrVal = typeInfo(ptrVal).NormaliseForStack(); + if (!tiCompatibleWith(value, normPtrVal, true)) + { + Verify(tiCompatibleWith(value, normPtrVal, true), "type mismatch"); + } + return ptrVal; +} + +typeInfo Compiler::verVerifyLDIND(const typeInfo& ptr, const typeInfo& instrType) +{ + assert(!instrType.IsStruct()); + + typeInfo ptrVal; + if (ptr.IsByRef()) + { + ptrVal = DereferenceByRef(ptr); + if (instrType.IsObjRef() && !ptrVal.IsObjRef()) + { + Verify(false, "bad pointer"); + } + else if (!instrType.IsObjRef() && !typeInfo::AreEquivalent(instrType, ptrVal)) + { + Verify(false, "pointer not consistent with instr"); + } + } + else + { + Verify(false, "pointer not byref"); + } + + return ptrVal; +} + +// Verify that the field is used properly. 'tiThis' is NULL for statics, +// 'fieldFlags' is the fields attributes, and mutator is true if it is a +// ld*flda or a st*fld. +// 'enclosingClass' is given if we are accessing a field in some specific type. + +void Compiler::verVerifyField(CORINFO_RESOLVED_TOKEN* pResolvedToken, + const CORINFO_FIELD_INFO& fieldInfo, + const typeInfo* tiThis, + bool mutator, + bool allowPlainStructAsThis) +{ + CORINFO_CLASS_HANDLE enclosingClass = pResolvedToken->hClass; + unsigned fieldFlags = fieldInfo.fieldFlags; + CORINFO_CLASS_HANDLE instanceClass = + info.compClassHnd; // for statics, we imagine the instance is the current class. + + bool isStaticField = ((fieldFlags & CORINFO_FLG_FIELD_STATIC) != 0); + if (mutator) + { + Verify(!(fieldFlags & CORINFO_FLG_FIELD_UNMANAGED), "mutating an RVA bases static"); + if ((fieldFlags & CORINFO_FLG_FIELD_FINAL)) + { + Verify((info.compFlags & CORINFO_FLG_CONSTRUCTOR) && enclosingClass == info.compClassHnd && + info.compIsStatic == isStaticField, + "bad use of initonly field (set or address taken)"); + } + } + + if (tiThis == nullptr) + { + Verify(isStaticField, "used static opcode with non-static field"); + } + else + { + typeInfo tThis = *tiThis; + + if (allowPlainStructAsThis && tThis.IsValueClass()) + { + tThis.MakeByRef(); + } + + // If it is null, we assume we can access it (since it will AV shortly) + // If it is anything but a refernce class, there is no hierarchy, so + // again, we don't need the precise instance class to compute 'protected' access + if (tiThis->IsType(TI_REF)) + { + instanceClass = tiThis->GetClassHandleForObjRef(); + } + + // Note that even if the field is static, we require that the this pointer + // satisfy the same constraints as a non-static field This happens to + // be simpler and seems reasonable + typeInfo tiDeclaredThis = verMakeTypeInfo(enclosingClass); + if (tiDeclaredThis.IsValueClass()) + { + tiDeclaredThis.MakeByRef(); + + // we allow read-only tThis, on any field access (even stores!), because if the + // class implementor wants to prohibit stores he should make the field private. + // we do this by setting the read-only bit on the type we compare tThis to. + tiDeclaredThis.SetIsReadonlyByRef(); + } + else if (verTrackObjCtorInitState && tThis.IsThisPtr()) + { + // Any field access is legal on "uninitialized" this pointers. + // The easiest way to implement this is to simply set the + // initialized bit for the duration of the type check on the + // field access only. It does not change the state of the "this" + // for the function as a whole. Note that the "tThis" is a copy + // of the original "this" type (*tiThis) passed in. + tThis.SetInitialisedObjRef(); + } + + Verify(tiCompatibleWith(tThis, tiDeclaredThis, true), "this type mismatch"); + } + + // Presently the JIT does not check that we don't store or take the address of init-only fields + // since we cannot guarantee their immutability and it is not a security issue. + + // check any constraints on the fields's class --- accessing the field might cause a class constructor to run. + VerifyOrReturn(info.compCompHnd->satisfiesClassConstraints(enclosingClass), + "field has unsatisfied class constraints"); + if (fieldFlags & CORINFO_FLG_FIELD_PROTECTED) + { + Verify(info.compCompHnd->canAccessFamily(info.compMethodHnd, instanceClass), + "Accessing protected method through wrong type."); + } +} + +void Compiler::verVerifyCond(const typeInfo& tiOp1, const typeInfo& tiOp2, unsigned opcode) +{ + if (tiOp1.IsNumberType()) + { +#ifdef TARGET_64BIT + Verify(tiCompatibleWith(tiOp1, tiOp2, true), "Cond type mismatch"); +#else // TARGET_64BIT + // [10/17/2013] Consider changing this: to put on my verification lawyer hat, + // this is non-conforming to the ECMA Spec: types don't have to be equivalent, + // but compatible, since we can coalesce native int with int32 (see section III.1.5). + Verify(typeInfo::AreEquivalent(tiOp1, tiOp2), "Cond type mismatch"); +#endif // !TARGET_64BIT + } + else if (tiOp1.IsObjRef()) + { + switch (opcode) + { + case CEE_BEQ_S: + case CEE_BEQ: + case CEE_BNE_UN_S: + case CEE_BNE_UN: + case CEE_CEQ: + case CEE_CGT_UN: + break; + default: + Verify(false, "Cond not allowed on object types"); + } + Verify(tiOp2.IsObjRef(), "Cond type mismatch"); + } + else if (tiOp1.IsByRef()) + { + Verify(tiOp2.IsByRef(), "Cond type mismatch"); + } + else + { + Verify(tiOp1.IsMethod() && tiOp2.IsMethod(), "Cond type mismatch"); + } +} + +void Compiler::verVerifyThisPtrInitialised() +{ + if (verTrackObjCtorInitState) + { + Verify(verCurrentState.thisInitialized == TIS_Init, "this ptr is not initialized"); + } +} + +bool Compiler::verIsCallToInitThisPtr(CORINFO_CLASS_HANDLE context, CORINFO_CLASS_HANDLE target) +{ + // Either target == context, in this case calling an alternate .ctor + // Or target is the immediate parent of context + + return ((target == context) || (target == info.compCompHnd->getParentType(context))); +} + +GenTree* Compiler::impImportLdvirtftn(GenTree* thisPtr, + CORINFO_RESOLVED_TOKEN* pResolvedToken, + CORINFO_CALL_INFO* pCallInfo) +{ + if ((pCallInfo->methodFlags & CORINFO_FLG_EnC) && !(pCallInfo->classFlags & CORINFO_FLG_INTERFACE)) + { + NO_WAY("Virtual call to a function added via EnC is not supported"); + } + + // NativeAOT generic virtual method + if ((pCallInfo->sig.sigInst.methInstCount != 0) && IsTargetAbi(CORINFO_NATIVEAOT_ABI)) + { + GenTree* runtimeMethodHandle = + impLookupToTree(pResolvedToken, &pCallInfo->codePointerLookup, GTF_ICON_METHOD_HDL, pCallInfo->hMethod); + return gtNewHelperCallNode(CORINFO_HELP_GVMLOOKUP_FOR_SLOT, TYP_I_IMPL, thisPtr, runtimeMethodHandle); + } + +#ifdef FEATURE_READYTORUN + if (opts.IsReadyToRun()) + { + if (!pCallInfo->exactContextNeedsRuntimeLookup) + { + GenTreeCall* call = gtNewHelperCallNode(CORINFO_HELP_READYTORUN_VIRTUAL_FUNC_PTR, TYP_I_IMPL, thisPtr); + + call->setEntryPoint(pCallInfo->codePointerLookup.constLookup); + + return call; + } + + // We need a runtime lookup. NativeAOT has a ReadyToRun helper for that too. + if (IsTargetAbi(CORINFO_NATIVEAOT_ABI)) + { + GenTree* ctxTree = getRuntimeContextTree(pCallInfo->codePointerLookup.lookupKind.runtimeLookupKind); + + return impReadyToRunHelperToTree(pResolvedToken, CORINFO_HELP_READYTORUN_GENERIC_HANDLE, TYP_I_IMPL, + &pCallInfo->codePointerLookup.lookupKind, ctxTree); + } + } +#endif + + // Get the exact descriptor for the static callsite + GenTree* exactTypeDesc = impParentClassTokenToHandle(pResolvedToken); + if (exactTypeDesc == nullptr) + { // compDonotInline() + return nullptr; + } + + GenTree* exactMethodDesc = impTokenToHandle(pResolvedToken); + if (exactMethodDesc == nullptr) + { // compDonotInline() + return nullptr; + } + + // Call helper function. This gets the target address of the final destination callsite. + + return gtNewHelperCallNode(CORINFO_HELP_VIRTUAL_FUNC_PTR, TYP_I_IMPL, thisPtr, exactTypeDesc, exactMethodDesc); +} + +//------------------------------------------------------------------------ +// impBoxPatternMatch: match and import common box idioms +// +// Arguments: +// pResolvedToken - resolved token from the box operation +// codeAddr - position in IL stream after the box instruction +// codeEndp - end of IL stream +// opts - dictate pattern matching behavior +// +// Return Value: +// Number of IL bytes matched and imported, -1 otherwise +// +// Notes: +// pResolvedToken is known to be a value type; ref type boxing +// is handled in the CEE_BOX clause. + +int Compiler::impBoxPatternMatch(CORINFO_RESOLVED_TOKEN* pResolvedToken, + const BYTE* codeAddr, + const BYTE* codeEndp, + BoxPatterns opts) +{ + if (codeAddr >= codeEndp) + { + return -1; + } + + switch (codeAddr[0]) + { + case CEE_UNBOX_ANY: + // box + unbox.any + if (codeAddr + 1 + sizeof(mdToken) <= codeEndp) + { + if (opts == BoxPatterns::MakeInlineObservation) + { + compInlineResult->Note(InlineObservation::CALLEE_FOLDABLE_BOX); + return 1 + sizeof(mdToken); + } + + CORINFO_RESOLVED_TOKEN unboxResolvedToken; + + impResolveToken(codeAddr + 1, &unboxResolvedToken, CORINFO_TOKENKIND_Class); + + // See if the resolved tokens describe types that are equal. + const TypeCompareState compare = + info.compCompHnd->compareTypesForEquality(unboxResolvedToken.hClass, pResolvedToken->hClass); + + bool optimize = false; + + // If so, box/unbox.any is a nop. + if (compare == TypeCompareState::Must) + { + optimize = true; + } + else if (compare == TypeCompareState::MustNot) + { + // An attempt to catch cases where we mix enums and primitives, e.g.: + // (IntEnum)(object)myInt + // (byte)(object)myByteEnum + // + CorInfoType typ = info.compCompHnd->getTypeForPrimitiveValueClass(unboxResolvedToken.hClass); + if ((typ >= CORINFO_TYPE_BYTE) && (typ <= CORINFO_TYPE_ULONG) && + (info.compCompHnd->getTypeForPrimitiveValueClass(pResolvedToken->hClass) == typ)) + { + optimize = true; + } + } + + if (optimize) + { + JITDUMP("\n Importing BOX; UNBOX.ANY as NOP\n"); + // Skip the next unbox.any instruction + return 1 + sizeof(mdToken); + } + } + break; + + case CEE_BRTRUE: + case CEE_BRTRUE_S: + case CEE_BRFALSE: + case CEE_BRFALSE_S: + // box + br_true/false + if ((codeAddr + ((codeAddr[0] >= CEE_BRFALSE) ? 5 : 2)) <= codeEndp) + { + if (opts == BoxPatterns::MakeInlineObservation) + { + compInlineResult->Note(InlineObservation::CALLEE_FOLDABLE_BOX); + return 0; + } + + GenTree* const treeToBox = impStackTop().val; + bool canOptimize = true; + GenTree* treeToNullcheck = nullptr; + + // Can the thing being boxed cause a side effect? + if ((treeToBox->gtFlags & GTF_SIDE_EFFECT) != 0) + { + // Is this a side effect we can replicate cheaply? + if (((treeToBox->gtFlags & GTF_SIDE_EFFECT) == GTF_EXCEPT) && + treeToBox->OperIs(GT_OBJ, GT_BLK, GT_IND)) + { + // If the only side effect comes from the dereference itself, yes. + GenTree* const addr = treeToBox->AsOp()->gtGetOp1(); + + if ((addr->gtFlags & GTF_SIDE_EFFECT) != 0) + { + canOptimize = false; + } + else if (fgAddrCouldBeNull(addr)) + { + treeToNullcheck = addr; + } + } + else + { + canOptimize = false; + } + } + + if (canOptimize) + { + if ((opts == BoxPatterns::IsByRefLike) || + info.compCompHnd->getBoxHelper(pResolvedToken->hClass) == CORINFO_HELP_BOX) + { + JITDUMP("\n Importing BOX; BR_TRUE/FALSE as %sconstant\n", + treeToNullcheck == nullptr ? "" : "nullcheck+"); + impPopStack(); + + GenTree* result = gtNewIconNode(1); + + if (treeToNullcheck != nullptr) + { + GenTree* nullcheck = gtNewNullCheck(treeToNullcheck, compCurBB); + result = gtNewOperNode(GT_COMMA, TYP_INT, nullcheck, result); + } + + impPushOnStack(result, typeInfo(TI_INT)); + return 0; + } + } + } + break; + + case CEE_ISINST: + if (codeAddr + 1 + sizeof(mdToken) + 1 <= codeEndp) + { + const BYTE* nextCodeAddr = codeAddr + 1 + sizeof(mdToken); + + switch (nextCodeAddr[0]) + { + // box + isinst + br_true/false + case CEE_BRTRUE: + case CEE_BRTRUE_S: + case CEE_BRFALSE: + case CEE_BRFALSE_S: + if ((nextCodeAddr + ((nextCodeAddr[0] >= CEE_BRFALSE) ? 5 : 2)) <= codeEndp) + { + if (opts == BoxPatterns::MakeInlineObservation) + { + compInlineResult->Note(InlineObservation::CALLEE_FOLDABLE_BOX); + return 1 + sizeof(mdToken); + } + + if ((impStackTop().val->gtFlags & GTF_SIDE_EFFECT) == 0) + { + CorInfoHelpFunc foldAsHelper; + if (opts == BoxPatterns::IsByRefLike) + { + // Treat ByRefLike types as if they were regular boxing operations + // so they can be elided. + foldAsHelper = CORINFO_HELP_BOX; + } + else + { + foldAsHelper = info.compCompHnd->getBoxHelper(pResolvedToken->hClass); + } + + if (foldAsHelper == CORINFO_HELP_BOX) + { + CORINFO_RESOLVED_TOKEN isInstResolvedToken; + + impResolveToken(codeAddr + 1, &isInstResolvedToken, CORINFO_TOKENKIND_Casting); + + TypeCompareState castResult = + info.compCompHnd->compareTypesForCast(pResolvedToken->hClass, + isInstResolvedToken.hClass); + if (castResult != TypeCompareState::May) + { + JITDUMP("\n Importing BOX; ISINST; BR_TRUE/FALSE as constant\n"); + impPopStack(); + + impPushOnStack(gtNewIconNode((castResult == TypeCompareState::Must) ? 1 : 0), + typeInfo(TI_INT)); + + // Skip the next isinst instruction + return 1 + sizeof(mdToken); + } + } + else if (foldAsHelper == CORINFO_HELP_BOX_NULLABLE) + { + // For nullable we're going to fold it to "ldfld hasValue + brtrue/brfalse" or + // "ldc.i4.0 + brtrue/brfalse" in case if the underlying type is not castable to + // the target type. + CORINFO_RESOLVED_TOKEN isInstResolvedToken; + impResolveToken(codeAddr + 1, &isInstResolvedToken, CORINFO_TOKENKIND_Casting); + + CORINFO_CLASS_HANDLE nullableCls = pResolvedToken->hClass; + CORINFO_CLASS_HANDLE underlyingCls = info.compCompHnd->getTypeForBox(nullableCls); + + TypeCompareState castResult = + info.compCompHnd->compareTypesForCast(underlyingCls, + isInstResolvedToken.hClass); + + if (castResult == TypeCompareState::Must) + { + const CORINFO_FIELD_HANDLE hasValueFldHnd = + info.compCompHnd->getFieldInClass(nullableCls, 0); + + assert(info.compCompHnd->getFieldOffset(hasValueFldHnd) == 0); + assert(!strcmp(info.compCompHnd->getFieldName(hasValueFldHnd, nullptr), + "hasValue")); + + GenTree* objToBox = impPopStack().val; + + // Spill struct to get its address (to access hasValue field) + objToBox = + impGetStructAddr(objToBox, nullableCls, (unsigned)CHECK_SPILL_ALL, true); + + impPushOnStack(gtNewFieldRef(TYP_BOOL, hasValueFldHnd, objToBox, 0), + typeInfo(TI_INT)); + + JITDUMP("\n Importing BOX; ISINST; BR_TRUE/FALSE as nullableVT.hasValue\n"); + return 1 + sizeof(mdToken); + } + else if (castResult == TypeCompareState::MustNot) + { + impPopStack(); + impPushOnStack(gtNewIconNode(0), typeInfo(TI_INT)); + JITDUMP("\n Importing BOX; ISINST; BR_TRUE/FALSE as constant (false)\n"); + return 1 + sizeof(mdToken); + } + } + } + } + break; + + // box + isinst + unbox.any + case CEE_UNBOX_ANY: + if ((nextCodeAddr + 1 + sizeof(mdToken)) <= codeEndp) + { + if (opts == BoxPatterns::MakeInlineObservation) + { + compInlineResult->Note(InlineObservation::CALLEE_FOLDABLE_BOX); + return 2 + sizeof(mdToken) * 2; + } + + // See if the resolved tokens in box, isinst and unbox.any describe types that are equal. + CORINFO_RESOLVED_TOKEN isinstResolvedToken = {}; + impResolveToken(codeAddr + 1, &isinstResolvedToken, CORINFO_TOKENKIND_Class); + + if (info.compCompHnd->compareTypesForEquality(isinstResolvedToken.hClass, + pResolvedToken->hClass) == + TypeCompareState::Must) + { + CORINFO_RESOLVED_TOKEN unboxResolvedToken = {}; + impResolveToken(nextCodeAddr + 1, &unboxResolvedToken, CORINFO_TOKENKIND_Class); + + // If so, box + isinst + unbox.any is a nop. + if (info.compCompHnd->compareTypesForEquality(unboxResolvedToken.hClass, + pResolvedToken->hClass) == + TypeCompareState::Must) + { + JITDUMP("\n Importing BOX; ISINST, UNBOX.ANY as NOP\n"); + return 2 + sizeof(mdToken) * 2; + } + } + } + break; + } + } + break; + + default: + break; + } + + return -1; +} + +//------------------------------------------------------------------------ +// impImportAndPushBox: build and import a value-type box +// +// Arguments: +// pResolvedToken - resolved token from the box operation +// +// Return Value: +// None. +// +// Side Effects: +// The value to be boxed is popped from the stack, and a tree for +// the boxed value is pushed. This method may create upstream +// statements, spill side effecting trees, and create new temps. +// +// If importing an inlinee, we may also discover the inline must +// fail. If so there is no new value pushed on the stack. Callers +// should use CompDoNotInline after calling this method to see if +// ongoing importation should be aborted. +// +// Notes: +// Boxing of ref classes results in the same value as the value on +// the top of the stack, so is handled inline in impImportBlockCode +// for the CEE_BOX case. Only value or primitive type boxes make it +// here. +// +// Boxing for nullable types is done via a helper call; boxing +// of other value types is expanded inline or handled via helper +// call, depending on the jit's codegen mode. +// +// When the jit is operating in size and time constrained modes, +// using a helper call here can save jit time and code size. But it +// also may inhibit cleanup optimizations that could have also had a +// even greater benefit effect on code size and jit time. An optimal +// strategy may need to peek ahead and see if it is easy to tell how +// the box is being used. For now, we defer. + +void Compiler::impImportAndPushBox(CORINFO_RESOLVED_TOKEN* pResolvedToken) +{ + // Spill any special side effects + impSpillSpecialSideEff(); + + // Get get the expression to box from the stack. + GenTree* op1 = nullptr; + GenTree* op2 = nullptr; + StackEntry se = impPopStack(); + CORINFO_CLASS_HANDLE operCls = se.seTypeInfo.GetClassHandle(); + GenTree* exprToBox = se.val; + + // Look at what helper we should use. + CorInfoHelpFunc boxHelper = info.compCompHnd->getBoxHelper(pResolvedToken->hClass); + + // Determine what expansion to prefer. + // + // In size/time/debuggable constrained modes, the helper call + // expansion for box is generally smaller and is preferred, unless + // the value to box is a struct that comes from a call. In that + // case the call can construct its return value directly into the + // box payload, saving possibly some up-front zeroing. + // + // Currently primitive type boxes always get inline expanded. We may + // want to do the same for small structs if they don't come from + // calls and don't have GC pointers, since explicitly copying such + // structs is cheap. + JITDUMP("\nCompiler::impImportAndPushBox -- handling BOX(value class) via"); + bool canExpandInline = (boxHelper == CORINFO_HELP_BOX); + bool optForSize = !exprToBox->IsCall() && (operCls != nullptr) && opts.OptimizationDisabled(); + bool expandInline = canExpandInline && !optForSize; + + if (expandInline) + { + JITDUMP(" inline allocate/copy sequence\n"); + + // we are doing 'normal' boxing. This means that we can inline the box operation + // Box(expr) gets morphed into + // temp = new(clsHnd) + // cpobj(temp+4, expr, clsHnd) + // push temp + // The code paths differ slightly below for structs and primitives because + // "cpobj" differs in these cases. In one case you get + // impAssignStructPtr(temp+4, expr, clsHnd) + // and the other you get + // *(temp+4) = expr + + if (opts.OptimizationDisabled()) + { + // For minopts/debug code, try and minimize the total number + // of box temps by reusing an existing temp when possible. + if (impBoxTempInUse || impBoxTemp == BAD_VAR_NUM) + { + impBoxTemp = lvaGrabTemp(true DEBUGARG("Reusable Box Helper")); + } + } + else + { + // When optimizing, use a new temp for each box operation + // since we then know the exact class of the box temp. + impBoxTemp = lvaGrabTemp(true DEBUGARG("Single-def Box Helper")); + lvaTable[impBoxTemp].lvType = TYP_REF; + lvaTable[impBoxTemp].lvSingleDef = 1; + JITDUMP("Marking V%02u as a single def local\n", impBoxTemp); + const bool isExact = true; + lvaSetClass(impBoxTemp, pResolvedToken->hClass, isExact); + } + + // needs to stay in use until this box expression is appended + // some other node. We approximate this by keeping it alive until + // the opcode stack becomes empty + impBoxTempInUse = true; + + // Remember the current last statement in case we need to move + // a range of statements to ensure the box temp is initialized + // before it's used. + // + Statement* const cursor = impLastStmt; + + const bool useParent = false; + op1 = gtNewAllocObjNode(pResolvedToken, useParent); + if (op1 == nullptr) + { + // If we fail to create the newobj node, we must be inlining + // and have run across a type we can't describe. + // + assert(compDonotInline()); + return; + } + + // Remember that this basic block contains 'new' of an object, + // and so does this method + // + compCurBB->bbFlags |= BBF_HAS_NEWOBJ; + optMethodFlags |= OMF_HAS_NEWOBJ; + + // Assign the boxed object to the box temp. + // + GenTree* asg = gtNewTempAssign(impBoxTemp, op1); + Statement* asgStmt = impAppendTree(asg, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); + + // If the exprToBox is a call that returns its value via a ret buf arg, + // move the assignment statement(s) before the call (which must be a top level tree). + // + // We do this because impAssignStructPtr (invoked below) will + // back-substitute into a call when it sees a GT_RET_EXPR and the call + // has a hidden buffer pointer, So we need to reorder things to avoid + // creating out-of-sequence IR. + // + if (varTypeIsStruct(exprToBox) && exprToBox->OperIs(GT_RET_EXPR)) + { + GenTreeCall* const call = exprToBox->AsRetExpr()->gtInlineCandidate->AsCall(); + + if (call->ShouldHaveRetBufArg()) + { + JITDUMP("Must insert newobj stmts for box before call [%06u]\n", dspTreeID(call)); + + // Walk back through the statements in this block, looking for the one + // that has this call as the root node. + // + // Because gtNewTempAssign (above) may have added statements that + // feed into the actual assignment we need to move this set of added + // statements as a group. + // + // Note boxed allocations are side-effect free (no com or finalizer) so + // our only worries here are (correctness) not overlapping the box temp + // lifetime and (perf) stretching the temp lifetime across the inlinee + // body. + // + // Since this is an inline candidate, we must be optimizing, and so we have + // a unique box temp per call. So no worries about overlap. + // + assert(!opts.OptimizationDisabled()); + + // Lifetime stretching could addressed with some extra cleverness--sinking + // the allocation back down to just before the copy, once we figure out + // where the copy is. We defer for now. + // + Statement* insertBeforeStmt = cursor; + noway_assert(insertBeforeStmt != nullptr); + + while (true) + { + if (insertBeforeStmt->GetRootNode() == call) + { + break; + } + + // If we've searched all the statements in the block and failed to + // find the call, then something's wrong. + // + noway_assert(insertBeforeStmt != impStmtList); + + insertBeforeStmt = insertBeforeStmt->GetPrevStmt(); + } + + // Found the call. Move the statements comprising the assignment. + // + JITDUMP("Moving " FMT_STMT "..." FMT_STMT " before " FMT_STMT "\n", cursor->GetNextStmt()->GetID(), + asgStmt->GetID(), insertBeforeStmt->GetID()); + assert(asgStmt == impLastStmt); + do + { + Statement* movingStmt = impExtractLastStmt(); + impInsertStmtBefore(movingStmt, insertBeforeStmt); + insertBeforeStmt = movingStmt; + } while (impLastStmt != cursor); + } + } + + // Create a pointer to the box payload in op1. + // + op1 = gtNewLclvNode(impBoxTemp, TYP_REF); + op2 = gtNewIconNode(TARGET_POINTER_SIZE, TYP_I_IMPL); + op1 = gtNewOperNode(GT_ADD, TYP_BYREF, op1, op2); + + // Copy from the exprToBox to the box payload. + // + if (varTypeIsStruct(exprToBox)) + { + assert(info.compCompHnd->getClassSize(pResolvedToken->hClass) == info.compCompHnd->getClassSize(operCls)); + op1 = impAssignStructPtr(op1, exprToBox, operCls, (unsigned)CHECK_SPILL_ALL); + } + else + { + var_types lclTyp = exprToBox->TypeGet(); + if (lclTyp == TYP_BYREF) + { + lclTyp = TYP_I_IMPL; + } + CorInfoType jitType = info.compCompHnd->asCorInfoType(pResolvedToken->hClass); + if (impIsPrimitive(jitType)) + { + lclTyp = JITtype2varType(jitType); + } + + var_types srcTyp = exprToBox->TypeGet(); + var_types dstTyp = lclTyp; + + // We allow float <-> double mismatches and implicit truncation for small types. + assert((genActualType(srcTyp) == genActualType(dstTyp)) || + (varTypeIsFloating(srcTyp) == varTypeIsFloating(dstTyp))); + + // Note regarding small types. + // We are going to store to the box here via an indirection, so the cast added below is + // redundant, since the store has an implicit truncation semantic. The reason we still + // add this cast is so that the code which deals with GT_BOX optimizations does not have + // to account for this implicit truncation (e. g. understand that BOX(0xFF + 1) is + // actually BOX(0) or deal with signedness mismatch and other GT_CAST complexities). + if (srcTyp != dstTyp) + { + exprToBox = gtNewCastNode(genActualType(dstTyp), exprToBox, false, dstTyp); + } + + op1 = gtNewAssignNode(gtNewOperNode(GT_IND, dstTyp, op1), exprToBox); + } + + // Spill eval stack to flush out any pending side effects. + impSpillSideEffects(true, (unsigned)CHECK_SPILL_ALL DEBUGARG("impImportAndPushBox")); + + // Set up this copy as a second assignment. + Statement* copyStmt = impAppendTree(op1, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); + + op1 = gtNewLclvNode(impBoxTemp, TYP_REF); + + // Record that this is a "box" node and keep track of the matching parts. + op1 = new (this, GT_BOX) GenTreeBox(TYP_REF, op1, asgStmt, copyStmt); + + // If it is a value class, mark the "box" node. We can use this information + // to optimise several cases: + // "box(x) == null" --> false + // "(box(x)).CallAnInterfaceMethod(...)" --> "(&x).CallAValueTypeMethod" + // "(box(x)).CallAnObjectMethod(...)" --> "(&x).CallAValueTypeMethod" + + op1->gtFlags |= GTF_BOX_VALUE; + assert(op1->IsBoxedValue()); + assert(asg->gtOper == GT_ASG); + } + else + { + // Don't optimize, just call the helper and be done with it. + JITDUMP(" helper call because: %s\n", canExpandInline ? "optimizing for size" : "nullable"); + assert(operCls != nullptr); + + // Ensure that the value class is restored + op2 = impTokenToHandle(pResolvedToken, nullptr, true /* mustRestoreHandle */); + if (op2 == nullptr) + { + // We must be backing out of an inline. + assert(compDonotInline()); + return; + } + + op1 = gtNewHelperCallNode(boxHelper, TYP_REF, op2, + impGetStructAddr(exprToBox, operCls, (unsigned)CHECK_SPILL_ALL, true)); + } + + /* Push the result back on the stack, */ + /* even if clsHnd is a value class we want the TI_REF */ + typeInfo tiRetVal = typeInfo(TI_REF, info.compCompHnd->getTypeForBox(pResolvedToken->hClass)); + impPushOnStack(op1, tiRetVal); +} + +//------------------------------------------------------------------------ +// impImportNewObjArray: Build and import `new` of multi-dimensional array +// +// Arguments: +// pResolvedToken - The CORINFO_RESOLVED_TOKEN that has been initialized +// by a call to CEEInfo::resolveToken(). +// pCallInfo - The CORINFO_CALL_INFO that has been initialized +// by a call to CEEInfo::getCallInfo(). +// +// Assumptions: +// The multi-dimensional array constructor arguments (array dimensions) are +// pushed on the IL stack on entry to this method. +// +// Notes: +// Multi-dimensional array constructors are imported as calls to a JIT +// helper, not as regular calls. +// +void Compiler::impImportNewObjArray(CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_CALL_INFO* pCallInfo) +{ + GenTree* classHandle = impParentClassTokenToHandle(pResolvedToken); + if (classHandle == nullptr) + { // compDonotInline() + return; + } + + assert(pCallInfo->sig.numArgs); + + GenTree* node; + + // Reuse the temp used to pass the array dimensions to avoid bloating + // the stack frame in case there are multiple calls to multi-dim array + // constructors within a single method. + if (lvaNewObjArrayArgs == BAD_VAR_NUM) + { + lvaNewObjArrayArgs = lvaGrabTemp(false DEBUGARG("NewObjArrayArgs")); + lvaTable[lvaNewObjArrayArgs].lvType = TYP_BLK; + lvaTable[lvaNewObjArrayArgs].lvExactSize = 0; + } + + // Increase size of lvaNewObjArrayArgs to be the largest size needed to hold 'numArgs' integers + // for our call to CORINFO_HELP_NEW_MDARR. + lvaTable[lvaNewObjArrayArgs].lvExactSize = + max(lvaTable[lvaNewObjArrayArgs].lvExactSize, pCallInfo->sig.numArgs * sizeof(INT32)); + + // The side-effects may include allocation of more multi-dimensional arrays. Spill all side-effects + // to ensure that the shared lvaNewObjArrayArgs local variable is only ever used to pass arguments + // to one allocation at a time. + impSpillSideEffects(true, (unsigned)CHECK_SPILL_ALL DEBUGARG("impImportNewObjArray")); + + // + // The arguments of the CORINFO_HELP_NEW_MDARR helper are: + // - Array class handle + // - Number of dimension arguments + // - Pointer to block of int32 dimensions: address of lvaNewObjArrayArgs temp. + // + + node = gtNewLclvNode(lvaNewObjArrayArgs, TYP_BLK); + node = gtNewOperNode(GT_ADDR, TYP_I_IMPL, node); + + // Pop dimension arguments from the stack one at a time and store it + // into lvaNewObjArrayArgs temp. + for (int i = pCallInfo->sig.numArgs - 1; i >= 0; i--) + { + GenTree* arg = impImplicitIorI4Cast(impPopStack().val, TYP_INT); + + GenTree* dest = gtNewLclvNode(lvaNewObjArrayArgs, TYP_BLK); + dest = gtNewOperNode(GT_ADDR, TYP_I_IMPL, dest); + dest = gtNewOperNode(GT_ADD, TYP_I_IMPL, dest, + new (this, GT_CNS_INT) GenTreeIntCon(TYP_I_IMPL, sizeof(INT32) * i)); + dest = gtNewOperNode(GT_IND, TYP_INT, dest); + + node = gtNewOperNode(GT_COMMA, node->TypeGet(), gtNewAssignNode(dest, arg), node); + } + + node = + gtNewHelperCallNode(CORINFO_HELP_NEW_MDARR, TYP_REF, classHandle, gtNewIconNode(pCallInfo->sig.numArgs), node); + + node->AsCall()->compileTimeHelperArgumentHandle = (CORINFO_GENERIC_HANDLE)pResolvedToken->hClass; + + // Remember that this function contains 'new' of a MD array. + optMethodFlags |= OMF_HAS_MDNEWARRAY; + + impPushOnStack(node, typeInfo(TI_REF, pResolvedToken->hClass)); +} + +GenTree* Compiler::impTransformThis(GenTree* thisPtr, + CORINFO_RESOLVED_TOKEN* pConstrainedResolvedToken, + CORINFO_THIS_TRANSFORM transform) +{ + switch (transform) + { + case CORINFO_DEREF_THIS: + { + GenTree* obj = thisPtr; + + // This does a LDIND on the obj, which should be a byref. pointing to a ref + impBashVarAddrsToI(obj); + assert(genActualType(obj->gtType) == TYP_I_IMPL || obj->gtType == TYP_BYREF); + CorInfoType constraintTyp = info.compCompHnd->asCorInfoType(pConstrainedResolvedToken->hClass); + + obj = gtNewOperNode(GT_IND, JITtype2varType(constraintTyp), obj); + // ldind could point anywhere, example a boxed class static int + obj->gtFlags |= (GTF_EXCEPT | GTF_GLOB_REF); + + return obj; + } + + case CORINFO_BOX_THIS: + { + // Constraint calls where there might be no + // unboxed entry point require us to implement the call via helper. + // These only occur when a possible target of the call + // may have inherited an implementation of an interface + // method from System.Object or System.ValueType. The EE does not provide us with + // "unboxed" versions of these methods. + + GenTree* obj = thisPtr; + + assert(obj->TypeGet() == TYP_BYREF || obj->TypeGet() == TYP_I_IMPL); + obj = gtNewObjNode(pConstrainedResolvedToken->hClass, obj); + obj->gtFlags |= GTF_EXCEPT; + + CorInfoType jitTyp = info.compCompHnd->asCorInfoType(pConstrainedResolvedToken->hClass); + if (impIsPrimitive(jitTyp)) + { + if (obj->OperIsBlk()) + { + obj->ChangeOperUnchecked(GT_IND); + obj->AsOp()->gtOp2 = nullptr; // must be zero for tree walkers + } + + obj->gtType = JITtype2varType(jitTyp); + assert(varTypeIsArithmetic(obj->gtType)); + } + + // This pushes on the dereferenced byref + // This is then used immediately to box. + impPushOnStack(obj, verMakeTypeInfo(pConstrainedResolvedToken->hClass).NormaliseForStack()); + + // This pops off the byref-to-a-value-type remaining on the stack and + // replaces it with a boxed object. + // This is then used as the object to the virtual call immediately below. + impImportAndPushBox(pConstrainedResolvedToken); + if (compDonotInline()) + { + return nullptr; + } + + obj = impPopStack().val; + return obj; + } + case CORINFO_NO_THIS_TRANSFORM: + default: + return thisPtr; + } +} + +//------------------------------------------------------------------------ +// impCanPInvokeInline: check whether PInvoke inlining should enabled in current method. +// +// Return Value: +// true if PInvoke inlining should be enabled in current method, false otherwise +// +// Notes: +// Checks a number of ambient conditions where we could pinvoke but choose not to + +bool Compiler::impCanPInvokeInline() +{ + return getInlinePInvokeEnabled() && (!opts.compDbgCode) && (compCodeOpt() != SMALL_CODE) && + (!opts.compNoPInvokeInlineCB) // profiler is preventing inline pinvoke + ; +} + +//------------------------------------------------------------------------ +// impCanPInvokeInlineCallSite: basic legality checks using information +// from a call to see if the call qualifies as an inline pinvoke. +// +// Arguments: +// block - block contaning the call, or for inlinees, block +// containing the call being inlined +// +// Return Value: +// true if this call can legally qualify as an inline pinvoke, false otherwise +// +// Notes: +// For runtimes that support exception handling interop there are +// restrictions on using inline pinvoke in handler regions. +// +// * We have to disable pinvoke inlining inside of filters because +// in case the main execution (i.e. in the try block) is inside +// unmanaged code, we cannot reuse the inlined stub (we still need +// the original state until we are in the catch handler) +// +// * We disable pinvoke inlining inside handlers since the GSCookie +// is in the inlined Frame (see +// CORINFO_EE_INFO::InlinedCallFrameInfo::offsetOfGSCookie), but +// this would not protect framelets/return-address of handlers. +// +// These restrictions are currently also in place for CoreCLR but +// can be relaxed when coreclr/#8459 is addressed. + +bool Compiler::impCanPInvokeInlineCallSite(BasicBlock* block) +{ + if (block->hasHndIndex()) + { + return false; + } + + // The remaining limitations do not apply to NativeAOT + if (IsTargetAbi(CORINFO_NATIVEAOT_ABI)) + { + return true; + } + +#ifdef TARGET_64BIT + // On 64-bit platforms, we disable pinvoke inlining inside of try regions. + // Note that this could be needed on other architectures too, but we + // haven't done enough investigation to know for sure at this point. + // + // Here is the comment from JIT64 explaining why: + // [VSWhidbey: 611015] - because the jitted code links in the + // Frame (instead of the stub) we rely on the Frame not being + // 'active' until inside the stub. This normally happens by the + // stub setting the return address pointer in the Frame object + // inside the stub. On a normal return, the return address + // pointer is zeroed out so the Frame can be safely re-used, but + // if an exception occurs, nobody zeros out the return address + // pointer. Thus if we re-used the Frame object, it would go + // 'active' as soon as we link it into the Frame chain. + // + // Technically we only need to disable PInvoke inlining if we're + // in a handler or if we're in a try body with a catch or + // filter/except where other non-handler code in this method + // might run and try to re-use the dirty Frame object. + // + // A desktop test case where this seems to matter is + // jit\jit64\ebvts\mcpp\sources2\ijw\__clrcall\vector_ctor_dtor.02\deldtor_clr.exe + if (block->hasTryIndex()) + { + // This does not apply to the raw pinvoke call that is inside the pinvoke + // ILStub. In this case, we have to inline the raw pinvoke call into the stub, + // otherwise we would end up with a stub that recursively calls itself, and end + // up with a stack overflow. + if (opts.jitFlags->IsSet(JitFlags::JIT_FLAG_IL_STUB) && opts.ShouldUsePInvokeHelpers()) + { + return true; + } + + return false; + } +#endif // TARGET_64BIT + + return true; +} + +//------------------------------------------------------------------------ +// impCheckForPInvokeCall examine call to see if it is a pinvoke and if so +// if it can be expressed as an inline pinvoke. +// +// Arguments: +// call - tree for the call +// methHnd - handle for the method being called (may be null) +// sig - signature of the method being called +// mflags - method flags for the method being called +// block - block contaning the call, or for inlinees, block +// containing the call being inlined +// +// Notes: +// Sets GTF_CALL_M_PINVOKE on the call for pinvokes. +// +// Also sets GTF_CALL_UNMANAGED on call for inline pinvokes if the +// call passes a combination of legality and profitability checks. +// +// If GTF_CALL_UNMANAGED is set, increments info.compUnmanagedCallCountWithGCTransition + +void Compiler::impCheckForPInvokeCall( + GenTreeCall* call, CORINFO_METHOD_HANDLE methHnd, CORINFO_SIG_INFO* sig, unsigned mflags, BasicBlock* block) +{ + CorInfoCallConvExtension unmanagedCallConv; + + // If VM flagged it as Pinvoke, flag the call node accordingly + if ((mflags & CORINFO_FLG_PINVOKE) != 0) + { + call->gtCallMoreFlags |= GTF_CALL_M_PINVOKE; + } + + bool suppressGCTransition = false; + if (methHnd) + { + if ((mflags & CORINFO_FLG_PINVOKE) == 0) + { + return; + } + + unmanagedCallConv = info.compCompHnd->getUnmanagedCallConv(methHnd, nullptr, &suppressGCTransition); + } + else + { + if (sig->getCallConv() == CORINFO_CALLCONV_DEFAULT || sig->getCallConv() == CORINFO_CALLCONV_VARARG) + { + return; + } + + unmanagedCallConv = info.compCompHnd->getUnmanagedCallConv(nullptr, sig, &suppressGCTransition); + + assert(!call->gtCallCookie); + } + + if (suppressGCTransition) + { + call->gtCallMoreFlags |= GTF_CALL_M_SUPPRESS_GC_TRANSITION; + } + + if ((unmanagedCallConv == CorInfoCallConvExtension::Thiscall) && (sig->numArgs == 0)) + { + BADCODE("thiscall with 0 arguments"); + } + + // If we can't get the unmanaged calling convention or the calling convention is unsupported in the JIT, + // return here without inlining the native call. + if (unmanagedCallConv == CorInfoCallConvExtension::Managed || + unmanagedCallConv == CorInfoCallConvExtension::Fastcall || + unmanagedCallConv == CorInfoCallConvExtension::FastcallMemberFunction) + { + return; + } + optNativeCallCount++; + + if (methHnd == nullptr && (opts.jitFlags->IsSet(JitFlags::JIT_FLAG_IL_STUB) || IsTargetAbi(CORINFO_NATIVEAOT_ABI))) + { + // PInvoke in NativeAOT ABI must be always inlined. Non-inlineable CALLI cases have been + // converted to regular method calls earlier using convertPInvokeCalliToCall. + + // PInvoke CALLI in IL stubs must be inlined + } + else + { + // Check legality + if (!impCanPInvokeInlineCallSite(block)) + { + return; + } + + // Legal PInvoke CALL in PInvoke IL stubs must be inlined to avoid infinite recursive + // inlining in NativeAOT. Skip the ambient conditions checks and profitability checks. + if (!IsTargetAbi(CORINFO_NATIVEAOT_ABI) || (info.compFlags & CORINFO_FLG_PINVOKE) == 0) + { + if (opts.jitFlags->IsSet(JitFlags::JIT_FLAG_IL_STUB) && opts.ShouldUsePInvokeHelpers()) + { + // Raw PInvoke call in PInvoke IL stub generated must be inlined to avoid infinite + // recursive calls to the stub. + } + else + { + if (!impCanPInvokeInline()) + { + return; + } + + // Size-speed tradeoff: don't use inline pinvoke at rarely + // executed call sites. The non-inline version is more + // compact. + if (block->isRunRarely()) + { + return; + } + } + } + + // The expensive check should be last + if (info.compCompHnd->pInvokeMarshalingRequired(methHnd, sig)) + { + return; + } + } + + JITLOG((LL_INFO1000000, "\nInline a CALLI PINVOKE call from method %s\n", info.compFullName)); + + call->gtFlags |= GTF_CALL_UNMANAGED; + call->unmgdCallConv = unmanagedCallConv; + if (!call->IsSuppressGCTransition()) + { + info.compUnmanagedCallCountWithGCTransition++; + } + + // AMD64 convention is same for native and managed + if (unmanagedCallConv == CorInfoCallConvExtension::C || + unmanagedCallConv == CorInfoCallConvExtension::CMemberFunction) + { + call->gtFlags |= GTF_CALL_POP_ARGS; + } +} + +GenTreeCall* Compiler::impImportIndirectCall(CORINFO_SIG_INFO* sig, const DebugInfo& di) +{ + var_types callRetTyp = JITtype2varType(sig->retType); + + /* The function pointer is on top of the stack - It may be a + * complex expression. As it is evaluated after the args, + * it may cause registered args to be spilled. Simply spill it. + */ + + // Ignore this trivial case. + if (impStackTop().val->gtOper != GT_LCL_VAR) + { + impSpillStackEntry(verCurrentState.esStackDepth - 1, + BAD_VAR_NUM DEBUGARG(false) DEBUGARG("impImportIndirectCall")); + } + + /* Get the function pointer */ + + GenTree* fptr = impPopStack().val; + + // The function pointer is typically a sized to match the target pointer size + // However, stubgen IL optimization can change LDC.I8 to LDC.I4 + // See ILCodeStream::LowerOpcode + assert(genActualType(fptr->gtType) == TYP_I_IMPL || genActualType(fptr->gtType) == TYP_INT); + +#ifdef DEBUG + // This temporary must never be converted to a double in stress mode, + // because that can introduce a call to the cast helper after the + // arguments have already been evaluated. + + if (fptr->OperGet() == GT_LCL_VAR) + { + lvaTable[fptr->AsLclVarCommon()->GetLclNum()].lvKeepType = 1; + } +#endif + + /* Create the call node */ + + GenTreeCall* call = gtNewIndCallNode(fptr, callRetTyp, di); + + call->gtFlags |= GTF_EXCEPT | (fptr->gtFlags & GTF_GLOB_EFFECT); +#ifdef UNIX_X86_ABI + call->gtFlags &= ~GTF_CALL_POP_ARGS; +#endif + + return call; +} + +/*****************************************************************************/ + +void Compiler::impPopArgsForUnmanagedCall(GenTreeCall* call, CORINFO_SIG_INFO* sig) +{ + assert(call->gtFlags & GTF_CALL_UNMANAGED); + + /* Since we push the arguments in reverse order (i.e. right -> left) + * spill any side effects from the stack + * + * OBS: If there is only one side effect we do not need to spill it + * thus we have to spill all side-effects except last one + */ + + unsigned lastLevelWithSideEffects = UINT_MAX; + + unsigned argsToReverse = sig->numArgs; + + // For "thiscall", the first argument goes in a register. Since its + // order does not need to be changed, we do not need to spill it + + if (call->unmgdCallConv == CorInfoCallConvExtension::Thiscall) + { + assert(argsToReverse); + argsToReverse--; + } + +#ifndef TARGET_X86 + // Don't reverse args on ARM or x64 - first four args always placed in regs in order + argsToReverse = 0; +#endif + + for (unsigned level = verCurrentState.esStackDepth - argsToReverse; level < verCurrentState.esStackDepth; level++) + { + if (verCurrentState.esStack[level].val->gtFlags & GTF_ORDER_SIDEEFF) + { + assert(lastLevelWithSideEffects == UINT_MAX); + + impSpillStackEntry(level, + BAD_VAR_NUM DEBUGARG(false) DEBUGARG("impPopArgsForUnmanagedCall - other side effect")); + } + else if (verCurrentState.esStack[level].val->gtFlags & GTF_SIDE_EFFECT) + { + if (lastLevelWithSideEffects != UINT_MAX) + { + /* We had a previous side effect - must spill it */ + impSpillStackEntry(lastLevelWithSideEffects, + BAD_VAR_NUM DEBUGARG(false) DEBUGARG("impPopArgsForUnmanagedCall - side effect")); + + /* Record the level for the current side effect in case we will spill it */ + lastLevelWithSideEffects = level; + } + else + { + /* This is the first side effect encountered - record its level */ + + lastLevelWithSideEffects = level; + } + } + } + + /* The argument list is now "clean" - no out-of-order side effects + * Pop the argument list in reverse order */ + + impPopReverseCallArgs(sig, call, sig->numArgs - argsToReverse); + + if (call->unmgdCallConv == CorInfoCallConvExtension::Thiscall) + { + GenTree* thisPtr = call->gtArgs.GetArgByIndex(0)->GetNode(); + impBashVarAddrsToI(thisPtr); + assert(thisPtr->TypeGet() == TYP_I_IMPL || thisPtr->TypeGet() == TYP_BYREF); + } + + for (CallArg& arg : call->gtArgs.Args()) + { + GenTree* argNode = arg.GetEarlyNode(); + + // We should not be passing gc typed args to an unmanaged call. + if (varTypeIsGC(argNode->TypeGet())) + { + // Tolerate byrefs by retyping to native int. + // + // This is needed or we'll generate inconsistent GC info + // for this arg at the call site (gc info says byref, + // pinvoke sig says native int). + // + if (argNode->TypeGet() == TYP_BYREF) + { + argNode->ChangeType(TYP_I_IMPL); + } + else + { + assert(!"*** invalid IL: gc ref passed to unmanaged call"); + } + } + } +} + +//------------------------------------------------------------------------ +// impInitClass: Build a node to initialize the class before accessing the +// field if necessary +// +// Arguments: +// pResolvedToken - The CORINFO_RESOLVED_TOKEN that has been initialized +// by a call to CEEInfo::resolveToken(). +// +// Return Value: If needed, a pointer to the node that will perform the class +// initializtion. Otherwise, nullptr. +// + +GenTree* Compiler::impInitClass(CORINFO_RESOLVED_TOKEN* pResolvedToken) +{ + CorInfoInitClassResult initClassResult = + info.compCompHnd->initClass(pResolvedToken->hField, info.compMethodHnd, impTokenLookupContextHandle); + + if ((initClassResult & CORINFO_INITCLASS_USE_HELPER) == 0) + { + return nullptr; + } + bool runtimeLookup; + + GenTree* node = impParentClassTokenToHandle(pResolvedToken, &runtimeLookup); + + if (node == nullptr) + { + assert(compDonotInline()); + return nullptr; + } + + if (runtimeLookup) + { + node = gtNewHelperCallNode(CORINFO_HELP_INITCLASS, TYP_VOID, node); + } + else + { + // Call the shared non gc static helper, as its the fastest + node = fgGetSharedCCtor(pResolvedToken->hClass); + } + + return node; +} + +GenTree* Compiler::impImportStaticReadOnlyField(void* fldAddr, var_types lclTyp) +{ + GenTree* op1 = nullptr; + +#if defined(DEBUG) + // If we're replaying under SuperPMI, we're going to read the data stored by SuperPMI and use it + // for optimization. Unfortunately, SuperPMI doesn't implement a guarantee on the alignment of + // this data, so for some platforms which don't allow unaligned access (e.g., Linux arm32), + // this can fault. We should fix SuperPMI to guarantee alignment, but that is a big change. + // Instead, simply fix up the data here for future use. + + // This variable should be the largest size element, with the largest alignment requirement, + // and the native C++ compiler should guarantee sufficient alignment. + double aligned_data = 0.0; + void* p_aligned_data = &aligned_data; + if (info.compMethodSuperPMIIndex != -1) + { + switch (lclTyp) + { + case TYP_BOOL: + case TYP_BYTE: + case TYP_UBYTE: + static_assert_no_msg(sizeof(unsigned __int8) == sizeof(bool)); + static_assert_no_msg(sizeof(unsigned __int8) == sizeof(signed char)); + static_assert_no_msg(sizeof(unsigned __int8) == sizeof(unsigned char)); + // No alignment necessary for byte. + break; + + case TYP_SHORT: + case TYP_USHORT: + static_assert_no_msg(sizeof(unsigned __int16) == sizeof(short)); + static_assert_no_msg(sizeof(unsigned __int16) == sizeof(unsigned short)); + if ((size_t)fldAddr % sizeof(unsigned __int16) != 0) + { + *(unsigned __int16*)p_aligned_data = GET_UNALIGNED_16(fldAddr); + fldAddr = p_aligned_data; + } + break; + + case TYP_INT: + case TYP_UINT: + case TYP_FLOAT: + static_assert_no_msg(sizeof(unsigned __int32) == sizeof(int)); + static_assert_no_msg(sizeof(unsigned __int32) == sizeof(unsigned int)); + static_assert_no_msg(sizeof(unsigned __int32) == sizeof(float)); + if ((size_t)fldAddr % sizeof(unsigned __int32) != 0) + { + *(unsigned __int32*)p_aligned_data = GET_UNALIGNED_32(fldAddr); + fldAddr = p_aligned_data; + } + break; + + case TYP_LONG: + case TYP_ULONG: + case TYP_DOUBLE: + static_assert_no_msg(sizeof(unsigned __int64) == sizeof(__int64)); + static_assert_no_msg(sizeof(unsigned __int64) == sizeof(double)); + if ((size_t)fldAddr % sizeof(unsigned __int64) != 0) + { + *(unsigned __int64*)p_aligned_data = GET_UNALIGNED_64(fldAddr); + fldAddr = p_aligned_data; + } + break; + + default: + assert(!"Unexpected lclTyp"); + break; + } + } +#endif // DEBUG + + switch (lclTyp) + { + int ival; + __int64 lval; + double dval; + + case TYP_BOOL: + ival = *((bool*)fldAddr); + goto IVAL_COMMON; + + case TYP_BYTE: + ival = *((signed char*)fldAddr); + goto IVAL_COMMON; + + case TYP_UBYTE: + ival = *((unsigned char*)fldAddr); + goto IVAL_COMMON; + + case TYP_SHORT: + ival = *((short*)fldAddr); + goto IVAL_COMMON; + + case TYP_USHORT: + ival = *((unsigned short*)fldAddr); + goto IVAL_COMMON; + + case TYP_UINT: + case TYP_INT: + ival = *((int*)fldAddr); + IVAL_COMMON: + op1 = gtNewIconNode(ival); + break; + + case TYP_LONG: + case TYP_ULONG: + lval = *((__int64*)fldAddr); + op1 = gtNewLconNode(lval); + break; + + case TYP_FLOAT: + dval = *((float*)fldAddr); + op1 = gtNewDconNode(dval); + op1->gtType = TYP_FLOAT; + break; + + case TYP_DOUBLE: + dval = *((double*)fldAddr); + op1 = gtNewDconNode(dval); + break; + + default: + assert(!"Unexpected lclTyp"); + break; + } + + return op1; +} + +GenTree* Compiler::impImportStaticFieldAccess(CORINFO_RESOLVED_TOKEN* pResolvedToken, + CORINFO_ACCESS_FLAGS access, + CORINFO_FIELD_INFO* pFieldInfo, + var_types lclTyp) +{ + // Ordinary static fields never overlap. RVA statics, however, can overlap (if they're + // mapped to the same ".data" declaration). That said, such mappings only appear to be + // possible with ILASM, and in ILASM-produced (ILONLY) images, RVA statics are always + // read-only (using "stsfld" on them is UB). In mixed-mode assemblies, RVA statics can + // be mutable, but the only current producer of such images, the C++/CLI compiler, does + // not appear to support mapping different fields to the same address. So we will say + // that "mutable overlapping RVA statics" are UB as well. + + // For statics that are not "boxed", the initial address tree will contain the field sequence. + // For those that are, we will attach it later, when adding the indirection for the box, since + // that tree will represent the true address. + bool isBoxedStatic = (pFieldInfo->fieldFlags & CORINFO_FLG_FIELD_STATIC_IN_HEAP) != 0; + bool isSharedStatic = (pFieldInfo->fieldAccessor == CORINFO_FIELD_STATIC_GENERICS_STATIC_HELPER) || + (pFieldInfo->fieldAccessor == CORINFO_FIELD_STATIC_READYTORUN_HELPER); + FieldSeq::FieldKind fieldKind = + isSharedStatic ? FieldSeq::FieldKind::SharedStatic : FieldSeq::FieldKind::SimpleStatic; + + FieldSeq* innerFldSeq; + FieldSeq* outerFldSeq; + if (isBoxedStatic) + { + innerFldSeq = nullptr; + outerFldSeq = GetFieldSeqStore()->Create(pResolvedToken->hField, TARGET_POINTER_SIZE, fieldKind); + } + else + { + bool hasConstAddr = (pFieldInfo->fieldAccessor == CORINFO_FIELD_STATIC_ADDRESS) || + (pFieldInfo->fieldAccessor == CORINFO_FIELD_STATIC_RVA_ADDRESS); + + ssize_t offset; + if (hasConstAddr) + { + offset = reinterpret_cast(info.compCompHnd->getFieldAddress(pResolvedToken->hField)); + assert(offset != 0); + } + else + { + offset = pFieldInfo->offset; + } + + innerFldSeq = GetFieldSeqStore()->Create(pResolvedToken->hField, offset, fieldKind); + outerFldSeq = nullptr; + } + + GenTree* op1; + switch (pFieldInfo->fieldAccessor) + { + case CORINFO_FIELD_STATIC_GENERICS_STATIC_HELPER: + { + assert(!compIsForInlining()); + + // We first call a special helper to get the statics base pointer + op1 = impParentClassTokenToHandle(pResolvedToken); + + // compIsForInlining() is false so we should not get NULL here + assert(op1 != nullptr); + + var_types type = TYP_BYREF; + + switch (pFieldInfo->helper) + { + case CORINFO_HELP_GETGENERICS_NONGCTHREADSTATIC_BASE: + type = TYP_I_IMPL; + break; + case CORINFO_HELP_GETGENERICS_GCSTATIC_BASE: + case CORINFO_HELP_GETGENERICS_NONGCSTATIC_BASE: + case CORINFO_HELP_GETGENERICS_GCTHREADSTATIC_BASE: + break; + default: + assert(!"unknown generic statics helper"); + break; + } + + op1 = gtNewHelperCallNode(pFieldInfo->helper, type, op1); + op1 = gtNewOperNode(GT_ADD, type, op1, gtNewIconNode(pFieldInfo->offset, innerFldSeq)); + } + break; + + case CORINFO_FIELD_STATIC_SHARED_STATIC_HELPER: + { +#ifdef FEATURE_READYTORUN + if (opts.IsReadyToRun()) + { + GenTreeFlags callFlags = GTF_EMPTY; + + if (info.compCompHnd->getClassAttribs(pResolvedToken->hClass) & CORINFO_FLG_BEFOREFIELDINIT) + { + callFlags |= GTF_CALL_HOISTABLE; + } + + op1 = gtNewHelperCallNode(CORINFO_HELP_READYTORUN_STATIC_BASE, TYP_BYREF); + op1->gtFlags |= callFlags; + + op1->AsCall()->setEntryPoint(pFieldInfo->fieldLookup); + } + else +#endif + { + op1 = fgGetStaticsCCtorHelper(pResolvedToken->hClass, pFieldInfo->helper); + } + + op1 = gtNewOperNode(GT_ADD, op1->TypeGet(), op1, gtNewIconNode(pFieldInfo->offset, innerFldSeq)); + break; + } + + case CORINFO_FIELD_STATIC_READYTORUN_HELPER: + { +#ifdef FEATURE_READYTORUN + assert(opts.IsReadyToRun()); + assert(!compIsForInlining()); + CORINFO_LOOKUP_KIND kind; + info.compCompHnd->getLocationOfThisType(info.compMethodHnd, &kind); + assert(kind.needsRuntimeLookup); + + GenTree* ctxTree = getRuntimeContextTree(kind.runtimeLookupKind); + + GenTreeFlags callFlags = GTF_EMPTY; + + if (info.compCompHnd->getClassAttribs(pResolvedToken->hClass) & CORINFO_FLG_BEFOREFIELDINIT) + { + callFlags |= GTF_CALL_HOISTABLE; + } + var_types type = TYP_BYREF; + op1 = gtNewHelperCallNode(CORINFO_HELP_READYTORUN_GENERIC_STATIC_BASE, type, ctxTree); + op1->gtFlags |= callFlags; + + op1->AsCall()->setEntryPoint(pFieldInfo->fieldLookup); + op1 = gtNewOperNode(GT_ADD, type, op1, gtNewIconNode(pFieldInfo->offset, innerFldSeq)); +#else + unreached(); +#endif // FEATURE_READYTORUN + } + break; + + default: + { + // Do we need the address of a static field? + // + if (access & CORINFO_ACCESS_ADDRESS) + { + void** pFldAddr = nullptr; + void* fldAddr = info.compCompHnd->getFieldAddress(pResolvedToken->hField, (void**)&pFldAddr); + + // We should always be able to access this static's address directly. + assert(pFldAddr == nullptr); + + // Create the address node. + GenTreeFlags handleKind = isBoxedStatic ? GTF_ICON_STATIC_BOX_PTR : GTF_ICON_STATIC_HDL; + op1 = gtNewIconHandleNode((size_t)fldAddr, handleKind, innerFldSeq); + INDEBUG(op1->AsIntCon()->gtTargetHandle = reinterpret_cast(pResolvedToken->hField)); + + if (pFieldInfo->fieldFlags & CORINFO_FLG_FIELD_INITCLASS) + { + op1->gtFlags |= GTF_ICON_INITCLASS; + } + } + else // We need the value of a static field + { + // In future, it may be better to just create the right tree here instead of folding it later. + op1 = gtNewFieldRef(lclTyp, pResolvedToken->hField); + + if (pFieldInfo->fieldFlags & CORINFO_FLG_FIELD_INITCLASS) + { + op1->gtFlags |= GTF_FLD_INITCLASS; + } + + if (isBoxedStatic) + { + op1->ChangeType(TYP_REF); // points at boxed object + op1 = gtNewOperNode(GT_ADD, TYP_BYREF, op1, gtNewIconNode(TARGET_POINTER_SIZE, outerFldSeq)); + + if (varTypeIsStruct(lclTyp)) + { + // Constructor adds GTF_GLOB_REF. Note that this is *not* GTF_EXCEPT. + op1 = gtNewObjNode(pFieldInfo->structType, op1); + } + else + { + op1 = gtNewOperNode(GT_IND, lclTyp, op1); + op1->gtFlags |= (GTF_GLOB_REF | GTF_IND_NONFAULTING); + } + } + + return op1; + } + break; + } + } + + if (isBoxedStatic) + { + op1 = gtNewOperNode(GT_IND, TYP_REF, op1); + op1->gtFlags |= (GTF_IND_INVARIANT | GTF_IND_NONFAULTING | GTF_IND_NONNULL); + + op1 = gtNewOperNode(GT_ADD, TYP_BYREF, op1, gtNewIconNode(TARGET_POINTER_SIZE, outerFldSeq)); + } + + if (!(access & CORINFO_ACCESS_ADDRESS)) + { + if (varTypeIsStruct(lclTyp)) + { + // Constructor adds GTF_GLOB_REF. Note that this is *not* GTF_EXCEPT. + op1 = gtNewObjNode(pFieldInfo->structType, op1); + } + else + { + op1 = gtNewOperNode(GT_IND, lclTyp, op1); + op1->gtFlags |= GTF_GLOB_REF; + } + } + + return op1; +} + +// In general try to call this before most of the verification work. Most people expect the access +// exceptions before the verification exceptions. If you do this after, that usually doesn't happen. Turns +// out if you can't access something we also think that you're unverifiable for other reasons. +void Compiler::impHandleAccessAllowed(CorInfoIsAccessAllowedResult result, CORINFO_HELPER_DESC* helperCall) +{ + if (result != CORINFO_ACCESS_ALLOWED) + { + impHandleAccessAllowedInternal(result, helperCall); + } +} + +void Compiler::impHandleAccessAllowedInternal(CorInfoIsAccessAllowedResult result, CORINFO_HELPER_DESC* helperCall) +{ + switch (result) + { + case CORINFO_ACCESS_ALLOWED: + break; + case CORINFO_ACCESS_ILLEGAL: + // if we're verifying, then we need to reject the illegal access to ensure that we don't think the + // method is verifiable. Otherwise, delay the exception to runtime. + if (compIsForImportOnly()) + { + info.compCompHnd->ThrowExceptionForHelper(helperCall); + } + else + { + impInsertHelperCall(helperCall); + } + break; + } +} + +void Compiler::impInsertHelperCall(CORINFO_HELPER_DESC* helperInfo) +{ + assert(helperInfo->helperNum != CORINFO_HELP_UNDEF); + + /* TODO-Review: + * Mark as CSE'able, and hoistable. Consider marking hoistable unless you're in the inlinee. + * Also, consider sticking this in the first basic block. + */ + GenTreeCall* callout = gtNewHelperCallNode(helperInfo->helperNum, TYP_VOID); + // Add the arguments + for (unsigned i = helperInfo->numArgs; i > 0; --i) + { + const CORINFO_HELPER_ARG& helperArg = helperInfo->args[i - 1]; + GenTree* currentArg = nullptr; + switch (helperArg.argType) + { + case CORINFO_HELPER_ARG_TYPE_Field: + info.compCompHnd->classMustBeLoadedBeforeCodeIsRun( + info.compCompHnd->getFieldClass(helperArg.fieldHandle)); + currentArg = gtNewIconEmbFldHndNode(helperArg.fieldHandle); + break; + case CORINFO_HELPER_ARG_TYPE_Method: + info.compCompHnd->methodMustBeLoadedBeforeCodeIsRun(helperArg.methodHandle); + currentArg = gtNewIconEmbMethHndNode(helperArg.methodHandle); + break; + case CORINFO_HELPER_ARG_TYPE_Class: + info.compCompHnd->classMustBeLoadedBeforeCodeIsRun(helperArg.classHandle); + currentArg = gtNewIconEmbClsHndNode(helperArg.classHandle); + break; + case CORINFO_HELPER_ARG_TYPE_Module: + currentArg = gtNewIconEmbScpHndNode(helperArg.moduleHandle); + break; + case CORINFO_HELPER_ARG_TYPE_Const: + currentArg = gtNewIconNode(helperArg.constant); + break; + default: + NO_WAY("Illegal helper arg type"); + } + callout->gtArgs.PushFront(this, NewCallArg::Primitive(currentArg)); + } + + impAppendTree(callout, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); +} + +//------------------------------------------------------------------------ +// impTailCallRetTypeCompatible: Checks whether the return types of caller +// and callee are compatible so that calle can be tail called. +// sizes are not supported integral type sizes return values to temps. +// +// Arguments: +// allowWidening -- whether to allow implicit widening by the callee. +// For instance, allowing int32 -> int16 tailcalls. +// The managed calling convention allows this, but +// we don't want explicit tailcalls to depend on this +// detail of the managed calling convention. +// callerRetType -- the caller's return type +// callerRetTypeClass - the caller's return struct type +// callerCallConv -- calling convention of the caller +// calleeRetType -- the callee's return type +// calleeRetTypeClass - the callee return struct type +// calleeCallConv -- calling convention of the callee +// +// Returns: +// True if the tailcall types are compatible. +// +// Remarks: +// Note that here we don't check compatibility in IL Verifier sense, but on the +// lines of return types getting returned in the same return register. +bool Compiler::impTailCallRetTypeCompatible(bool allowWidening, + var_types callerRetType, + CORINFO_CLASS_HANDLE callerRetTypeClass, + CorInfoCallConvExtension callerCallConv, + var_types calleeRetType, + CORINFO_CLASS_HANDLE calleeRetTypeClass, + CorInfoCallConvExtension calleeCallConv) +{ + // Early out if the types are the same. + if (callerRetType == calleeRetType) + { + return true; + } + + // For integral types the managed calling convention dictates that callee + // will widen the return value to 4 bytes, so we can allow implicit widening + // in managed to managed tailcalls when dealing with <= 4 bytes. + bool isManaged = + (callerCallConv == CorInfoCallConvExtension::Managed) && (calleeCallConv == CorInfoCallConvExtension::Managed); + + if (allowWidening && isManaged && varTypeIsIntegral(callerRetType) && varTypeIsIntegral(calleeRetType) && + (genTypeSize(callerRetType) <= 4) && (genTypeSize(calleeRetType) <= genTypeSize(callerRetType))) + { + return true; + } + + // If the class handles are the same and not null, the return types are compatible. + if ((callerRetTypeClass != nullptr) && (callerRetTypeClass == calleeRetTypeClass)) + { + return true; + } + +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) + // Jit64 compat: + if (callerRetType == TYP_VOID) + { + // This needs to be allowed to support the following IL pattern that Jit64 allows: + // tail.call + // pop + // ret + // + // Note that the above IL pattern is not valid as per IL verification rules. + // Therefore, only full trust code can take advantage of this pattern. + return true; + } + + // These checks return true if the return value type sizes are the same and + // get returned in the same return register i.e. caller doesn't need to normalize + // return value. Some of the tail calls permitted by below checks would have + // been rejected by IL Verifier before we reached here. Therefore, only full + // trust code can make those tail calls. + unsigned callerRetTypeSize = 0; + unsigned calleeRetTypeSize = 0; + bool isCallerRetTypMBEnreg = VarTypeIsMultiByteAndCanEnreg(callerRetType, callerRetTypeClass, &callerRetTypeSize, + true, info.compIsVarArgs, callerCallConv); + bool isCalleeRetTypMBEnreg = VarTypeIsMultiByteAndCanEnreg(calleeRetType, calleeRetTypeClass, &calleeRetTypeSize, + true, info.compIsVarArgs, calleeCallConv); + + if (varTypeIsIntegral(callerRetType) || isCallerRetTypMBEnreg) + { + return (varTypeIsIntegral(calleeRetType) || isCalleeRetTypMBEnreg) && (callerRetTypeSize == calleeRetTypeSize); + } +#endif // TARGET_AMD64 || TARGET_ARM64 || TARGET_LOONGARCH64 + + return false; +} + +/******************************************************************************** + * + * Returns true if the current opcode and and the opcodes following it correspond + * to a supported tail call IL pattern. + * + */ +bool Compiler::impIsTailCallILPattern( + bool tailPrefixed, OPCODE curOpcode, const BYTE* codeAddrOfNextOpcode, const BYTE* codeEnd, bool isRecursive) +{ + // Bail out if the current opcode is not a call. + if (!impOpcodeIsCallOpcode(curOpcode)) + { + return false; + } + +#if !FEATURE_TAILCALL_OPT_SHARED_RETURN + // If shared ret tail opt is not enabled, we will enable + // it for recursive methods. + if (isRecursive) +#endif + { + // we can actually handle if the ret is in a fallthrough block, as long as that is the only part of the + // sequence. Make sure we don't go past the end of the IL however. + codeEnd = min(codeEnd + 1, info.compCode + info.compILCodeSize); + } + + // Bail out if there is no next opcode after call + if (codeAddrOfNextOpcode >= codeEnd) + { + return false; + } + + OPCODE nextOpcode = (OPCODE)getU1LittleEndian(codeAddrOfNextOpcode); + + return (nextOpcode == CEE_RET); +} + +/***************************************************************************** + * + * Determine whether the call could be converted to an implicit tail call + * + */ +bool Compiler::impIsImplicitTailCallCandidate( + OPCODE opcode, const BYTE* codeAddrOfNextOpcode, const BYTE* codeEnd, int prefixFlags, bool isRecursive) +{ + +#if FEATURE_TAILCALL_OPT + if (!opts.compTailCallOpt) + { + return false; + } + + if (opts.OptimizationDisabled()) + { + return false; + } + + // must not be tail prefixed + if (prefixFlags & PREFIX_TAILCALL_EXPLICIT) + { + return false; + } + +#if !FEATURE_TAILCALL_OPT_SHARED_RETURN + // the block containing call is marked as BBJ_RETURN + // We allow shared ret tail call optimization on recursive calls even under + // !FEATURE_TAILCALL_OPT_SHARED_RETURN. + if (!isRecursive && (compCurBB->bbJumpKind != BBJ_RETURN)) + return false; +#endif // !FEATURE_TAILCALL_OPT_SHARED_RETURN + + // must be call+ret or call+pop+ret + if (!impIsTailCallILPattern(false, opcode, codeAddrOfNextOpcode, codeEnd, isRecursive)) + { + return false; + } + + return true; +#else + return false; +#endif // FEATURE_TAILCALL_OPT +} + +//------------------------------------------------------------------------ +// impImportCall: import a call-inspiring opcode +// +// Arguments: +// opcode - opcode that inspires the call +// pResolvedToken - resolved token for the call target +// pConstrainedResolvedToken - resolved constraint token (or nullptr) +// newObjThis - tree for this pointer or uninitialized newobj temp (or nullptr) +// prefixFlags - IL prefix flags for the call +// callInfo - EE supplied info for the call +// rawILOffset - IL offset of the opcode, used for guarded devirtualization. +// +// Returns: +// Type of the call's return value. +// If we're importing an inlinee and have realized the inline must fail, the call return type should be TYP_UNDEF. +// However we can't assert for this here yet because there are cases we miss. See issue #13272. +// +// +// Notes: +// opcode can be CEE_CALL, CEE_CALLI, CEE_CALLVIRT, or CEE_NEWOBJ. +// +// For CEE_NEWOBJ, newobjThis should be the temp grabbed for the allocated +// uninitialized object. + +#ifdef _PREFAST_ +#pragma warning(push) +#pragma warning(disable : 21000) // Suppress PREFast warning about overly large function +#endif + +var_types Compiler::impImportCall(OPCODE opcode, + CORINFO_RESOLVED_TOKEN* pResolvedToken, + CORINFO_RESOLVED_TOKEN* pConstrainedResolvedToken, + GenTree* newobjThis, + int prefixFlags, + CORINFO_CALL_INFO* callInfo, + IL_OFFSET rawILOffset) +{ + assert(opcode == CEE_CALL || opcode == CEE_CALLVIRT || opcode == CEE_NEWOBJ || opcode == CEE_CALLI); + + // The current statement DI may not refer to the exact call, but for calls + // we wish to be able to attach the exact IL instruction to get "return + // value" support in the debugger, so create one with the exact IL offset. + DebugInfo di = impCreateDIWithCurrentStackInfo(rawILOffset, true); + + var_types callRetTyp = TYP_COUNT; + CORINFO_SIG_INFO* sig = nullptr; + CORINFO_METHOD_HANDLE methHnd = nullptr; + CORINFO_CLASS_HANDLE clsHnd = nullptr; + unsigned clsFlags = 0; + unsigned mflags = 0; + GenTree* call = nullptr; + CORINFO_THIS_TRANSFORM constraintCallThisTransform = CORINFO_NO_THIS_TRANSFORM; + CORINFO_CONTEXT_HANDLE exactContextHnd = nullptr; + bool exactContextNeedsRuntimeLookup = false; + bool canTailCall = true; + const char* szCanTailCallFailReason = nullptr; + const int tailCallFlags = (prefixFlags & PREFIX_TAILCALL); + const bool isReadonlyCall = (prefixFlags & PREFIX_READONLY) != 0; + + methodPointerInfo* ldftnInfo = nullptr; + + // Synchronized methods need to call CORINFO_HELP_MON_EXIT at the end. We could + // do that before tailcalls, but that is probably not the intended + // semantic. So just disallow tailcalls from synchronized methods. + // Also, popping arguments in a varargs function is more work and NYI + // If we have a security object, we have to keep our frame around for callers + // to see any imperative security. + // Reverse P/Invokes need a call to CORINFO_HELP_JIT_REVERSE_PINVOKE_EXIT + // at the end, so tailcalls should be disabled. + if (info.compFlags & CORINFO_FLG_SYNCH) + { + canTailCall = false; + szCanTailCallFailReason = "Caller is synchronized"; + } + else if (opts.IsReversePInvoke()) + { + canTailCall = false; + szCanTailCallFailReason = "Caller is Reverse P/Invoke"; + } +#if !FEATURE_FIXED_OUT_ARGS + else if (info.compIsVarArgs) + { + canTailCall = false; + szCanTailCallFailReason = "Caller is varargs"; + } +#endif // FEATURE_FIXED_OUT_ARGS + + // We only need to cast the return value of pinvoke inlined calls that return small types + + bool checkForSmallType = false; + bool bIntrinsicImported = false; + + CORINFO_SIG_INFO calliSig; + NewCallArg extraArg; + + /*------------------------------------------------------------------------- + * First create the call node + */ + + if (opcode == CEE_CALLI) + { + if (IsTargetAbi(CORINFO_NATIVEAOT_ABI)) + { + // See comment in impCheckForPInvokeCall + BasicBlock* block = compIsForInlining() ? impInlineInfo->iciBlock : compCurBB; + if (info.compCompHnd->convertPInvokeCalliToCall(pResolvedToken, !impCanPInvokeInlineCallSite(block))) + { + eeGetCallInfo(pResolvedToken, nullptr, CORINFO_CALLINFO_ALLOWINSTPARAM, callInfo); + return impImportCall(CEE_CALL, pResolvedToken, nullptr, nullptr, prefixFlags, callInfo, rawILOffset); + } + } + + /* Get the call site sig */ + eeGetSig(pResolvedToken->token, pResolvedToken->tokenScope, pResolvedToken->tokenContext, &calliSig); + + callRetTyp = JITtype2varType(calliSig.retType); + + call = impImportIndirectCall(&calliSig, di); + + // We don't know the target method, so we have to infer the flags, or + // assume the worst-case. + mflags = (calliSig.callConv & CORINFO_CALLCONV_HASTHIS) ? 0 : CORINFO_FLG_STATIC; + +#ifdef DEBUG + if (verbose) + { + unsigned structSize = (callRetTyp == TYP_STRUCT) ? eeTryGetClassSize(calliSig.retTypeSigClass) : 0; + printf("\nIn Compiler::impImportCall: opcode is %s, kind=%d, callRetType is %s, structSize is %u\n", + opcodeNames[opcode], callInfo->kind, varTypeName(callRetTyp), structSize); + } +#endif + sig = &calliSig; + } + else // (opcode != CEE_CALLI) + { + NamedIntrinsic ni = NI_Illegal; + + // Passing CORINFO_CALLINFO_ALLOWINSTPARAM indicates that this JIT is prepared to + // supply the instantiation parameters necessary to make direct calls to underlying + // shared generic code, rather than calling through instantiating stubs. If the + // returned signature has CORINFO_CALLCONV_PARAMTYPE then this indicates that the JIT + // must indeed pass an instantiation parameter. + + methHnd = callInfo->hMethod; + + sig = &(callInfo->sig); + callRetTyp = JITtype2varType(sig->retType); + + mflags = callInfo->methodFlags; + +#ifdef DEBUG + if (verbose) + { + unsigned structSize = (callRetTyp == TYP_STRUCT) ? eeTryGetClassSize(sig->retTypeSigClass) : 0; + printf("\nIn Compiler::impImportCall: opcode is %s, kind=%d, callRetType is %s, structSize is %u\n", + opcodeNames[opcode], callInfo->kind, varTypeName(callRetTyp), structSize); + } +#endif + if (compIsForInlining()) + { + /* Does the inlinee use StackCrawlMark */ + + if (mflags & CORINFO_FLG_DONT_INLINE_CALLER) + { + compInlineResult->NoteFatal(InlineObservation::CALLEE_STACK_CRAWL_MARK); + return TYP_UNDEF; + } + + /* For now ignore varargs */ + if ((sig->callConv & CORINFO_CALLCONV_MASK) == CORINFO_CALLCONV_NATIVEVARARG) + { + compInlineResult->NoteFatal(InlineObservation::CALLEE_HAS_NATIVE_VARARGS); + return TYP_UNDEF; + } + + if ((sig->callConv & CORINFO_CALLCONV_MASK) == CORINFO_CALLCONV_VARARG) + { + compInlineResult->NoteFatal(InlineObservation::CALLEE_HAS_MANAGED_VARARGS); + return TYP_UNDEF; + } + + if ((mflags & CORINFO_FLG_VIRTUAL) && (sig->sigInst.methInstCount != 0) && (opcode == CEE_CALLVIRT)) + { + compInlineResult->NoteFatal(InlineObservation::CALLEE_IS_GENERIC_VIRTUAL); + return TYP_UNDEF; + } + } + + clsHnd = pResolvedToken->hClass; + + clsFlags = callInfo->classFlags; + +#ifdef DEBUG + // If this is a call to JitTestLabel.Mark, do "early inlining", and record the test attribute. + + // This recognition should really be done by knowing the methHnd of the relevant Mark method(s). + // These should be in corelib.h, and available through a JIT/EE interface call. + const char* modName; + const char* className; + const char* methodName; + if ((className = eeGetClassName(clsHnd)) != nullptr && + strcmp(className, "System.Runtime.CompilerServices.JitTestLabel") == 0 && + (methodName = eeGetMethodName(methHnd, &modName)) != nullptr && strcmp(methodName, "Mark") == 0) + { + return impImportJitTestLabelMark(sig->numArgs); + } +#endif // DEBUG + + // Factor this into getCallInfo + bool isSpecialIntrinsic = false; + if ((mflags & CORINFO_FLG_INTRINSIC) != 0) + { + const bool isTailCall = canTailCall && (tailCallFlags != 0); + + call = + impIntrinsic(newobjThis, clsHnd, methHnd, sig, mflags, pResolvedToken->token, isReadonlyCall, + isTailCall, pConstrainedResolvedToken, callInfo->thisTransform, &ni, &isSpecialIntrinsic); + + if (compDonotInline()) + { + return TYP_UNDEF; + } + + if (call != nullptr) + { +#ifdef FEATURE_READYTORUN + if (call->OperGet() == GT_INTRINSIC) + { + if (opts.IsReadyToRun()) + { + noway_assert(callInfo->kind == CORINFO_CALL); + call->AsIntrinsic()->gtEntryPoint = callInfo->codePointerLookup.constLookup; + } + else + { + call->AsIntrinsic()->gtEntryPoint.addr = nullptr; + call->AsIntrinsic()->gtEntryPoint.accessType = IAT_VALUE; + } + } +#endif + + bIntrinsicImported = true; + goto DONE_CALL; + } + } + +#ifdef FEATURE_SIMD + call = impSIMDIntrinsic(opcode, newobjThis, clsHnd, methHnd, sig, mflags, pResolvedToken->token); + if (call != nullptr) + { + bIntrinsicImported = true; + goto DONE_CALL; + } +#endif // FEATURE_SIMD + + if ((mflags & CORINFO_FLG_VIRTUAL) && (mflags & CORINFO_FLG_EnC) && (opcode == CEE_CALLVIRT)) + { + NO_WAY("Virtual call to a function added via EnC is not supported"); + } + + if ((sig->callConv & CORINFO_CALLCONV_MASK) != CORINFO_CALLCONV_DEFAULT && + (sig->callConv & CORINFO_CALLCONV_MASK) != CORINFO_CALLCONV_VARARG && + (sig->callConv & CORINFO_CALLCONV_MASK) != CORINFO_CALLCONV_NATIVEVARARG) + { + BADCODE("Bad calling convention"); + } + + //------------------------------------------------------------------------- + // Construct the call node + // + // Work out what sort of call we're making. + // Dispense with virtual calls implemented via LDVIRTFTN immediately. + + constraintCallThisTransform = callInfo->thisTransform; + exactContextHnd = callInfo->contextHandle; + exactContextNeedsRuntimeLookup = callInfo->exactContextNeedsRuntimeLookup; + + switch (callInfo->kind) + { + case CORINFO_VIRTUALCALL_STUB: + { + assert(!(mflags & CORINFO_FLG_STATIC)); // can't call a static method + assert(!(clsFlags & CORINFO_FLG_VALUECLASS)); + if (callInfo->stubLookup.lookupKind.needsRuntimeLookup) + { + if (callInfo->stubLookup.lookupKind.runtimeLookupKind == CORINFO_LOOKUP_NOT_SUPPORTED) + { + // Runtime does not support inlining of all shapes of runtime lookups + // Inlining has to be aborted in such a case + compInlineResult->NoteFatal(InlineObservation::CALLSITE_HAS_COMPLEX_HANDLE); + return TYP_UNDEF; + } + + GenTree* stubAddr = impRuntimeLookupToTree(pResolvedToken, &callInfo->stubLookup, methHnd); + assert(!compDonotInline()); + + // This is the rough code to set up an indirect stub call + assert(stubAddr != nullptr); + + // The stubAddr may be a + // complex expression. As it is evaluated after the args, + // it may cause registered args to be spilled. Simply spill it. + + unsigned lclNum = lvaGrabTemp(true DEBUGARG("VirtualCall with runtime lookup")); + impAssignTempGen(lclNum, stubAddr, (unsigned)CHECK_SPILL_NONE); + stubAddr = gtNewLclvNode(lclNum, TYP_I_IMPL); + + // Create the actual call node + + assert((sig->callConv & CORINFO_CALLCONV_MASK) != CORINFO_CALLCONV_VARARG && + (sig->callConv & CORINFO_CALLCONV_MASK) != CORINFO_CALLCONV_NATIVEVARARG); + + call = gtNewIndCallNode(stubAddr, callRetTyp); + + call->gtFlags |= GTF_EXCEPT | (stubAddr->gtFlags & GTF_GLOB_EFFECT); + call->gtFlags |= GTF_CALL_VIRT_STUB; + +#ifdef TARGET_X86 + // No tailcalls allowed for these yet... + canTailCall = false; + szCanTailCallFailReason = "VirtualCall with runtime lookup"; +#endif + } + else + { + // The stub address is known at compile time + call = gtNewCallNode(CT_USER_FUNC, callInfo->hMethod, callRetTyp, di); + call->AsCall()->gtStubCallStubAddr = callInfo->stubLookup.constLookup.addr; + call->gtFlags |= GTF_CALL_VIRT_STUB; + assert(callInfo->stubLookup.constLookup.accessType != IAT_PPVALUE && + callInfo->stubLookup.constLookup.accessType != IAT_RELPVALUE); + if (callInfo->stubLookup.constLookup.accessType == IAT_PVALUE) + { + call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_VIRTSTUB_REL_INDIRECT; + } + } + +#ifdef FEATURE_READYTORUN + if (opts.IsReadyToRun()) + { + // Null check is sometimes needed for ready to run to handle + // non-virtual <-> virtual changes between versions + if (callInfo->nullInstanceCheck) + { + call->gtFlags |= GTF_CALL_NULLCHECK; + } + } +#endif + + break; + } + + case CORINFO_VIRTUALCALL_VTABLE: + { + assert(!(mflags & CORINFO_FLG_STATIC)); // can't call a static method + assert(!(clsFlags & CORINFO_FLG_VALUECLASS)); + call = gtNewCallNode(CT_USER_FUNC, callInfo->hMethod, callRetTyp, di); + call->gtFlags |= GTF_CALL_VIRT_VTABLE; + + // Should we expand virtual call targets early for this method? + // + if (opts.compExpandCallsEarly) + { + // Mark this method to expand the virtual call target early in fgMorpgCall + call->AsCall()->SetExpandedEarly(); + } + break; + } + + case CORINFO_VIRTUALCALL_LDVIRTFTN: + { + if (compIsForInlining()) + { + compInlineResult->NoteFatal(InlineObservation::CALLSITE_HAS_CALL_VIA_LDVIRTFTN); + return TYP_UNDEF; + } + + assert(!(mflags & CORINFO_FLG_STATIC)); // can't call a static method + assert(!(clsFlags & CORINFO_FLG_VALUECLASS)); + // OK, We've been told to call via LDVIRTFTN, so just + // take the call now.... + call = gtNewIndCallNode(nullptr, callRetTyp, di); + + impPopCallArgs(sig, call->AsCall()); + + GenTree* thisPtr = impPopStack().val; + thisPtr = impTransformThis(thisPtr, pConstrainedResolvedToken, callInfo->thisTransform); + assert(thisPtr != nullptr); + + // Clone the (possibly transformed) "this" pointer + GenTree* thisPtrCopy; + thisPtr = impCloneExpr(thisPtr, &thisPtrCopy, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, + nullptr DEBUGARG("LDVIRTFTN this pointer")); + + GenTree* fptr = impImportLdvirtftn(thisPtr, pResolvedToken, callInfo); + assert(fptr != nullptr); + + call->AsCall() + ->gtArgs.PushFront(this, NewCallArg::Primitive(thisPtrCopy).WellKnown(WellKnownArg::ThisPointer)); + + // Now make an indirect call through the function pointer + + unsigned lclNum = lvaGrabTemp(true DEBUGARG("VirtualCall through function pointer")); + impAssignTempGen(lclNum, fptr, (unsigned)CHECK_SPILL_ALL); + fptr = gtNewLclvNode(lclNum, TYP_I_IMPL); + + call->AsCall()->gtCallAddr = fptr; + call->gtFlags |= GTF_EXCEPT | (fptr->gtFlags & GTF_GLOB_EFFECT); + + if ((sig->sigInst.methInstCount != 0) && IsTargetAbi(CORINFO_NATIVEAOT_ABI)) + { + // NativeAOT generic virtual method: need to handle potential fat function pointers + addFatPointerCandidate(call->AsCall()); + } +#ifdef FEATURE_READYTORUN + if (opts.IsReadyToRun()) + { + // Null check is needed for ready to run to handle + // non-virtual <-> virtual changes between versions + call->gtFlags |= GTF_CALL_NULLCHECK; + } +#endif + + // Sine we are jumping over some code, check that its OK to skip that code + assert((sig->callConv & CORINFO_CALLCONV_MASK) != CORINFO_CALLCONV_VARARG && + (sig->callConv & CORINFO_CALLCONV_MASK) != CORINFO_CALLCONV_NATIVEVARARG); + goto DONE; + } + + case CORINFO_CALL: + { + // This is for a non-virtual, non-interface etc. call + call = gtNewCallNode(CT_USER_FUNC, callInfo->hMethod, callRetTyp, di); + + // We remove the nullcheck for the GetType call intrinsic. + // TODO-CQ: JIT64 does not introduce the null check for many more helper calls + // and intrinsics. + if (callInfo->nullInstanceCheck && + !((mflags & CORINFO_FLG_INTRINSIC) != 0 && (ni == NI_System_Object_GetType))) + { + call->gtFlags |= GTF_CALL_NULLCHECK; + } + +#ifdef FEATURE_READYTORUN + if (opts.IsReadyToRun()) + { + call->AsCall()->setEntryPoint(callInfo->codePointerLookup.constLookup); + } +#endif + break; + } + + case CORINFO_CALL_CODE_POINTER: + { + // The EE has asked us to call by computing a code pointer and then doing an + // indirect call. This is because a runtime lookup is required to get the code entry point. + + // These calls always follow a uniform calling convention, i.e. no extra hidden params + assert((sig->callConv & CORINFO_CALLCONV_PARAMTYPE) == 0); + + assert((sig->callConv & CORINFO_CALLCONV_MASK) != CORINFO_CALLCONV_VARARG); + assert((sig->callConv & CORINFO_CALLCONV_MASK) != CORINFO_CALLCONV_NATIVEVARARG); + + GenTree* fptr = + impLookupToTree(pResolvedToken, &callInfo->codePointerLookup, GTF_ICON_FTN_ADDR, callInfo->hMethod); + + if (compDonotInline()) + { + return TYP_UNDEF; + } + + // Now make an indirect call through the function pointer + + unsigned lclNum = lvaGrabTemp(true DEBUGARG("Indirect call through function pointer")); + impAssignTempGen(lclNum, fptr, (unsigned)CHECK_SPILL_ALL); + fptr = gtNewLclvNode(lclNum, TYP_I_IMPL); + + call = gtNewIndCallNode(fptr, callRetTyp, di); + call->gtFlags |= GTF_EXCEPT | (fptr->gtFlags & GTF_GLOB_EFFECT); + if (callInfo->nullInstanceCheck) + { + call->gtFlags |= GTF_CALL_NULLCHECK; + } + + break; + } + + default: + assert(!"unknown call kind"); + break; + } + + //------------------------------------------------------------------------- + // Set more flags + + PREFIX_ASSUME(call != nullptr); + + if (mflags & CORINFO_FLG_NOGCCHECK) + { + call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_NOGCCHECK; + } + + // Mark call if it's one of the ones we will maybe treat as an intrinsic + if (isSpecialIntrinsic) + { + call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_SPECIAL_INTRINSIC; + } + } + assert(sig); + assert(clsHnd || (opcode == CEE_CALLI)); // We're never verifying for CALLI, so this is not set. + + /* Some sanity checks */ + + // CALL_VIRT and NEWOBJ must have a THIS pointer + assert((opcode != CEE_CALLVIRT && opcode != CEE_NEWOBJ) || (sig->callConv & CORINFO_CALLCONV_HASTHIS)); + // static bit and hasThis are negations of one another + assert(((mflags & CORINFO_FLG_STATIC) != 0) == ((sig->callConv & CORINFO_CALLCONV_HASTHIS) == 0)); + assert(call != nullptr); + + /*------------------------------------------------------------------------- + * Check special-cases etc + */ + + /* Special case - Check if it is a call to Delegate.Invoke(). */ + + if (mflags & CORINFO_FLG_DELEGATE_INVOKE) + { + assert(!(mflags & CORINFO_FLG_STATIC)); // can't call a static method + assert(mflags & CORINFO_FLG_FINAL); + + /* Set the delegate flag */ + call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_DELEGATE_INV; + + if (callInfo->wrapperDelegateInvoke) + { + call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_WRAPPER_DELEGATE_INV; + } + + if (opcode == CEE_CALLVIRT) + { + assert(mflags & CORINFO_FLG_FINAL); + + /* It should have the GTF_CALL_NULLCHECK flag set. Reset it */ + assert(call->gtFlags & GTF_CALL_NULLCHECK); + call->gtFlags &= ~GTF_CALL_NULLCHECK; + } + } + + CORINFO_CLASS_HANDLE actualMethodRetTypeSigClass; + actualMethodRetTypeSigClass = sig->retTypeSigClass; + + /* Check for varargs */ + if (!compFeatureVarArg() && ((sig->callConv & CORINFO_CALLCONV_MASK) == CORINFO_CALLCONV_VARARG || + (sig->callConv & CORINFO_CALLCONV_MASK) == CORINFO_CALLCONV_NATIVEVARARG)) + { + BADCODE("Varargs not supported."); + } + + if ((sig->callConv & CORINFO_CALLCONV_MASK) == CORINFO_CALLCONV_VARARG || + (sig->callConv & CORINFO_CALLCONV_MASK) == CORINFO_CALLCONV_NATIVEVARARG) + { + assert(!compIsForInlining()); + + /* Set the right flags */ + + call->gtFlags |= GTF_CALL_POP_ARGS; + call->AsCall()->gtArgs.SetIsVarArgs(); + + /* Can't allow tailcall for varargs as it is caller-pop. The caller + will be expecting to pop a certain number of arguments, but if we + tailcall to a function with a different number of arguments, we + are hosed. There are ways around this (caller remembers esp value, + varargs is not caller-pop, etc), but not worth it. */ + CLANG_FORMAT_COMMENT_ANCHOR; + +#ifdef TARGET_X86 + if (canTailCall) + { + canTailCall = false; + szCanTailCallFailReason = "Callee is varargs"; + } +#endif + + /* Get the total number of arguments - this is already correct + * for CALLI - for methods we have to get it from the call site */ + + if (opcode != CEE_CALLI) + { +#ifdef DEBUG + unsigned numArgsDef = sig->numArgs; +#endif + eeGetCallSiteSig(pResolvedToken->token, pResolvedToken->tokenScope, pResolvedToken->tokenContext, sig); + + // For vararg calls we must be sure to load the return type of the + // method actually being called, as well as the return types of the + // specified in the vararg signature. With type equivalency, these types + // may not be the same. + if (sig->retTypeSigClass != actualMethodRetTypeSigClass) + { + if (actualMethodRetTypeSigClass != nullptr && sig->retType != CORINFO_TYPE_CLASS && + sig->retType != CORINFO_TYPE_BYREF && sig->retType != CORINFO_TYPE_PTR && + sig->retType != CORINFO_TYPE_VAR) + { + // Make sure that all valuetypes (including enums) that we push are loaded. + // This is to guarantee that if a GC is triggerred from the prestub of this methods, + // all valuetypes in the method signature are already loaded. + // We need to be able to find the size of the valuetypes, but we cannot + // do a class-load from within GC. + info.compCompHnd->classMustBeLoadedBeforeCodeIsRun(actualMethodRetTypeSigClass); + } + } + + assert(numArgsDef <= sig->numArgs); + } + + /* We will have "cookie" as the last argument but we cannot push + * it on the operand stack because we may overflow, so we append it + * to the arg list next after we pop them */ + } + + //--------------------------- Inline NDirect ------------------------------ + + // For inline cases we technically should look at both the current + // block and the call site block (or just the latter if we've + // fused the EH trees). However the block-related checks pertain to + // EH and we currently won't inline a method with EH. So for + // inlinees, just checking the call site block is sufficient. + { + // New lexical block here to avoid compilation errors because of GOTOs. + BasicBlock* block = compIsForInlining() ? impInlineInfo->iciBlock : compCurBB; + impCheckForPInvokeCall(call->AsCall(), methHnd, sig, mflags, block); + } + +#ifdef UNIX_X86_ABI + // On Unix x86 we use caller-cleaned convention. + if ((call->gtFlags & GTF_CALL_UNMANAGED) == 0) + call->gtFlags |= GTF_CALL_POP_ARGS; +#endif // UNIX_X86_ABI + + if (call->gtFlags & GTF_CALL_UNMANAGED) + { + // We set up the unmanaged call by linking the frame, disabling GC, etc + // This needs to be cleaned up on return. + // In addition, native calls have different normalization rules than managed code + // (managed calling convention always widens return values in the callee) + if (canTailCall) + { + canTailCall = false; + szCanTailCallFailReason = "Callee is native"; + } + + checkForSmallType = true; + + impPopArgsForUnmanagedCall(call->AsCall(), sig); + + goto DONE; + } + else if ((opcode == CEE_CALLI) && ((sig->callConv & CORINFO_CALLCONV_MASK) != CORINFO_CALLCONV_DEFAULT) && + ((sig->callConv & CORINFO_CALLCONV_MASK) != CORINFO_CALLCONV_VARARG)) + { + if (!info.compCompHnd->canGetCookieForPInvokeCalliSig(sig)) + { + // Normally this only happens with inlining. + // However, a generic method (or type) being NGENd into another module + // can run into this issue as well. There's not an easy fall-back for NGEN + // so instead we fallback to JIT. + if (compIsForInlining()) + { + compInlineResult->NoteFatal(InlineObservation::CALLSITE_CANT_EMBED_PINVOKE_COOKIE); + } + else + { + IMPL_LIMITATION("Can't get PInvoke cookie (cross module generics)"); + } + + return TYP_UNDEF; + } + + GenTree* cookie = eeGetPInvokeCookie(sig); + + // This cookie is required to be either a simple GT_CNS_INT or + // an indirection of a GT_CNS_INT + // + GenTree* cookieConst = cookie; + if (cookie->gtOper == GT_IND) + { + cookieConst = cookie->AsOp()->gtOp1; + } + assert(cookieConst->gtOper == GT_CNS_INT); + + // Setting GTF_DONT_CSE on the GT_CNS_INT as well as on the GT_IND (if it exists) will ensure that + // we won't allow this tree to participate in any CSE logic + // + cookie->gtFlags |= GTF_DONT_CSE; + cookieConst->gtFlags |= GTF_DONT_CSE; + + call->AsCall()->gtCallCookie = cookie; + + if (canTailCall) + { + canTailCall = false; + szCanTailCallFailReason = "PInvoke calli"; + } + } + + /*------------------------------------------------------------------------- + * Create the argument list + */ + + //------------------------------------------------------------------------- + // Special case - for varargs we have an implicit last argument + + if ((sig->callConv & CORINFO_CALLCONV_MASK) == CORINFO_CALLCONV_VARARG) + { + assert(!compIsForInlining()); + + void *varCookie, *pVarCookie; + if (!info.compCompHnd->canGetVarArgsHandle(sig)) + { + compInlineResult->NoteFatal(InlineObservation::CALLSITE_CANT_EMBED_VARARGS_COOKIE); + return TYP_UNDEF; + } + + varCookie = info.compCompHnd->getVarArgsHandle(sig, &pVarCookie); + assert((!varCookie) != (!pVarCookie)); + GenTree* cookieNode = gtNewIconEmbHndNode(varCookie, pVarCookie, GTF_ICON_VARG_HDL, sig); + assert(extraArg.Node == nullptr); + extraArg = NewCallArg::Primitive(cookieNode).WellKnown(WellKnownArg::VarArgsCookie); + } + + //------------------------------------------------------------------------- + // Extra arg for shared generic code and array methods + // + // Extra argument containing instantiation information is passed in the + // following circumstances: + // (a) To the "Address" method on array classes; the extra parameter is + // the array's type handle (a TypeDesc) + // (b) To shared-code instance methods in generic structs; the extra parameter + // is the struct's type handle (a vtable ptr) + // (c) To shared-code per-instantiation non-generic static methods in generic + // classes and structs; the extra parameter is the type handle + // (d) To shared-code generic methods; the extra parameter is an + // exact-instantiation MethodDesc + // + // We also set the exact type context associated with the call so we can + // inline the call correctly later on. + + if (sig->callConv & CORINFO_CALLCONV_PARAMTYPE) + { + assert(call->AsCall()->gtCallType == CT_USER_FUNC); + if (clsHnd == nullptr) + { + NO_WAY("CALLI on parameterized type"); + } + + assert(opcode != CEE_CALLI); + + GenTree* instParam; + bool runtimeLookup; + + // Instantiated generic method + if (((SIZE_T)exactContextHnd & CORINFO_CONTEXTFLAGS_MASK) == CORINFO_CONTEXTFLAGS_METHOD) + { + assert(exactContextHnd != METHOD_BEING_COMPILED_CONTEXT()); + + CORINFO_METHOD_HANDLE exactMethodHandle = + (CORINFO_METHOD_HANDLE)((SIZE_T)exactContextHnd & ~CORINFO_CONTEXTFLAGS_MASK); + + if (!exactContextNeedsRuntimeLookup) + { +#ifdef FEATURE_READYTORUN + if (opts.IsReadyToRun()) + { + instParam = + impReadyToRunLookupToTree(&callInfo->instParamLookup, GTF_ICON_METHOD_HDL, exactMethodHandle); + if (instParam == nullptr) + { + assert(compDonotInline()); + return TYP_UNDEF; + } + } + else +#endif + { + instParam = gtNewIconEmbMethHndNode(exactMethodHandle); + info.compCompHnd->methodMustBeLoadedBeforeCodeIsRun(exactMethodHandle); + } + } + else + { + instParam = impTokenToHandle(pResolvedToken, &runtimeLookup, true /*mustRestoreHandle*/); + if (instParam == nullptr) + { + assert(compDonotInline()); + return TYP_UNDEF; + } + } + } + + // otherwise must be an instance method in a generic struct, + // a static method in a generic type, or a runtime-generated array method + else + { + assert(((SIZE_T)exactContextHnd & CORINFO_CONTEXTFLAGS_MASK) == CORINFO_CONTEXTFLAGS_CLASS); + CORINFO_CLASS_HANDLE exactClassHandle = eeGetClassFromContext(exactContextHnd); + + if (compIsForInlining() && (clsFlags & CORINFO_FLG_ARRAY) != 0) + { + compInlineResult->NoteFatal(InlineObservation::CALLEE_IS_ARRAY_METHOD); + return TYP_UNDEF; + } + + if ((clsFlags & CORINFO_FLG_ARRAY) && isReadonlyCall) + { + // We indicate "readonly" to the Address operation by using a null + // instParam. + instParam = gtNewIconNode(0, TYP_REF); + } + else if (!exactContextNeedsRuntimeLookup) + { +#ifdef FEATURE_READYTORUN + if (opts.IsReadyToRun()) + { + instParam = + impReadyToRunLookupToTree(&callInfo->instParamLookup, GTF_ICON_CLASS_HDL, exactClassHandle); + if (instParam == nullptr) + { + assert(compDonotInline()); + return TYP_UNDEF; + } + } + else +#endif + { + instParam = gtNewIconEmbClsHndNode(exactClassHandle); + info.compCompHnd->classMustBeLoadedBeforeCodeIsRun(exactClassHandle); + } + } + else + { + instParam = impParentClassTokenToHandle(pResolvedToken, &runtimeLookup, true /*mustRestoreHandle*/); + if (instParam == nullptr) + { + assert(compDonotInline()); + return TYP_UNDEF; + } + } + } + + assert(extraArg.Node == nullptr); + extraArg = NewCallArg::Primitive(instParam).WellKnown(WellKnownArg::InstParam); + } + + if ((opcode == CEE_NEWOBJ) && ((clsFlags & CORINFO_FLG_DELEGATE) != 0)) + { + // Only verifiable cases are supported. + // dup; ldvirtftn; newobj; or ldftn; newobj. + // IL test could contain unverifiable sequence, in this case optimization should not be done. + if (impStackHeight() > 0) + { + typeInfo delegateTypeInfo = impStackTop().seTypeInfo; + if (delegateTypeInfo.IsMethod()) + { + ldftnInfo = delegateTypeInfo.GetMethodPointerInfo(); + } + } + } + + //------------------------------------------------------------------------- + // The main group of arguments + + impPopCallArgs(sig, call->AsCall()); + if (extraArg.Node != nullptr) + { + if (Target::g_tgtArgOrder == Target::ARG_ORDER_R2L) + { + call->AsCall()->gtArgs.PushFront(this, extraArg); + } + else + { + call->AsCall()->gtArgs.PushBack(this, extraArg); + } + + call->gtFlags |= extraArg.Node->gtFlags & GTF_GLOB_EFFECT; + } + + //------------------------------------------------------------------------- + // The "this" pointer + + if (((mflags & CORINFO_FLG_STATIC) == 0) && ((sig->callConv & CORINFO_CALLCONV_EXPLICITTHIS) == 0) && + !((opcode == CEE_NEWOBJ) && (newobjThis == nullptr))) + { + GenTree* obj; + + if (opcode == CEE_NEWOBJ) + { + obj = newobjThis; + } + else + { + obj = impPopStack().val; + obj = impTransformThis(obj, pConstrainedResolvedToken, constraintCallThisTransform); + if (compDonotInline()) + { + return TYP_UNDEF; + } + } + + // Store the "this" value in the call + call->gtFlags |= obj->gtFlags & GTF_GLOB_EFFECT; + call->AsCall()->gtArgs.PushFront(this, NewCallArg::Primitive(obj).WellKnown(WellKnownArg::ThisPointer)); + + if (impIsThis(obj)) + { + call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_NONVIRT_SAME_THIS; + } + } + + bool probing; + probing = impConsiderCallProbe(call->AsCall(), rawILOffset); + + // See if we can devirt if we aren't probing. + if (!probing && opts.OptimizationEnabled()) + { + if (call->AsCall()->IsVirtual()) + { + // only true object pointers can be virtual + assert(call->AsCall()->gtArgs.HasThisPointer() && + call->AsCall()->gtArgs.GetThisArg()->GetNode()->TypeIs(TYP_REF)); + + // See if we can devirtualize. + + const bool isExplicitTailCall = (tailCallFlags & PREFIX_TAILCALL_EXPLICIT) != 0; + const bool isLateDevirtualization = false; + impDevirtualizeCall(call->AsCall(), pResolvedToken, &callInfo->hMethod, &callInfo->methodFlags, + &callInfo->contextHandle, &exactContextHnd, isLateDevirtualization, isExplicitTailCall, + // Take care to pass raw IL offset here as the 'debug info' might be different for + // inlinees. + rawILOffset); + + // Devirtualization may change which method gets invoked. Update our local cache. + // + methHnd = callInfo->hMethod; + } + else if (call->AsCall()->IsDelegateInvoke()) + { + considerGuardedDevirtualization(call->AsCall(), rawILOffset, false, NO_METHOD_HANDLE, NO_CLASS_HANDLE, + nullptr); + } + } + + //------------------------------------------------------------------------- + // The "this" pointer for "newobj" + + if (opcode == CEE_NEWOBJ) + { + if (clsFlags & CORINFO_FLG_VAROBJSIZE) + { + assert(!(clsFlags & CORINFO_FLG_ARRAY)); // arrays handled separately + // This is a 'new' of a variable sized object, wher + // the constructor is to return the object. In this case + // the constructor claims to return VOID but we know it + // actually returns the new object + assert(callRetTyp == TYP_VOID); + callRetTyp = TYP_REF; + call->gtType = TYP_REF; + impSpillSpecialSideEff(); + + impPushOnStack(call, typeInfo(TI_REF, clsHnd)); + } + else + { + if (clsFlags & CORINFO_FLG_DELEGATE) + { + // New inliner morph it in impImportCall. + // This will allow us to inline the call to the delegate constructor. + call = fgOptimizeDelegateConstructor(call->AsCall(), &exactContextHnd, ldftnInfo); + } + + if (!bIntrinsicImported) + { + +#if defined(DEBUG) || defined(INLINE_DATA) + + // Keep track of the raw IL offset of the call + call->AsCall()->gtRawILOffset = rawILOffset; + +#endif // defined(DEBUG) || defined(INLINE_DATA) + + // Is it an inline candidate? + impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, rawILOffset); + } + + // append the call node. + impAppendTree(call, (unsigned)CHECK_SPILL_ALL, impCurStmtDI); + + // Now push the value of the 'new onto the stack + + // This is a 'new' of a non-variable sized object. + // Append the new node (op1) to the statement list, + // and then push the local holding the value of this + // new instruction on the stack. + + if (clsFlags & CORINFO_FLG_VALUECLASS) + { + assert(newobjThis->gtOper == GT_ADDR && newobjThis->AsOp()->gtOp1->gtOper == GT_LCL_VAR); + + unsigned tmp = newobjThis->AsOp()->gtOp1->AsLclVarCommon()->GetLclNum(); + impPushOnStack(gtNewLclvNode(tmp, lvaGetRealType(tmp)), verMakeTypeInfo(clsHnd).NormaliseForStack()); + } + else + { + if (newobjThis->gtOper == GT_COMMA) + { + // We must have inserted the callout. Get the real newobj. + newobjThis = newobjThis->AsOp()->gtOp2; + } + + assert(newobjThis->gtOper == GT_LCL_VAR); + impPushOnStack(gtNewLclvNode(newobjThis->AsLclVarCommon()->GetLclNum(), TYP_REF), + typeInfo(TI_REF, clsHnd)); + } + } + return callRetTyp; + } + +DONE: + +#ifdef DEBUG + // In debug we want to be able to register callsites with the EE. + assert(call->AsCall()->callSig == nullptr); + call->AsCall()->callSig = new (this, CMK_Generic) CORINFO_SIG_INFO; + *call->AsCall()->callSig = *sig; +#endif + + // Final importer checks for calls flagged as tail calls. + // + if (tailCallFlags != 0) + { + const bool isExplicitTailCall = (tailCallFlags & PREFIX_TAILCALL_EXPLICIT) != 0; + const bool isImplicitTailCall = (tailCallFlags & PREFIX_TAILCALL_IMPLICIT) != 0; + const bool isStressTailCall = (tailCallFlags & PREFIX_TAILCALL_STRESS) != 0; + + // Exactly one of these should be true. + assert(isExplicitTailCall != isImplicitTailCall); + + // This check cannot be performed for implicit tail calls for the reason + // that impIsImplicitTailCallCandidate() is not checking whether return + // types are compatible before marking a call node with PREFIX_TAILCALL_IMPLICIT. + // As a result it is possible that in the following case, we find that + // the type stack is non-empty if Callee() is considered for implicit + // tail calling. + // int Caller(..) { .... void Callee(); ret val; ... } + // + // Note that we cannot check return type compatibility before ImpImportCall() + // as we don't have required info or need to duplicate some of the logic of + // ImpImportCall(). + // + // For implicit tail calls, we perform this check after return types are + // known to be compatible. + if (isExplicitTailCall && (verCurrentState.esStackDepth != 0)) + { + BADCODE("Stack should be empty after tailcall"); + } + + // For opportunistic tailcalls we allow implicit widening, i.e. tailcalls from int32 -> int16, since the + // managed calling convention dictates that the callee widens the value. For explicit tailcalls we don't + // want to require this detail of the calling convention to bubble up to the tailcall helpers + bool allowWidening = isImplicitTailCall; + if (canTailCall && + !impTailCallRetTypeCompatible(allowWidening, info.compRetType, info.compMethodInfo->args.retTypeClass, + info.compCallConv, callRetTyp, sig->retTypeClass, + call->AsCall()->GetUnmanagedCallConv())) + { + canTailCall = false; + szCanTailCallFailReason = "Return types are not tail call compatible"; + } + + // Stack empty check for implicit tail calls. + if (canTailCall && isImplicitTailCall && (verCurrentState.esStackDepth != 0)) + { +#ifdef TARGET_AMD64 + // JIT64 Compatibility: Opportunistic tail call stack mismatch throws a VerificationException + // in JIT64, not an InvalidProgramException. + Verify(false, "Stack should be empty after tailcall"); +#else // TARGET_64BIT + BADCODE("Stack should be empty after tailcall"); +#endif //! TARGET_64BIT + } + + // assert(compCurBB is not a catch, finally or filter block); + // assert(compCurBB is not a try block protected by a finally block); + assert(!isExplicitTailCall || compCurBB->bbJumpKind == BBJ_RETURN); + + // Ask VM for permission to tailcall + if (canTailCall) + { + // True virtual or indirect calls, shouldn't pass in a callee handle. + CORINFO_METHOD_HANDLE exactCalleeHnd = + ((call->AsCall()->gtCallType != CT_USER_FUNC) || call->AsCall()->IsVirtual()) ? nullptr : methHnd; + + if (info.compCompHnd->canTailCall(info.compMethodHnd, methHnd, exactCalleeHnd, isExplicitTailCall)) + { + if (isExplicitTailCall) + { + // In case of explicit tail calls, mark it so that it is not considered + // for in-lining. + call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_EXPLICIT_TAILCALL; + JITDUMP("\nGTF_CALL_M_EXPLICIT_TAILCALL set for call [%06u]\n", dspTreeID(call)); + + if (isStressTailCall) + { + call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_STRESS_TAILCALL; + JITDUMP("\nGTF_CALL_M_STRESS_TAILCALL set for call [%06u]\n", dspTreeID(call)); + } + } + else + { +#if FEATURE_TAILCALL_OPT + // Must be an implicit tail call. + assert(isImplicitTailCall); + + // It is possible that a call node is both an inline candidate and marked + // for opportunistic tail calling. In-lining happens before morhphing of + // trees. If in-lining of an in-line candidate gets aborted for whatever + // reason, it will survive to the morphing stage at which point it will be + // transformed into a tail call after performing additional checks. + + call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_IMPLICIT_TAILCALL; + JITDUMP("\nGTF_CALL_M_IMPLICIT_TAILCALL set for call [%06u]\n", dspTreeID(call)); + +#else //! FEATURE_TAILCALL_OPT + NYI("Implicit tail call prefix on a target which doesn't support opportunistic tail calls"); + +#endif // FEATURE_TAILCALL_OPT + } + + // This might or might not turn into a tailcall. We do more + // checks in morph. For explicit tailcalls we need more + // information in morph in case it turns out to be a + // helper-based tailcall. + if (isExplicitTailCall) + { + assert(call->AsCall()->tailCallInfo == nullptr); + call->AsCall()->tailCallInfo = new (this, CMK_CorTailCallInfo) TailCallSiteInfo; + switch (opcode) + { + case CEE_CALLI: + call->AsCall()->tailCallInfo->SetCalli(sig); + break; + case CEE_CALLVIRT: + call->AsCall()->tailCallInfo->SetCallvirt(sig, pResolvedToken); + break; + default: + call->AsCall()->tailCallInfo->SetCall(sig, pResolvedToken); + break; + } + } + } + else + { + // canTailCall reported its reasons already + canTailCall = false; + JITDUMP("\ninfo.compCompHnd->canTailCall returned false for call [%06u]\n", dspTreeID(call)); + } + } + else + { + // If this assert fires it means that canTailCall was set to false without setting a reason! + assert(szCanTailCallFailReason != nullptr); + JITDUMP("\nRejecting %splicit tail call for [%06u], reason: '%s'\n", isExplicitTailCall ? "ex" : "im", + dspTreeID(call), szCanTailCallFailReason); + info.compCompHnd->reportTailCallDecision(info.compMethodHnd, methHnd, isExplicitTailCall, TAILCALL_FAIL, + szCanTailCallFailReason); + } + } + + // Note: we assume that small return types are already normalized by the managed callee + // or by the pinvoke stub for calls to unmanaged code. + + if (!bIntrinsicImported) + { + // + // Things needed to be checked when bIntrinsicImported is false. + // + + assert(call->gtOper == GT_CALL); + assert(callInfo != nullptr); + + if (compIsForInlining() && opcode == CEE_CALLVIRT) + { + assert(call->AsCall()->gtArgs.HasThisPointer()); + GenTree* callObj = call->AsCall()->gtArgs.GetThisArg()->GetEarlyNode(); + + if ((call->AsCall()->IsVirtual() || (call->gtFlags & GTF_CALL_NULLCHECK)) && + impInlineIsGuaranteedThisDerefBeforeAnySideEffects(nullptr, &call->AsCall()->gtArgs, callObj, + impInlineInfo->inlArgInfo)) + { + impInlineInfo->thisDereferencedFirst = true; + } + } + +#if defined(DEBUG) || defined(INLINE_DATA) + + // Keep track of the raw IL offset of the call + call->AsCall()->gtRawILOffset = rawILOffset; + +#endif // defined(DEBUG) || defined(INLINE_DATA) + + // Is it an inline candidate? + impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, rawILOffset); + } + + // Extra checks for tail calls and tail recursion. + // + // A tail recursive call is a potential loop from the current block to the start of the root method. + // If we see a tail recursive call, mark the blocks from the call site back to the entry as potentially + // being in a loop. + // + // Note: if we're importing an inlinee we don't mark the right set of blocks, but by then it's too + // late. Currently this doesn't lead to problems. See GitHub issue 33529. + // + // OSR also needs to handle tail calls specially: + // * block profiling in OSR methods needs to ensure probes happen before tail calls, not after. + // * the root method entry must be imported if there's a recursive tail call or a potentially + // inlineable tail call. + // + if ((tailCallFlags != 0) && canTailCall) + { + if (gtIsRecursiveCall(methHnd)) + { + assert(verCurrentState.esStackDepth == 0); + BasicBlock* loopHead = nullptr; + if (!compIsForInlining() && opts.IsOSR()) + { + // For root method OSR we may branch back to the actual method entry, + // which is not fgFirstBB, and which we will need to import. + assert(fgEntryBB != nullptr); + loopHead = fgEntryBB; + } + else + { + // For normal jitting we may branch back to the firstBB; this + // should already be imported. + loopHead = fgFirstBB; + } + + JITDUMP("\nTail recursive call [%06u] in the method. Mark " FMT_BB " to " FMT_BB + " as having a backward branch.\n", + dspTreeID(call), loopHead->bbNum, compCurBB->bbNum); + fgMarkBackwardJump(loopHead, compCurBB); + } + + // We only do these OSR checks in the root method because: + // * If we fail to import the root method entry when importing the root method, we can't go back + // and import it during inlining. So instead of checking jsut for recursive tail calls we also + // have to check for anything that might introduce a recursive tail call. + // * We only instrument root method blocks in OSR methods, + // + if (opts.IsOSR() && !compIsForInlining()) + { + // If a root method tail call candidate block is not a BBJ_RETURN, it should have a unique + // BBJ_RETURN successor. Mark that successor so we can handle it specially during profile + // instrumentation. + // + if (compCurBB->bbJumpKind != BBJ_RETURN) + { + BasicBlock* const successor = compCurBB->GetUniqueSucc(); + assert(successor->bbJumpKind == BBJ_RETURN); + successor->bbFlags |= BBF_TAILCALL_SUCCESSOR; + optMethodFlags |= OMF_HAS_TAILCALL_SUCCESSOR; + } + + // If this call might eventually turn into a loop back to method entry, make sure we + // import the method entry. + // + assert(call->IsCall()); + GenTreeCall* const actualCall = call->AsCall(); + const bool mustImportEntryBlock = gtIsRecursiveCall(methHnd) || actualCall->IsInlineCandidate() || + actualCall->IsGuardedDevirtualizationCandidate(); + + // Only schedule importation if we're not currently importing. + // + if (mustImportEntryBlock && (compCurBB != fgEntryBB)) + { + JITDUMP("\nOSR: inlineable or recursive tail call [%06u] in the method, so scheduling " FMT_BB + " for importation\n", + dspTreeID(call), fgEntryBB->bbNum); + impImportBlockPending(fgEntryBB); + } + } + } + + if ((sig->flags & CORINFO_SIGFLAG_FAT_CALL) != 0) + { + assert(opcode == CEE_CALLI || callInfo->kind == CORINFO_CALL_CODE_POINTER); + addFatPointerCandidate(call->AsCall()); + } + +DONE_CALL: + // Push or append the result of the call + if (callRetTyp == TYP_VOID) + { + if (opcode == CEE_NEWOBJ) + { + // we actually did push something, so don't spill the thing we just pushed. + assert(verCurrentState.esStackDepth > 0); + impAppendTree(call, verCurrentState.esStackDepth - 1, impCurStmtDI); + } + else + { + impAppendTree(call, (unsigned)CHECK_SPILL_ALL, impCurStmtDI); + } + } + else + { + impSpillSpecialSideEff(); + + if (clsFlags & CORINFO_FLG_ARRAY) + { + eeGetCallSiteSig(pResolvedToken->token, pResolvedToken->tokenScope, pResolvedToken->tokenContext, sig); + } + + typeInfo tiRetVal = verMakeTypeInfo(sig->retType, sig->retTypeClass); + tiRetVal.NormaliseForStack(); + + // The CEE_READONLY prefix modifies the verification semantics of an Address + // operation on an array type. + if ((clsFlags & CORINFO_FLG_ARRAY) && isReadonlyCall && tiRetVal.IsByRef()) + { + tiRetVal.SetIsReadonlyByRef(); + } + + if (call->IsCall()) + { + // Sometimes "call" is not a GT_CALL (if we imported an intrinsic that didn't turn into a call) + + GenTreeCall* origCall = call->AsCall(); + + const bool isFatPointerCandidate = origCall->IsFatPointerCandidate(); + const bool isInlineCandidate = origCall->IsInlineCandidate(); + const bool isGuardedDevirtualizationCandidate = origCall->IsGuardedDevirtualizationCandidate(); + + if (varTypeIsStruct(callRetTyp)) + { + // Need to treat all "split tree" cases here, not just inline candidates + call = impFixupCallStructReturn(call->AsCall(), sig->retTypeClass); + } + + // TODO: consider handling fatcalli cases this way too...? + if (isInlineCandidate || isGuardedDevirtualizationCandidate) + { + // We should not have made any adjustments in impFixupCallStructReturn + // as we defer those until we know the fate of the call. + assert(call == origCall); + + assert(opts.OptEnabled(CLFLG_INLINING)); + assert(!isFatPointerCandidate); // We should not try to inline calli. + + // Make the call its own tree (spill the stack if needed). + // Do not consume the debug info here. This is particularly + // important if we give up on the inline, in which case the + // call will typically end up in the statement that contains + // the GT_RET_EXPR that we leave on the stack. + impAppendTree(call, (unsigned)CHECK_SPILL_ALL, impCurStmtDI, false); + + // TODO: Still using the widened type. + GenTree* retExpr = gtNewInlineCandidateReturnExpr(call, genActualType(callRetTyp), compCurBB->bbFlags); + + // Link the retExpr to the call so if necessary we can manipulate it later. + origCall->gtInlineCandidateInfo->retExpr = retExpr; + + // Propagate retExpr as the placeholder for the call. + call = retExpr; + } + else + { + // If the call is virtual, and has a generics context, and is not going to have a class probe, + // record the context for possible use during late devirt. + // + // If we ever want to devirt at Tier0, and/or see issues where OSR methods under PGO lose + // important devirtualizations, we'll want to allow both a class probe and a captured context. + // + if (origCall->IsVirtual() && (origCall->gtCallType != CT_INDIRECT) && (exactContextHnd != nullptr) && + (origCall->gtHandleHistogramProfileCandidateInfo == nullptr)) + { + JITDUMP("\nSaving context %p for call [%06u]\n", exactContextHnd, dspTreeID(origCall)); + origCall->gtCallMoreFlags |= GTF_CALL_M_HAS_LATE_DEVIRT_INFO; + LateDevirtualizationInfo* const info = new (this, CMK_Inlining) LateDevirtualizationInfo; + info->exactContextHnd = exactContextHnd; + origCall->gtLateDevirtualizationInfo = info; + } + + if (isFatPointerCandidate) + { + // fatPointer candidates should be in statements of the form call() or var = call(). + // Such form allows to find statements with fat calls without walking through whole trees + // and removes problems with cutting trees. + assert(!bIntrinsicImported); + assert(IsTargetAbi(CORINFO_NATIVEAOT_ABI)); + if (call->OperGet() != GT_LCL_VAR) // can be already converted by impFixupCallStructReturn. + { + unsigned calliSlot = lvaGrabTemp(true DEBUGARG("calli")); + LclVarDsc* varDsc = lvaGetDesc(calliSlot); + varDsc->lvVerTypeInfo = tiRetVal; + impAssignTempGen(calliSlot, call, tiRetVal.GetClassHandle(), (unsigned)CHECK_SPILL_NONE); + // impAssignTempGen can change src arg list and return type for call that returns struct. + var_types type = genActualType(lvaTable[calliSlot].TypeGet()); + call = gtNewLclvNode(calliSlot, type); + } + } + + // For non-candidates we must also spill, since we + // might have locals live on the eval stack that this + // call can modify. + // + // Suppress this for certain well-known call targets + // that we know won't modify locals, eg calls that are + // recognized in gtCanOptimizeTypeEquality. Otherwise + // we may break key fragile pattern matches later on. + bool spillStack = true; + if (call->IsCall()) + { + GenTreeCall* callNode = call->AsCall(); + if ((callNode->gtCallType == CT_HELPER) && (gtIsTypeHandleToRuntimeTypeHelper(callNode) || + gtIsTypeHandleToRuntimeTypeHandleHelper(callNode))) + { + spillStack = false; + } + else if ((callNode->gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC) != 0) + { + spillStack = false; + } + } + + if (spillStack) + { + impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("non-inline candidate call")); + } + } + } + + if (!bIntrinsicImported) + { + //------------------------------------------------------------------------- + // + /* If the call is of a small type and the callee is managed, the callee will normalize the result + before returning. + However, we need to normalize small type values returned by unmanaged + functions (pinvoke). The pinvoke stub does the normalization, but we need to do it here + if we use the shorter inlined pinvoke stub. */ + + if (checkForSmallType && varTypeIsIntegral(callRetTyp) && genTypeSize(callRetTyp) < genTypeSize(TYP_INT)) + { + call = gtNewCastNode(genActualType(callRetTyp), call, false, callRetTyp); + } + } + + impPushOnStack(call, tiRetVal); + } + + // VSD functions get a new call target each time we getCallInfo, so clear the cache. + // Also, the call info cache for CALLI instructions is largely incomplete, so clear it out. + // if ( (opcode == CEE_CALLI) || (callInfoCache.fetchCallInfo().kind == CORINFO_VIRTUALCALL_STUB)) + // callInfoCache.uncacheCallInfo(); + + return callRetTyp; +} +#ifdef _PREFAST_ +#pragma warning(pop) +#endif + +bool Compiler::impMethodInfo_hasRetBuffArg(CORINFO_METHOD_INFO* methInfo, CorInfoCallConvExtension callConv) +{ + CorInfoType corType = methInfo->args.retType; + + if ((corType == CORINFO_TYPE_VALUECLASS) || (corType == CORINFO_TYPE_REFANY)) + { + // We have some kind of STRUCT being returned + structPassingKind howToReturnStruct = SPK_Unknown; + + var_types returnType = getReturnTypeForStruct(methInfo->args.retTypeClass, callConv, &howToReturnStruct); + + if (howToReturnStruct == SPK_ByReference) + { + return true; + } + } + + return false; +} + +#ifdef DEBUG +// +var_types Compiler::impImportJitTestLabelMark(int numArgs) +{ + TestLabelAndNum tlAndN; + if (numArgs == 2) + { + tlAndN.m_num = 0; + StackEntry se = impPopStack(); + assert(se.seTypeInfo.GetType() == TI_INT); + GenTree* val = se.val; + assert(val->IsCnsIntOrI()); + tlAndN.m_tl = (TestLabel)val->AsIntConCommon()->IconValue(); + } + else if (numArgs == 3) + { + StackEntry se = impPopStack(); + assert(se.seTypeInfo.GetType() == TI_INT); + GenTree* val = se.val; + assert(val->IsCnsIntOrI()); + tlAndN.m_num = val->AsIntConCommon()->IconValue(); + se = impPopStack(); + assert(se.seTypeInfo.GetType() == TI_INT); + val = se.val; + assert(val->IsCnsIntOrI()); + tlAndN.m_tl = (TestLabel)val->AsIntConCommon()->IconValue(); + } + else + { + assert(false); + } + + StackEntry expSe = impPopStack(); + GenTree* node = expSe.val; + + // There are a small number of special cases, where we actually put the annotation on a subnode. + if (tlAndN.m_tl == TL_LoopHoist && tlAndN.m_num >= 100) + { + // A loop hoist annotation with value >= 100 means that the expression should be a static field access, + // a GT_IND of a static field address, which should be the sum of a (hoistable) helper call and possibly some + // offset within the static field block whose address is returned by the helper call. + // The annotation is saying that this address calculation, but not the entire access, should be hoisted. + assert(node->OperGet() == GT_IND); + tlAndN.m_num -= 100; + GetNodeTestData()->Set(node->AsOp()->gtOp1, tlAndN); + GetNodeTestData()->Remove(node); + } + else + { + GetNodeTestData()->Set(node, tlAndN); + } + + impPushOnStack(node, expSe.seTypeInfo); + return node->TypeGet(); +} +#endif // DEBUG + +//----------------------------------------------------------------------------------- +// impFixupCallStructReturn: For a call node that returns a struct do one of the following: +// - set the flag to indicate struct return via retbuf arg; +// - adjust the return type to a SIMD type if it is returned in 1 reg; +// - spill call result into a temp if it is returned into 2 registers or more and not tail call or inline candidate. +// +// Arguments: +// call - GT_CALL GenTree node +// retClsHnd - Class handle of return type of the call +// +// Return Value: +// Returns new GenTree node after fixing struct return of call node +// +GenTree* Compiler::impFixupCallStructReturn(GenTreeCall* call, CORINFO_CLASS_HANDLE retClsHnd) +{ + if (!varTypeIsStruct(call)) + { + return call; + } + + call->gtRetClsHnd = retClsHnd; + +#if FEATURE_MULTIREG_RET + call->InitializeStructReturnType(this, retClsHnd, call->GetUnmanagedCallConv()); + const ReturnTypeDesc* retTypeDesc = call->GetReturnTypeDesc(); + const unsigned retRegCount = retTypeDesc->GetReturnRegCount(); +#else // !FEATURE_MULTIREG_RET + const unsigned retRegCount = 1; +#endif // !FEATURE_MULTIREG_RET + + structPassingKind howToReturnStruct; + var_types returnType = getReturnTypeForStruct(retClsHnd, call->GetUnmanagedCallConv(), &howToReturnStruct); + + if (howToReturnStruct == SPK_ByReference) + { + assert(returnType == TYP_UNKNOWN); + call->gtCallMoreFlags |= GTF_CALL_M_RETBUFFARG; + return call; + } + + // Recognize SIMD types as we do for LCL_VARs, + // note it could be not the ABI specific type, for example, on x64 we can set 'TYP_SIMD8` + // for `System.Numerics.Vector2` here but lower will change it to long as ABI dictates. + var_types simdReturnType = impNormStructType(call->gtRetClsHnd); + if (simdReturnType != call->TypeGet()) + { + assert(varTypeIsSIMD(simdReturnType)); + JITDUMP("changing the type of a call [%06u] from %s to %s\n", dspTreeID(call), varTypeName(call->TypeGet()), + varTypeName(simdReturnType)); + call->ChangeType(simdReturnType); + } + + if (retRegCount == 1) + { + return call; + } + +#if FEATURE_MULTIREG_RET + assert(varTypeIsStruct(call)); // It could be a SIMD returned in several regs. + assert(returnType == TYP_STRUCT); + assert((howToReturnStruct == SPK_ByValueAsHfa) || (howToReturnStruct == SPK_ByValue)); + +#ifdef UNIX_AMD64_ABI + // must be a struct returned in two registers + assert(retRegCount == 2); +#else // not UNIX_AMD64_ABI + assert(retRegCount >= 2); +#endif // not UNIX_AMD64_ABI + + if (!call->CanTailCall() && !call->IsInlineCandidate()) + { + // Force a call returning multi-reg struct to be always of the IR form + // tmp = call + // + // No need to assign a multi-reg struct to a local var if: + // - It is a tail call or + // - The call is marked for in-lining later + return impAssignMultiRegTypeToVar(call, retClsHnd DEBUGARG(call->GetUnmanagedCallConv())); + } + return call; +#endif // FEATURE_MULTIREG_RET +} + +/***************************************************************************** + For struct return values, re-type the operand in the case where the ABI + does not use a struct return buffer + */ + +//------------------------------------------------------------------------ +// impFixupStructReturnType: For struct return values it sets appropriate flags in MULTIREG returns case; +// in non-multiref case it handles two special helpers: `CORINFO_HELP_GETFIELDSTRUCT`, `CORINFO_HELP_UNBOX_NULLABLE`. +// +// Arguments: +// op - the return value; +// retClsHnd - the struct handle; +// unmgdCallConv - the calling convention of the function that returns this struct. +// +// Return Value: +// the result tree that does the return. +// +GenTree* Compiler::impFixupStructReturnType(GenTree* op, + CORINFO_CLASS_HANDLE retClsHnd, + CorInfoCallConvExtension unmgdCallConv) +{ + assert(varTypeIsStruct(info.compRetType)); + assert(info.compRetBuffArg == BAD_VAR_NUM); + + JITDUMP("\nimpFixupStructReturnType: retyping\n"); + DISPTREE(op); + +#if defined(TARGET_XARCH) + +#if FEATURE_MULTIREG_RET + // No VarArgs for CoreCLR on x64 Unix + UNIX_AMD64_ABI_ONLY(assert(!info.compIsVarArgs)); + + // Is method returning a multi-reg struct? + if (varTypeIsStruct(info.compRetNativeType) && IsMultiRegReturnedType(retClsHnd, unmgdCallConv)) + { + // In case of multi-reg struct return, we force IR to be one of the following: + // GT_RETURN(lclvar) or GT_RETURN(call). If op is anything other than a + // lclvar or call, it is assigned to a temp to create: temp = op and GT_RETURN(tmp). + + if (op->gtOper == GT_LCL_VAR) + { + // Note that this is a multi-reg return. + unsigned lclNum = op->AsLclVarCommon()->GetLclNum(); + lvaTable[lclNum].lvIsMultiRegRet = true; + + // TODO-1stClassStructs: Handle constant propagation and CSE-ing of multireg returns. + op->gtFlags |= GTF_DONT_CSE; + + return op; + } + + if (op->IsCall() && (op->AsCall()->GetUnmanagedCallConv() == unmgdCallConv)) + { + return op; + } + + return impAssignMultiRegTypeToVar(op, retClsHnd DEBUGARG(unmgdCallConv)); + } +#else + assert(info.compRetNativeType != TYP_STRUCT); +#endif // defined(UNIX_AMD64_ABI) || defined(TARGET_X86) + +#elif FEATURE_MULTIREG_RET && defined(TARGET_ARM) + + if (varTypeIsStruct(info.compRetNativeType) && !info.compIsVarArgs && IsHfa(retClsHnd)) + { + if (op->gtOper == GT_LCL_VAR) + { + // This LCL_VAR is an HFA return value, it stays as a TYP_STRUCT + unsigned lclNum = op->AsLclVarCommon()->GetLclNum(); + // Make sure this struct type stays as struct so that we can return it as an HFA + lvaTable[lclNum].lvIsMultiRegRet = true; + + // TODO-1stClassStructs: Handle constant propagation and CSE-ing of multireg returns. + op->gtFlags |= GTF_DONT_CSE; + + return op; + } + + if (op->gtOper == GT_CALL) + { + if (op->AsCall()->IsVarargs()) + { + // We cannot tail call because control needs to return to fixup the calling + // convention for result return. + op->AsCall()->gtCallMoreFlags &= ~GTF_CALL_M_TAILCALL; + op->AsCall()->gtCallMoreFlags &= ~GTF_CALL_M_EXPLICIT_TAILCALL; + } + else + { + return op; + } + } + return impAssignMultiRegTypeToVar(op, retClsHnd DEBUGARG(unmgdCallConv)); + } + +#elif FEATURE_MULTIREG_RET && (defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64)) + + // Is method returning a multi-reg struct? + if (IsMultiRegReturnedType(retClsHnd, unmgdCallConv)) + { + if (op->gtOper == GT_LCL_VAR) + { + // This LCL_VAR stays as a TYP_STRUCT + unsigned lclNum = op->AsLclVarCommon()->GetLclNum(); + + if (!lvaIsImplicitByRefLocal(lclNum)) + { + // Make sure this struct type is not struct promoted + lvaTable[lclNum].lvIsMultiRegRet = true; + + // TODO-1stClassStructs: Handle constant propagation and CSE-ing of multireg returns. + op->gtFlags |= GTF_DONT_CSE; + + return op; + } + } + + if (op->gtOper == GT_CALL) + { + if (op->AsCall()->IsVarargs()) + { + // We cannot tail call because control needs to return to fixup the calling + // convention for result return. + op->AsCall()->gtCallMoreFlags &= ~GTF_CALL_M_TAILCALL; + op->AsCall()->gtCallMoreFlags &= ~GTF_CALL_M_EXPLICIT_TAILCALL; + } + else + { + return op; + } + } + return impAssignMultiRegTypeToVar(op, retClsHnd DEBUGARG(unmgdCallConv)); + } + +#endif // FEATURE_MULTIREG_RET && (TARGET_ARM64 || TARGET_LOONGARCH64) + + if (!op->IsCall() || !op->AsCall()->TreatAsShouldHaveRetBufArg(this)) + { + // Don't retype `struct` as a primitive type in `ret` instruction. + return op; + } + + // This must be one of those 'special' helpers that don't + // really have a return buffer, but instead use it as a way + // to keep the trees cleaner with fewer address-taken temps. + // + // Well now we have to materialize the return buffer as + // an address-taken temp. Then we can return the temp. + // + // NOTE: this code assumes that since the call directly + // feeds the return, then the call must be returning the + // same structure/class/type. + // + unsigned tmpNum = lvaGrabTemp(true DEBUGARG("pseudo return buffer")); + + // No need to spill anything as we're about to return. + impAssignTempGen(tmpNum, op, info.compMethodInfo->args.retTypeClass, (unsigned)CHECK_SPILL_NONE); + + op = gtNewLclvNode(tmpNum, info.compRetType); + JITDUMP("\nimpFixupStructReturnType: created a pseudo-return buffer for a special helper\n"); + DISPTREE(op); + return op; +} + +/***************************************************************************** + CEE_LEAVE may be jumping out of a protected block, viz, a catch or a + finally-protected try. We find the finally blocks protecting the current + offset (in order) by walking over the complete exception table and + finding enclosing clauses. This assumes that the table is sorted. + This will create a series of BBJ_CALLFINALLY -> BBJ_CALLFINALLY ... -> BBJ_ALWAYS. + + If we are leaving a catch handler, we need to attach the + CPX_ENDCATCHes to the correct BBJ_CALLFINALLY blocks. + + After this function, the BBJ_LEAVE block has been converted to a different type. + */ + +#if !defined(FEATURE_EH_FUNCLETS) + +void Compiler::impImportLeave(BasicBlock* block) +{ +#ifdef DEBUG + if (verbose) + { + printf("\nBefore import CEE_LEAVE:\n"); + fgDispBasicBlocks(); + fgDispHandlerTab(); + } +#endif // DEBUG + + bool invalidatePreds = false; // If we create new blocks, invalidate the predecessor lists (if created) + unsigned blkAddr = block->bbCodeOffs; + BasicBlock* leaveTarget = block->bbJumpDest; + unsigned jmpAddr = leaveTarget->bbCodeOffs; + + // LEAVE clears the stack, spill side effects, and set stack to 0 + + impSpillSideEffects(true, (unsigned)CHECK_SPILL_ALL DEBUGARG("impImportLeave")); + verCurrentState.esStackDepth = 0; + + assert(block->bbJumpKind == BBJ_LEAVE); + assert(fgBBs == (BasicBlock**)0xCDCD || fgLookupBB(jmpAddr) != NULL); // should be a BB boundary + + BasicBlock* step = DUMMY_INIT(NULL); + unsigned encFinallies = 0; // Number of enclosing finallies. + GenTree* endCatches = NULL; + Statement* endLFinStmt = NULL; // The statement tree to indicate the end of locally-invoked finally. + + unsigned XTnum; + EHblkDsc* HBtab; + + for (XTnum = 0, HBtab = compHndBBtab; XTnum < compHndBBtabCount; XTnum++, HBtab++) + { + // Grab the handler offsets + + IL_OFFSET tryBeg = HBtab->ebdTryBegOffs(); + IL_OFFSET tryEnd = HBtab->ebdTryEndOffs(); + IL_OFFSET hndBeg = HBtab->ebdHndBegOffs(); + IL_OFFSET hndEnd = HBtab->ebdHndEndOffs(); + + /* Is this a catch-handler we are CEE_LEAVEing out of? + * If so, we need to call CORINFO_HELP_ENDCATCH. + */ + + if (jitIsBetween(blkAddr, hndBeg, hndEnd) && !jitIsBetween(jmpAddr, hndBeg, hndEnd)) + { + // Can't CEE_LEAVE out of a finally/fault handler + if (HBtab->HasFinallyOrFaultHandler()) + BADCODE("leave out of fault/finally block"); + + // Create the call to CORINFO_HELP_ENDCATCH + GenTree* endCatch = gtNewHelperCallNode(CORINFO_HELP_ENDCATCH, TYP_VOID); + + // Make a list of all the currently pending endCatches + if (endCatches) + endCatches = gtNewOperNode(GT_COMMA, TYP_VOID, endCatches, endCatch); + else + endCatches = endCatch; + +#ifdef DEBUG + if (verbose) + { + printf("impImportLeave - " FMT_BB " jumping out of catch handler EH#%u, adding call to " + "CORINFO_HELP_ENDCATCH\n", + block->bbNum, XTnum); + } +#endif + } + else if (HBtab->HasFinallyHandler() && jitIsBetween(blkAddr, tryBeg, tryEnd) && + !jitIsBetween(jmpAddr, tryBeg, tryEnd)) + { + /* This is a finally-protected try we are jumping out of */ + + /* If there are any pending endCatches, and we have already + jumped out of a finally-protected try, then the endCatches + have to be put in a block in an outer try for async + exceptions to work correctly. + Else, just use append to the original block */ + + BasicBlock* callBlock; + + assert(!encFinallies == + !endLFinStmt); // if we have finallies, we better have an endLFin tree, and vice-versa + + if (encFinallies == 0) + { + assert(step == DUMMY_INIT(NULL)); + callBlock = block; + callBlock->bbJumpKind = BBJ_CALLFINALLY; // convert the BBJ_LEAVE to BBJ_CALLFINALLY + + if (endCatches) + impAppendTree(endCatches, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); + +#ifdef DEBUG + if (verbose) + { + printf("impImportLeave - jumping out of a finally-protected try, convert block to BBJ_CALLFINALLY " + "block %s\n", + callBlock->dspToString()); + } +#endif + } + else + { + assert(step != DUMMY_INIT(NULL)); + + /* Calling the finally block */ + callBlock = fgNewBBinRegion(BBJ_CALLFINALLY, XTnum + 1, 0, step); + assert(step->bbJumpKind == BBJ_ALWAYS); + step->bbJumpDest = callBlock; // the previous call to a finally returns to this call (to the next + // finally in the chain) + step->bbJumpDest->bbRefs++; + + /* The new block will inherit this block's weight */ + callBlock->inheritWeight(block); + +#ifdef DEBUG + if (verbose) + { + printf("impImportLeave - jumping out of a finally-protected try, new BBJ_CALLFINALLY block %s\n", + callBlock->dspToString()); + } +#endif + + Statement* lastStmt; + + if (endCatches) + { + lastStmt = gtNewStmt(endCatches); + endLFinStmt->SetNextStmt(lastStmt); + lastStmt->SetPrevStmt(endLFinStmt); + } + else + { + lastStmt = endLFinStmt; + } + + // note that this sets BBF_IMPORTED on the block + impEndTreeList(callBlock, endLFinStmt, lastStmt); + } + + step = fgNewBBafter(BBJ_ALWAYS, callBlock, true); + /* The new block will inherit this block's weight */ + step->inheritWeight(block); + step->bbFlags |= BBF_IMPORTED | BBF_KEEP_BBJ_ALWAYS; + +#ifdef DEBUG + if (verbose) + { + printf("impImportLeave - jumping out of a finally-protected try, created step (BBJ_ALWAYS) block %s\n", + step->dspToString()); + } +#endif + + unsigned finallyNesting = compHndBBtab[XTnum].ebdHandlerNestingLevel; + assert(finallyNesting <= compHndBBtabCount); + + callBlock->bbJumpDest = HBtab->ebdHndBeg; // This callBlock will call the "finally" handler. + GenTree* endLFin = new (this, GT_END_LFIN) GenTreeVal(GT_END_LFIN, TYP_VOID, finallyNesting); + endLFinStmt = gtNewStmt(endLFin); + endCatches = NULL; + + encFinallies++; + + invalidatePreds = true; + } + } + + /* Append any remaining endCatches, if any */ + + assert(!encFinallies == !endLFinStmt); + + if (encFinallies == 0) + { + assert(step == DUMMY_INIT(NULL)); + block->bbJumpKind = BBJ_ALWAYS; // convert the BBJ_LEAVE to a BBJ_ALWAYS + + if (endCatches) + impAppendTree(endCatches, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); + +#ifdef DEBUG + if (verbose) + { + printf("impImportLeave - no enclosing finally-protected try blocks; convert CEE_LEAVE block to BBJ_ALWAYS " + "block %s\n", + block->dspToString()); + } +#endif + } + else + { + // If leaveTarget is the start of another try block, we want to make sure that + // we do not insert finalStep into that try block. Hence, we find the enclosing + // try block. + unsigned tryIndex = bbFindInnermostCommonTryRegion(step, leaveTarget); + + // Insert a new BB either in the try region indicated by tryIndex or + // the handler region indicated by leaveTarget->bbHndIndex, + // depending on which is the inner region. + BasicBlock* finalStep = fgNewBBinRegion(BBJ_ALWAYS, tryIndex, leaveTarget->bbHndIndex, step); + finalStep->bbFlags |= BBF_KEEP_BBJ_ALWAYS; + step->bbJumpDest = finalStep; + + /* The new block will inherit this block's weight */ + finalStep->inheritWeight(block); + +#ifdef DEBUG + if (verbose) + { + printf("impImportLeave - finalStep block required (encFinallies(%d) > 0), new block %s\n", encFinallies, + finalStep->dspToString()); + } +#endif + + Statement* lastStmt; + + if (endCatches) + { + lastStmt = gtNewStmt(endCatches); + endLFinStmt->SetNextStmt(lastStmt); + lastStmt->SetPrevStmt(endLFinStmt); + } + else + { + lastStmt = endLFinStmt; + } + + impEndTreeList(finalStep, endLFinStmt, lastStmt); + + finalStep->bbJumpDest = leaveTarget; // this is the ultimate destination of the LEAVE + + // Queue up the jump target for importing + + impImportBlockPending(leaveTarget); + + invalidatePreds = true; + } + + if (invalidatePreds && fgComputePredsDone) + { + JITDUMP("\n**** impImportLeave - Removing preds after creating new blocks\n"); + fgRemovePreds(); + } + +#ifdef DEBUG + fgVerifyHandlerTab(); + + if (verbose) + { + printf("\nAfter import CEE_LEAVE:\n"); + fgDispBasicBlocks(); + fgDispHandlerTab(); + } +#endif // DEBUG +} + +#else // FEATURE_EH_FUNCLETS + +void Compiler::impImportLeave(BasicBlock* block) +{ +#ifdef DEBUG + if (verbose) + { + printf("\nBefore import CEE_LEAVE in " FMT_BB " (targeting " FMT_BB "):\n", block->bbNum, + block->bbJumpDest->bbNum); + fgDispBasicBlocks(); + fgDispHandlerTab(); + } +#endif // DEBUG + + bool invalidatePreds = false; // If we create new blocks, invalidate the predecessor lists (if created) + unsigned blkAddr = block->bbCodeOffs; + BasicBlock* leaveTarget = block->bbJumpDest; + unsigned jmpAddr = leaveTarget->bbCodeOffs; + + // LEAVE clears the stack, spill side effects, and set stack to 0 + + impSpillSideEffects(true, (unsigned)CHECK_SPILL_ALL DEBUGARG("impImportLeave")); + verCurrentState.esStackDepth = 0; + + assert(block->bbJumpKind == BBJ_LEAVE); + assert(fgBBs == (BasicBlock**)0xCDCD || fgLookupBB(jmpAddr) != nullptr); // should be a BB boundary + + BasicBlock* step = nullptr; + + enum StepType + { + // No step type; step == NULL. + ST_None, + + // Is the step block the BBJ_ALWAYS block of a BBJ_CALLFINALLY/BBJ_ALWAYS pair? + // That is, is step->bbJumpDest where a finally will return to? + ST_FinallyReturn, + + // The step block is a catch return. + ST_Catch, + + // The step block is in a "try", created as the target for a finally return or the target for a catch return. + ST_Try + }; + StepType stepType = ST_None; + + unsigned XTnum; + EHblkDsc* HBtab; + + for (XTnum = 0, HBtab = compHndBBtab; XTnum < compHndBBtabCount; XTnum++, HBtab++) + { + // Grab the handler offsets + + IL_OFFSET tryBeg = HBtab->ebdTryBegOffs(); + IL_OFFSET tryEnd = HBtab->ebdTryEndOffs(); + IL_OFFSET hndBeg = HBtab->ebdHndBegOffs(); + IL_OFFSET hndEnd = HBtab->ebdHndEndOffs(); + + /* Is this a catch-handler we are CEE_LEAVEing out of? + */ + + if (jitIsBetween(blkAddr, hndBeg, hndEnd) && !jitIsBetween(jmpAddr, hndBeg, hndEnd)) + { + // Can't CEE_LEAVE out of a finally/fault handler + if (HBtab->HasFinallyOrFaultHandler()) + { + BADCODE("leave out of fault/finally block"); + } + + /* We are jumping out of a catch */ + + if (step == nullptr) + { + step = block; + step->bbJumpKind = BBJ_EHCATCHRET; // convert the BBJ_LEAVE to BBJ_EHCATCHRET + stepType = ST_Catch; + +#ifdef DEBUG + if (verbose) + { + printf("impImportLeave - jumping out of a catch (EH#%u), convert block " FMT_BB + " to BBJ_EHCATCHRET " + "block\n", + XTnum, step->bbNum); + } +#endif + } + else + { + BasicBlock* exitBlock; + + /* Create a new catch exit block in the catch region for the existing step block to jump to in this + * scope */ + exitBlock = fgNewBBinRegion(BBJ_EHCATCHRET, 0, XTnum + 1, step); + + assert(step->KindIs(BBJ_ALWAYS, BBJ_EHCATCHRET)); + step->bbJumpDest = exitBlock; // the previous step (maybe a call to a nested finally, or a nested catch + // exit) returns to this block + step->bbJumpDest->bbRefs++; + +#if defined(TARGET_ARM) + if (stepType == ST_FinallyReturn) + { + assert(step->bbJumpKind == BBJ_ALWAYS); + // Mark the target of a finally return + step->bbJumpDest->bbFlags |= BBF_FINALLY_TARGET; + } +#endif // defined(TARGET_ARM) + + /* The new block will inherit this block's weight */ + exitBlock->inheritWeight(block); + exitBlock->bbFlags |= BBF_IMPORTED; + + /* This exit block is the new step */ + step = exitBlock; + stepType = ST_Catch; + + invalidatePreds = true; + +#ifdef DEBUG + if (verbose) + { + printf("impImportLeave - jumping out of a catch (EH#%u), new BBJ_EHCATCHRET block " FMT_BB "\n", + XTnum, exitBlock->bbNum); + } +#endif + } + } + else if (HBtab->HasFinallyHandler() && jitIsBetween(blkAddr, tryBeg, tryEnd) && + !jitIsBetween(jmpAddr, tryBeg, tryEnd)) + { + /* We are jumping out of a finally-protected try */ + + BasicBlock* callBlock; + + if (step == nullptr) + { +#if FEATURE_EH_CALLFINALLY_THUNKS + + // Put the call to the finally in the enclosing region. + unsigned callFinallyTryIndex = + (HBtab->ebdEnclosingTryIndex == EHblkDsc::NO_ENCLOSING_INDEX) ? 0 : HBtab->ebdEnclosingTryIndex + 1; + unsigned callFinallyHndIndex = + (HBtab->ebdEnclosingHndIndex == EHblkDsc::NO_ENCLOSING_INDEX) ? 0 : HBtab->ebdEnclosingHndIndex + 1; + callBlock = fgNewBBinRegion(BBJ_CALLFINALLY, callFinallyTryIndex, callFinallyHndIndex, block); + + // Convert the BBJ_LEAVE to BBJ_ALWAYS, jumping to the new BBJ_CALLFINALLY. This is because + // the new BBJ_CALLFINALLY is in a different EH region, thus it can't just replace the BBJ_LEAVE, + // which might be in the middle of the "try". In most cases, the BBJ_ALWAYS will jump to the + // next block, and flow optimizations will remove it. + block->bbJumpKind = BBJ_ALWAYS; + block->bbJumpDest = callBlock; + block->bbJumpDest->bbRefs++; + + /* The new block will inherit this block's weight */ + callBlock->inheritWeight(block); + callBlock->bbFlags |= BBF_IMPORTED; + +#ifdef DEBUG + if (verbose) + { + printf("impImportLeave - jumping out of a finally-protected try (EH#%u), convert block " FMT_BB + " to " + "BBJ_ALWAYS, add BBJ_CALLFINALLY block " FMT_BB "\n", + XTnum, block->bbNum, callBlock->bbNum); + } +#endif + +#else // !FEATURE_EH_CALLFINALLY_THUNKS + + callBlock = block; + callBlock->bbJumpKind = BBJ_CALLFINALLY; // convert the BBJ_LEAVE to BBJ_CALLFINALLY + +#ifdef DEBUG + if (verbose) + { + printf("impImportLeave - jumping out of a finally-protected try (EH#%u), convert block " FMT_BB + " to " + "BBJ_CALLFINALLY block\n", + XTnum, callBlock->bbNum); + } +#endif + +#endif // !FEATURE_EH_CALLFINALLY_THUNKS + } + else + { + // Calling the finally block. We already have a step block that is either the call-to-finally from a + // more nested try/finally (thus we are jumping out of multiple nested 'try' blocks, each protected by + // a 'finally'), or the step block is the return from a catch. + // + // Due to ThreadAbortException, we can't have the catch return target the call-to-finally block + // directly. Note that if a 'catch' ends without resetting the ThreadAbortException, the VM will + // automatically re-raise the exception, using the return address of the catch (that is, the target + // block of the BBJ_EHCATCHRET) as the re-raise address. If this address is in a finally, the VM will + // refuse to do the re-raise, and the ThreadAbortException will get eaten (and lost). On AMD64/ARM64, + // we put the call-to-finally thunk in a special "cloned finally" EH region that does look like a + // finally clause to the VM. Thus, on these platforms, we can't have BBJ_EHCATCHRET target a + // BBJ_CALLFINALLY directly. (Note that on ARM32, we don't mark the thunk specially -- it lives directly + // within the 'try' region protected by the finally, since we generate code in such a way that execution + // never returns to the call-to-finally call, and the finally-protected 'try' region doesn't appear on + // stack walks.) + + assert(step->KindIs(BBJ_ALWAYS, BBJ_EHCATCHRET)); + +#if FEATURE_EH_CALLFINALLY_THUNKS + if (step->bbJumpKind == BBJ_EHCATCHRET) + { + // Need to create another step block in the 'try' region that will actually branch to the + // call-to-finally thunk. + BasicBlock* step2 = fgNewBBinRegion(BBJ_ALWAYS, XTnum + 1, 0, step); + step->bbJumpDest = step2; + step->bbJumpDest->bbRefs++; + step2->inheritWeight(block); + step2->bbFlags |= (block->bbFlags & BBF_RUN_RARELY) | BBF_IMPORTED; + +#ifdef DEBUG + if (verbose) + { + printf("impImportLeave - jumping out of a finally-protected try (EH#%u), step block is " + "BBJ_EHCATCHRET (" FMT_BB "), new BBJ_ALWAYS step-step block " FMT_BB "\n", + XTnum, step->bbNum, step2->bbNum); + } +#endif + + step = step2; + assert(stepType == ST_Catch); // Leave it as catch type for now. + } +#endif // FEATURE_EH_CALLFINALLY_THUNKS + +#if FEATURE_EH_CALLFINALLY_THUNKS + unsigned callFinallyTryIndex = + (HBtab->ebdEnclosingTryIndex == EHblkDsc::NO_ENCLOSING_INDEX) ? 0 : HBtab->ebdEnclosingTryIndex + 1; + unsigned callFinallyHndIndex = + (HBtab->ebdEnclosingHndIndex == EHblkDsc::NO_ENCLOSING_INDEX) ? 0 : HBtab->ebdEnclosingHndIndex + 1; +#else // !FEATURE_EH_CALLFINALLY_THUNKS + unsigned callFinallyTryIndex = XTnum + 1; + unsigned callFinallyHndIndex = 0; // don't care +#endif // !FEATURE_EH_CALLFINALLY_THUNKS + + callBlock = fgNewBBinRegion(BBJ_CALLFINALLY, callFinallyTryIndex, callFinallyHndIndex, step); + step->bbJumpDest = callBlock; // the previous call to a finally returns to this call (to the next + // finally in the chain) + step->bbJumpDest->bbRefs++; + +#if defined(TARGET_ARM) + if (stepType == ST_FinallyReturn) + { + assert(step->bbJumpKind == BBJ_ALWAYS); + // Mark the target of a finally return + step->bbJumpDest->bbFlags |= BBF_FINALLY_TARGET; + } +#endif // defined(TARGET_ARM) + + /* The new block will inherit this block's weight */ + callBlock->inheritWeight(block); + callBlock->bbFlags |= BBF_IMPORTED; + +#ifdef DEBUG + if (verbose) + { + printf("impImportLeave - jumping out of a finally-protected try (EH#%u), new BBJ_CALLFINALLY " + "block " FMT_BB "\n", + XTnum, callBlock->bbNum); + } +#endif + } + + step = fgNewBBafter(BBJ_ALWAYS, callBlock, true); + stepType = ST_FinallyReturn; + + /* The new block will inherit this block's weight */ + step->inheritWeight(block); + step->bbFlags |= BBF_IMPORTED | BBF_KEEP_BBJ_ALWAYS; + +#ifdef DEBUG + if (verbose) + { + printf("impImportLeave - jumping out of a finally-protected try (EH#%u), created step (BBJ_ALWAYS) " + "block " FMT_BB "\n", + XTnum, step->bbNum); + } +#endif + + callBlock->bbJumpDest = HBtab->ebdHndBeg; // This callBlock will call the "finally" handler. + + invalidatePreds = true; + } + else if (HBtab->HasCatchHandler() && jitIsBetween(blkAddr, tryBeg, tryEnd) && + !jitIsBetween(jmpAddr, tryBeg, tryEnd)) + { + // We are jumping out of a catch-protected try. + // + // If we are returning from a call to a finally, then we must have a step block within a try + // that is protected by a catch. This is so when unwinding from that finally (e.g., if code within the + // finally raises an exception), the VM will find this step block, notice that it is in a protected region, + // and invoke the appropriate catch. + // + // We also need to handle a special case with the handling of ThreadAbortException. If a try/catch + // catches a ThreadAbortException (which might be because it catches a parent, e.g. System.Exception), + // and the catch doesn't call System.Threading.Thread::ResetAbort(), then when the catch returns to the VM, + // the VM will automatically re-raise the ThreadAbortException. When it does this, it uses the target + // address of the catch return as the new exception address. That is, the re-raised exception appears to + // occur at the catch return address. If this exception return address skips an enclosing try/catch that + // catches ThreadAbortException, then the enclosing try/catch will not catch the exception, as it should. + // For example: + // + // try { + // try { + // // something here raises ThreadAbortException + // LEAVE LABEL_1; // no need to stop at LABEL_2 + // } catch (Exception) { + // // This catches ThreadAbortException, but doesn't call System.Threading.Thread::ResetAbort(), so + // // ThreadAbortException is re-raised by the VM at the address specified by the LEAVE opcode. + // // This is bad, since it means the outer try/catch won't get a chance to catch the re-raised + // // ThreadAbortException. So, instead, create step block LABEL_2 and LEAVE to that. We only + // // need to do this transformation if the current EH block is a try/catch that catches + // // ThreadAbortException (or one of its parents), however we might not be able to find that + // // information, so currently we do it for all catch types. + // LEAVE LABEL_1; // Convert this to LEAVE LABEL2; + // } + // LABEL_2: LEAVE LABEL_1; // inserted by this step creation code + // } catch (ThreadAbortException) { + // } + // LABEL_1: + // + // Note that this pattern isn't theoretical: it occurs in ASP.NET, in IL code generated by the Roslyn C# + // compiler. + + if ((stepType == ST_FinallyReturn) || (stepType == ST_Catch)) + { + BasicBlock* catchStep; + + assert(step); + + if (stepType == ST_FinallyReturn) + { + assert(step->bbJumpKind == BBJ_ALWAYS); + } + else + { + assert(stepType == ST_Catch); + assert(step->bbJumpKind == BBJ_EHCATCHRET); + } + + /* Create a new exit block in the try region for the existing step block to jump to in this scope */ + catchStep = fgNewBBinRegion(BBJ_ALWAYS, XTnum + 1, 0, step); + step->bbJumpDest = catchStep; + step->bbJumpDest->bbRefs++; + +#if defined(TARGET_ARM) + if (stepType == ST_FinallyReturn) + { + // Mark the target of a finally return + step->bbJumpDest->bbFlags |= BBF_FINALLY_TARGET; + } +#endif // defined(TARGET_ARM) + + /* The new block will inherit this block's weight */ + catchStep->inheritWeight(block); + catchStep->bbFlags |= BBF_IMPORTED; + +#ifdef DEBUG + if (verbose) + { + if (stepType == ST_FinallyReturn) + { + printf("impImportLeave - return from finally jumping out of a catch-protected try (EH#%u), new " + "BBJ_ALWAYS block " FMT_BB "\n", + XTnum, catchStep->bbNum); + } + else + { + assert(stepType == ST_Catch); + printf("impImportLeave - return from catch jumping out of a catch-protected try (EH#%u), new " + "BBJ_ALWAYS block " FMT_BB "\n", + XTnum, catchStep->bbNum); + } + } +#endif // DEBUG + + /* This block is the new step */ + step = catchStep; + stepType = ST_Try; + + invalidatePreds = true; + } + } + } + + if (step == nullptr) + { + block->bbJumpKind = BBJ_ALWAYS; // convert the BBJ_LEAVE to a BBJ_ALWAYS + +#ifdef DEBUG + if (verbose) + { + printf("impImportLeave - no enclosing finally-protected try blocks or catch handlers; convert CEE_LEAVE " + "block " FMT_BB " to BBJ_ALWAYS\n", + block->bbNum); + } +#endif + } + else + { + step->bbJumpDest = leaveTarget; // this is the ultimate destination of the LEAVE + +#if defined(TARGET_ARM) + if (stepType == ST_FinallyReturn) + { + assert(step->bbJumpKind == BBJ_ALWAYS); + // Mark the target of a finally return + step->bbJumpDest->bbFlags |= BBF_FINALLY_TARGET; + } +#endif // defined(TARGET_ARM) + +#ifdef DEBUG + if (verbose) + { + printf("impImportLeave - final destination of step blocks set to " FMT_BB "\n", leaveTarget->bbNum); + } +#endif + + // Queue up the jump target for importing + + impImportBlockPending(leaveTarget); + } + + if (invalidatePreds && fgComputePredsDone) + { + JITDUMP("\n**** impImportLeave - Removing preds after creating new blocks\n"); + fgRemovePreds(); + } + +#ifdef DEBUG + fgVerifyHandlerTab(); + + if (verbose) + { + printf("\nAfter import CEE_LEAVE:\n"); + fgDispBasicBlocks(); + fgDispHandlerTab(); + } +#endif // DEBUG +} + +#endif // FEATURE_EH_FUNCLETS + +/*****************************************************************************/ +// This is called when reimporting a leave block. It resets the JumpKind, +// JumpDest, and bbNext to the original values + +void Compiler::impResetLeaveBlock(BasicBlock* block, unsigned jmpAddr) +{ +#if defined(FEATURE_EH_FUNCLETS) + // With EH Funclets, while importing leave opcode we create another block ending with BBJ_ALWAYS (call it B1) + // and the block containing leave (say B0) is marked as BBJ_CALLFINALLY. Say for some reason we reimport B0, + // it is reset (in this routine) by marking as ending with BBJ_LEAVE and further down when B0 is reimported, we + // create another BBJ_ALWAYS (call it B2). In this process B1 gets orphaned and any blocks to which B1 is the + // only predecessor are also considered orphans and attempted to be deleted. + // + // try { + // .... + // try + // { + // .... + // leave OUTSIDE; // B0 is the block containing this leave, following this would be B1 + // } finally { } + // } finally { } + // OUTSIDE: + // + // In the above nested try-finally example, we create a step block (call it Bstep) which in branches to a block + // where a finally would branch to (and such block is marked as finally target). Block B1 branches to step block. + // Because of re-import of B0, Bstep is also orphaned. Since Bstep is a finally target it cannot be removed. To + // work around this we will duplicate B0 (call it B0Dup) before resetting. B0Dup is marked as BBJ_CALLFINALLY and + // only serves to pair up with B1 (BBJ_ALWAYS) that got orphaned. Now during orphan block deletion B0Dup and B1 + // will be treated as pair and handled correctly. + if (block->bbJumpKind == BBJ_CALLFINALLY) + { + BasicBlock* dupBlock = bbNewBasicBlock(block->bbJumpKind); + dupBlock->bbFlags = block->bbFlags; + dupBlock->bbJumpDest = block->bbJumpDest; + dupBlock->copyEHRegion(block); + dupBlock->bbCatchTyp = block->bbCatchTyp; + + // Mark this block as + // a) not referenced by any other block to make sure that it gets deleted + // b) weight zero + // c) prevent from being imported + // d) as internal + // e) as rarely run + dupBlock->bbRefs = 0; + dupBlock->bbWeight = BB_ZERO_WEIGHT; + dupBlock->bbFlags |= BBF_IMPORTED | BBF_INTERNAL | BBF_RUN_RARELY; + + // Insert the block right after the block which is getting reset so that BBJ_CALLFINALLY and BBJ_ALWAYS + // will be next to each other. + fgInsertBBafter(block, dupBlock); + +#ifdef DEBUG + if (verbose) + { + printf("New Basic Block " FMT_BB " duplicate of " FMT_BB " created.\n", dupBlock->bbNum, block->bbNum); + } +#endif + } +#endif // FEATURE_EH_FUNCLETS + + block->bbJumpKind = BBJ_LEAVE; + fgInitBBLookup(); + block->bbJumpDest = fgLookupBB(jmpAddr); + + // We will leave the BBJ_ALWAYS block we introduced. When it's reimported + // the BBJ_ALWAYS block will be unreachable, and will be removed after. The + // reason we don't want to remove the block at this point is that if we call + // fgInitBBLookup() again we will do it wrong as the BBJ_ALWAYS block won't be + // added and the linked list length will be different than fgBBcount. +} + +/*****************************************************************************/ +// Get the first non-prefix opcode. Used for verification of valid combinations +// of prefixes and actual opcodes. + +OPCODE Compiler::impGetNonPrefixOpcode(const BYTE* codeAddr, const BYTE* codeEndp) +{ + while (codeAddr < codeEndp) + { + OPCODE opcode = (OPCODE)getU1LittleEndian(codeAddr); + codeAddr += sizeof(__int8); + + if (opcode == CEE_PREFIX1) + { + if (codeAddr >= codeEndp) + { + break; + } + opcode = (OPCODE)(getU1LittleEndian(codeAddr) + 256); + codeAddr += sizeof(__int8); + } + + switch (opcode) + { + case CEE_UNALIGNED: + case CEE_VOLATILE: + case CEE_TAILCALL: + case CEE_CONSTRAINED: + case CEE_READONLY: + break; + default: + return opcode; + } + + codeAddr += opcodeSizes[opcode]; + } + + return CEE_ILLEGAL; +} + +/*****************************************************************************/ +// Checks whether the opcode is a valid opcode for volatile. and unaligned. prefixes + +void Compiler::impValidateMemoryAccessOpcode(const BYTE* codeAddr, const BYTE* codeEndp, bool volatilePrefix) +{ + OPCODE opcode = impGetNonPrefixOpcode(codeAddr, codeEndp); + + if (!( + // Opcode of all ldind and stdind happen to be in continuous, except stind.i. + ((CEE_LDIND_I1 <= opcode) && (opcode <= CEE_STIND_R8)) || (opcode == CEE_STIND_I) || + (opcode == CEE_LDFLD) || (opcode == CEE_STFLD) || (opcode == CEE_LDOBJ) || (opcode == CEE_STOBJ) || + (opcode == CEE_INITBLK) || (opcode == CEE_CPBLK) || + // volatile. prefix is allowed with the ldsfld and stsfld + (volatilePrefix && ((opcode == CEE_LDSFLD) || (opcode == CEE_STSFLD))))) + { + BADCODE("Invalid opcode for unaligned. or volatile. prefix"); + } +} + +/*****************************************************************************/ + +#ifdef DEBUG + +#undef RETURN // undef contracts RETURN macro + +enum controlFlow_t +{ + NEXT, + CALL, + RETURN, + THROW, + BRANCH, + COND_BRANCH, + BREAK, + PHI, + META, +}; + +const static controlFlow_t controlFlow[] = { +#define OPDEF(c, s, pop, push, args, type, l, s1, s2, flow) flow, +#include "opcode.def" +#undef OPDEF +}; + +#endif // DEBUG + +/***************************************************************************** + * Determine the result type of an arithemetic operation + * On 64-bit inserts upcasts when native int is mixed with int32 + */ +var_types Compiler::impGetByRefResultType(genTreeOps oper, bool fUnsigned, GenTree** pOp1, GenTree** pOp2) +{ + var_types type = TYP_UNDEF; + GenTree* op1 = *pOp1; + GenTree* op2 = *pOp2; + + // Arithemetic operations are generally only allowed with + // primitive types, but certain operations are allowed + // with byrefs + + if ((oper == GT_SUB) && (genActualType(op1->TypeGet()) == TYP_BYREF || genActualType(op2->TypeGet()) == TYP_BYREF)) + { + if ((genActualType(op1->TypeGet()) == TYP_BYREF) && (genActualType(op2->TypeGet()) == TYP_BYREF)) + { + // byref1-byref2 => gives a native int + type = TYP_I_IMPL; + } + else if (genActualTypeIsIntOrI(op1->TypeGet()) && (genActualType(op2->TypeGet()) == TYP_BYREF)) + { + // [native] int - byref => gives a native int + + // + // The reason is that it is possible, in managed C++, + // to have a tree like this: + // + // - + // / \. + // / \. + // / \. + // / \. + // const(h) int addr byref + // + // VSW 318822 + // + // So here we decide to make the resulting type to be a native int. + CLANG_FORMAT_COMMENT_ANCHOR; + +#ifdef TARGET_64BIT + if (genActualType(op1->TypeGet()) != TYP_I_IMPL) + { + // insert an explicit upcast + op1 = *pOp1 = gtNewCastNode(TYP_I_IMPL, op1, fUnsigned, fUnsigned ? TYP_U_IMPL : TYP_I_IMPL); + } +#endif // TARGET_64BIT + + type = TYP_I_IMPL; + } + else + { + // byref - [native] int => gives a byref + assert(genActualType(op1->TypeGet()) == TYP_BYREF && genActualTypeIsIntOrI(op2->TypeGet())); + +#ifdef TARGET_64BIT + if ((genActualType(op2->TypeGet()) != TYP_I_IMPL)) + { + // insert an explicit upcast + op2 = *pOp2 = gtNewCastNode(TYP_I_IMPL, op2, fUnsigned, fUnsigned ? TYP_U_IMPL : TYP_I_IMPL); + } +#endif // TARGET_64BIT + + type = TYP_BYREF; + } + } + else if ((oper == GT_ADD) && + (genActualType(op1->TypeGet()) == TYP_BYREF || genActualType(op2->TypeGet()) == TYP_BYREF)) + { + // byref + [native] int => gives a byref + // (or) + // [native] int + byref => gives a byref + + // only one can be a byref : byref op byref not allowed + assert(genActualType(op1->TypeGet()) != TYP_BYREF || genActualType(op2->TypeGet()) != TYP_BYREF); + assert(genActualTypeIsIntOrI(op1->TypeGet()) || genActualTypeIsIntOrI(op2->TypeGet())); + +#ifdef TARGET_64BIT + if (genActualType(op2->TypeGet()) == TYP_BYREF) + { + if (genActualType(op1->TypeGet()) != TYP_I_IMPL) + { + // insert an explicit upcast + op1 = *pOp1 = gtNewCastNode(TYP_I_IMPL, op1, fUnsigned, fUnsigned ? TYP_U_IMPL : TYP_I_IMPL); + } + } + else if (genActualType(op2->TypeGet()) != TYP_I_IMPL) + { + // insert an explicit upcast + op2 = *pOp2 = gtNewCastNode(TYP_I_IMPL, op2, fUnsigned, fUnsigned ? TYP_U_IMPL : TYP_I_IMPL); + } +#endif // TARGET_64BIT + + type = TYP_BYREF; + } +#ifdef TARGET_64BIT + else if (genActualType(op1->TypeGet()) == TYP_I_IMPL || genActualType(op2->TypeGet()) == TYP_I_IMPL) + { + assert(!varTypeIsFloating(op1->gtType) && !varTypeIsFloating(op2->gtType)); + + // int + long => gives long + // long + int => gives long + // we get this because in the IL the long isn't Int64, it's just IntPtr + + if (genActualType(op1->TypeGet()) != TYP_I_IMPL) + { + // insert an explicit upcast + op1 = *pOp1 = gtNewCastNode(TYP_I_IMPL, op1, fUnsigned, fUnsigned ? TYP_U_IMPL : TYP_I_IMPL); + } + else if (genActualType(op2->TypeGet()) != TYP_I_IMPL) + { + // insert an explicit upcast + op2 = *pOp2 = gtNewCastNode(TYP_I_IMPL, op2, fUnsigned, fUnsigned ? TYP_U_IMPL : TYP_I_IMPL); + } + + type = TYP_I_IMPL; + } +#else // 32-bit TARGET + else if (genActualType(op1->TypeGet()) == TYP_LONG || genActualType(op2->TypeGet()) == TYP_LONG) + { + assert(!varTypeIsFloating(op1->gtType) && !varTypeIsFloating(op2->gtType)); + + // int + long => gives long + // long + int => gives long + + type = TYP_LONG; + } +#endif // TARGET_64BIT + else + { + // int + int => gives an int + assert(genActualType(op1->TypeGet()) != TYP_BYREF && genActualType(op2->TypeGet()) != TYP_BYREF); + + assert(genActualType(op1->TypeGet()) == genActualType(op2->TypeGet()) || + (varTypeIsFloating(op1->gtType) && varTypeIsFloating(op2->gtType))); + + type = genActualType(op1->gtType); + + // If both operands are TYP_FLOAT, then leave it as TYP_FLOAT. + // Otherwise, turn floats into doubles + if ((type == TYP_FLOAT) && (genActualType(op2->gtType) != TYP_FLOAT)) + { + assert(genActualType(op2->gtType) == TYP_DOUBLE); + type = TYP_DOUBLE; + } + } + + assert(type == TYP_BYREF || type == TYP_DOUBLE || type == TYP_FLOAT || type == TYP_LONG || type == TYP_INT); + return type; +} + +//------------------------------------------------------------------------ +// impOptimizeCastClassOrIsInst: attempt to resolve a cast when jitting +// +// Arguments: +// op1 - value to cast +// pResolvedToken - resolved token for type to cast to +// isCastClass - true if this is a castclass, false if isinst +// +// Return Value: +// tree representing optimized cast, or null if no optimization possible + +GenTree* Compiler::impOptimizeCastClassOrIsInst(GenTree* op1, CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isCastClass) +{ + assert(op1->TypeGet() == TYP_REF); + + // Don't optimize for minopts or debug codegen. + if (opts.OptimizationDisabled()) + { + return nullptr; + } + + // See what we know about the type of the object being cast. + bool isExact = false; + bool isNonNull = false; + CORINFO_CLASS_HANDLE fromClass = gtGetClassHandle(op1, &isExact, &isNonNull); + + if (fromClass != nullptr) + { + CORINFO_CLASS_HANDLE toClass = pResolvedToken->hClass; + JITDUMP("\nConsidering optimization of %s from %s%p (%s) to %p (%s)\n", isCastClass ? "castclass" : "isinst", + isExact ? "exact " : "", dspPtr(fromClass), eeGetClassName(fromClass), dspPtr(toClass), + eeGetClassName(toClass)); + + // Perhaps we know if the cast will succeed or fail. + TypeCompareState castResult = info.compCompHnd->compareTypesForCast(fromClass, toClass); + + if (castResult == TypeCompareState::Must) + { + // Cast will succeed, result is simply op1. + JITDUMP("Cast will succeed, optimizing to simply return input\n"); + return op1; + } + else if (castResult == TypeCompareState::MustNot) + { + // See if we can sharpen exactness by looking for final classes + if (!isExact) + { + isExact = impIsClassExact(fromClass); + } + + // Cast to exact type will fail. Handle case where we have + // an exact type (that is, fromClass is not a subtype) + // and we're not going to throw on failure. + if (isExact && !isCastClass) + { + JITDUMP("Cast will fail, optimizing to return null\n"); + GenTree* result = gtNewIconNode(0, TYP_REF); + + // If the cast was fed by a box, we can remove that too. + if (op1->IsBoxedValue()) + { + JITDUMP("Also removing upstream box\n"); + gtTryRemoveBoxUpstreamEffects(op1); + } + + return result; + } + else if (isExact) + { + JITDUMP("Not optimizing failing castclass (yet)\n"); + } + else + { + JITDUMP("Can't optimize since fromClass is inexact\n"); + } + } + else + { + JITDUMP("Result of cast unknown, must generate runtime test\n"); + } + } + else + { + JITDUMP("\nCan't optimize since fromClass is unknown\n"); + } + + return nullptr; +} + +//------------------------------------------------------------------------ +// impCastClassOrIsInstToTree: build and import castclass/isinst +// +// Arguments: +// op1 - value to cast +// op2 - type handle for type to cast to +// pResolvedToken - resolved token from the cast operation +// isCastClass - true if this is castclass, false means isinst +// +// Return Value: +// Tree representing the cast +// +// Notes: +// May expand into a series of runtime checks or a helper call. + +GenTree* Compiler::impCastClassOrIsInstToTree( + GenTree* op1, GenTree* op2, CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isCastClass, IL_OFFSET ilOffset) +{ + assert(op1->TypeGet() == TYP_REF); + + // Optimistically assume the jit should expand this as an inline test + bool shouldExpandInline = true; + bool isClassExact = impIsClassExact(pResolvedToken->hClass); + + // Profitability check. + // + // Don't bother with inline expansion when jit is trying to generate code quickly + if (opts.OptimizationDisabled()) + { + // not worth the code expansion if jitting fast or in a rarely run block + shouldExpandInline = false; + } + else if ((op1->gtFlags & GTF_GLOB_EFFECT) && lvaHaveManyLocals()) + { + // not worth creating an untracked local variable + shouldExpandInline = false; + } + else if (opts.jitFlags->IsSet(JitFlags::JIT_FLAG_BBINSTR) && (JitConfig.JitProfileCasts() == 1)) + { + // Optimizations are enabled but we're still instrumenting (including casts) + if (isCastClass && !isClassExact) + { + // Usually, we make a speculative assumption that it makes sense to expand castclass + // even for non-sealed classes, but let's rely on PGO in this specific case + shouldExpandInline = false; + } + } + + if (shouldExpandInline && compCurBB->isRunRarely()) + { + // For cold blocks we only expand castclass against exact classes becauses because it's cheap + shouldExpandInline = isCastClass && isClassExact; + } + + // Pessimistically assume the jit cannot expand this as an inline test + bool canExpandInline = false; + bool partialExpand = false; + const CorInfoHelpFunc helper = info.compCompHnd->getCastingHelper(pResolvedToken, isCastClass); + + GenTree* exactCls = nullptr; + + // Legality check. + // + // Not all classclass/isinst operations can be inline expanded. + // Check legality only if an inline expansion is desirable. + if (shouldExpandInline) + { + if (isCastClass) + { + // Jit can only inline expand the normal CHKCASTCLASS helper. + canExpandInline = (helper == CORINFO_HELP_CHKCASTCLASS); + } + else + { + if (helper == CORINFO_HELP_ISINSTANCEOFCLASS) + { + // If the class is exact, the jit can expand the IsInst check inline. + canExpandInline = isClassExact; + } + } + + // Check if this cast helper have some profile data + if (impIsCastHelperMayHaveProfileData(helper)) + { + const int maxLikelyClasses = 32; + LikelyClassMethodRecord likelyClasses[maxLikelyClasses]; + unsigned likelyClassCount = + getLikelyClasses(likelyClasses, maxLikelyClasses, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset); + + if (likelyClassCount > 0) + { +#ifdef DEBUG + // Optional stress mode to pick a random known class, rather than + // the most likely known class. + if (JitConfig.JitRandomGuardedDevirtualization() != 0) + { + // Reuse the random inliner's random state. + CLRRandom* const random = + impInlineRoot()->m_inlineStrategy->GetRandom(JitConfig.JitRandomGuardedDevirtualization()); + + unsigned index = static_cast(random->Next(static_cast(likelyClassCount))); + likelyClasses[0].handle = likelyClasses[index].handle; + likelyClasses[0].likelihood = 100; + likelyClassCount = 1; + } +#endif + + LikelyClassMethodRecord likelyClass = likelyClasses[0]; + CORINFO_CLASS_HANDLE likelyCls = (CORINFO_CLASS_HANDLE)likelyClass.handle; + + if ((likelyCls != NO_CLASS_HANDLE) && + (likelyClass.likelihood > (UINT32)JitConfig.JitGuardedDevirtualizationChainLikelihood())) + { + if ((info.compCompHnd->compareTypesForCast(likelyCls, pResolvedToken->hClass) == + TypeCompareState::Must)) + { + assert((info.compCompHnd->getClassAttribs(likelyCls) & + (CORINFO_FLG_INTERFACE | CORINFO_FLG_ABSTRACT)) == 0); + JITDUMP("Adding \"is %s (%X)\" check as a fast path for %s using PGO data.\n", + eeGetClassName(likelyCls), likelyCls, isCastClass ? "castclass" : "isinst"); + + canExpandInline = true; + partialExpand = true; + exactCls = gtNewIconEmbClsHndNode(likelyCls); + } + } + } + } + } + + const bool expandInline = canExpandInline && shouldExpandInline; + + if (!expandInline) + { + JITDUMP("\nExpanding %s as call because %s\n", isCastClass ? "castclass" : "isinst", + canExpandInline ? "want smaller code or faster jitting" : "inline expansion not legal"); + + // If we CSE this class handle we prevent assertionProp from making SubType assertions + // so instead we force the CSE logic to not consider CSE-ing this class handle. + // + op2->gtFlags |= GTF_DONT_CSE; + + GenTreeCall* call = gtNewHelperCallNode(helper, TYP_REF, op2, op1); + if ((JitConfig.JitClassProfiling() > 0) && impIsCastHelperEligibleForClassProbe(call) && !isClassExact) + { + HandleHistogramProfileCandidateInfo* pInfo = new (this, CMK_Inlining) HandleHistogramProfileCandidateInfo; + pInfo->ilOffset = ilOffset; + pInfo->probeIndex = info.compHandleHistogramProbeCount++; + call->gtHandleHistogramProfileCandidateInfo = pInfo; + compCurBB->bbFlags |= BBF_HAS_HISTOGRAM_PROFILE; + } + return call; + } + + JITDUMP("\nExpanding %s inline\n", isCastClass ? "castclass" : "isinst"); + + impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("bubbling QMark2")); + + GenTree* temp; + GenTree* condMT; + // + // expand the methodtable match: + // + // condMT ==> GT_NE + // / \. + // GT_IND op2 (typically CNS_INT) + // | + // op1Copy + // + + // This can replace op1 with a GT_COMMA that evaluates op1 into a local + // + op1 = impCloneExpr(op1, &temp, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, nullptr DEBUGARG("CASTCLASS eval op1")); + // + // op1 is now known to be a non-complex tree + // thus we can use gtClone(op1) from now on + // + + GenTree* op2Var = op2; + if (isCastClass && !partialExpand) + { + op2Var = fgInsertCommaFormTemp(&op2); + lvaTable[op2Var->AsLclVarCommon()->GetLclNum()].lvIsCSE = true; + } + temp = gtNewMethodTableLookup(temp); + condMT = gtNewOperNode(GT_NE, TYP_INT, temp, exactCls != nullptr ? exactCls : op2); + + GenTree* condNull; + // + // expand the null check: + // + // condNull ==> GT_EQ + // / \. + // op1Copy CNS_INT + // null + // + condNull = gtNewOperNode(GT_EQ, TYP_INT, gtClone(op1), gtNewIconNode(0, TYP_REF)); + + // + // expand the true and false trees for the condMT + // + GenTree* condFalse = gtClone(op1); + GenTree* condTrue; + if (isCastClass) + { + // + // use the special helper that skips the cases checked by our inlined cast + // + const CorInfoHelpFunc specialHelper = CORINFO_HELP_CHKCASTCLASS_SPECIAL; + + condTrue = gtNewHelperCallNode(specialHelper, TYP_REF, partialExpand ? op2 : op2Var, gtClone(op1)); + } + else if (partialExpand) + { + condTrue = gtNewHelperCallNode(helper, TYP_REF, op2, gtClone(op1)); + } + else + { + condTrue = gtNewIconNode(0, TYP_REF); + } + + GenTree* qmarkMT; + // + // Generate first QMARK - COLON tree + // + // qmarkMT ==> GT_QMARK + // / \. + // condMT GT_COLON + // / \. + // condFalse condTrue + // + temp = new (this, GT_COLON) GenTreeColon(TYP_REF, condTrue, condFalse); + qmarkMT = gtNewQmarkNode(TYP_REF, condMT, temp->AsColon()); + + if (isCastClass && isClassExact && condTrue->OperIs(GT_CALL)) + { + // condTrue is used only for throwing InvalidCastException in case of casting to an exact class. + condTrue->AsCall()->gtCallMoreFlags |= GTF_CALL_M_DOES_NOT_RETURN; + } + + GenTree* qmarkNull; + // + // Generate second QMARK - COLON tree + // + // qmarkNull ==> GT_QMARK + // / \. + // condNull GT_COLON + // / \. + // qmarkMT op1Copy + // + temp = new (this, GT_COLON) GenTreeColon(TYP_REF, gtClone(op1), qmarkMT); + qmarkNull = gtNewQmarkNode(TYP_REF, condNull, temp->AsColon()); + qmarkNull->gtFlags |= GTF_QMARK_CAST_INSTOF; + + // Make QMark node a top level node by spilling it. + unsigned tmp = lvaGrabTemp(true DEBUGARG("spilling QMark2")); + impAssignTempGen(tmp, qmarkNull, (unsigned)CHECK_SPILL_NONE); + + // TODO-CQ: Is it possible op1 has a better type? + // + // See also gtGetHelperCallClassHandle where we make the same + // determination for the helper call variants. + LclVarDsc* lclDsc = lvaGetDesc(tmp); + assert(lclDsc->lvSingleDef == 0); + lclDsc->lvSingleDef = 1; + JITDUMP("Marked V%02u as a single def temp\n", tmp); + lvaSetClass(tmp, pResolvedToken->hClass); + return gtNewLclvNode(tmp, TYP_REF); +} + +#ifndef DEBUG +#define assertImp(cond) ((void)0) +#else +#define assertImp(cond) \ + do \ + { \ + if (!(cond)) \ + { \ + const int cchAssertImpBuf = 600; \ + char* assertImpBuf = (char*)_alloca(cchAssertImpBuf); \ + _snprintf_s(assertImpBuf, cchAssertImpBuf, cchAssertImpBuf - 1, \ + "%s : Possibly bad IL with CEE_%s at offset %04Xh (op1=%s op2=%s stkDepth=%d)", #cond, \ + impCurOpcName, impCurOpcOffs, op1 ? varTypeName(op1->TypeGet()) : "NULL", \ + op2 ? varTypeName(op2->TypeGet()) : "NULL", verCurrentState.esStackDepth); \ + assertAbort(assertImpBuf, __FILE__, __LINE__); \ + } \ + } while (0) +#endif // DEBUG + +//------------------------------------------------------------------------ +// impBlockIsInALoop: check if a block might be in a loop +// +// Arguments: +// block - block to check +// +// Returns: +// true if the block might be in a loop. +// +// Notes: +// Conservatively correct; may return true for some blocks that are +// not actually in loops. +// +bool Compiler::impBlockIsInALoop(BasicBlock* block) +{ + return (compIsForInlining() && ((impInlineInfo->iciBlock->bbFlags & BBF_BACKWARD_JUMP) != 0)) || + ((block->bbFlags & BBF_BACKWARD_JUMP) != 0); +} + +#ifdef _PREFAST_ +#pragma warning(push) +#pragma warning(disable : 21000) // Suppress PREFast warning about overly large function +#endif +/***************************************************************************** + * Import the instr for the given basic block + */ +void Compiler::impImportBlockCode(BasicBlock* block) +{ +#define _impResolveToken(kind) impResolveToken(codeAddr, &resolvedToken, kind) + +#ifdef DEBUG + + if (verbose) + { + printf("\nImporting " FMT_BB " (PC=%03u) of '%s'", block->bbNum, block->bbCodeOffs, info.compFullName); + } +#endif + + unsigned nxtStmtIndex = impInitBlockLineInfo(); + IL_OFFSET nxtStmtOffs; + CorInfoHelpFunc helper; + CorInfoIsAccessAllowedResult accessAllowedResult; + CORINFO_HELPER_DESC calloutHelper; + const BYTE* lastLoadToken = nullptr; + + /* Get the tree list started */ + + impBeginTreeList(); + +#ifdef FEATURE_ON_STACK_REPLACEMENT + + bool enablePatchpoints = opts.jitFlags->IsSet(JitFlags::JIT_FLAG_TIER0) && (JitConfig.TC_OnStackReplacement() > 0); + +#ifdef DEBUG + + // Optionally suppress patchpoints by method hash + // + static ConfigMethodRange JitEnablePatchpointRange; + JitEnablePatchpointRange.EnsureInit(JitConfig.JitEnablePatchpointRange()); + const unsigned hash = impInlineRoot()->info.compMethodHash(); + const bool inRange = JitEnablePatchpointRange.Contains(hash); + enablePatchpoints &= inRange; + +#endif // DEBUG + + if (enablePatchpoints) + { + // We don't inline at Tier0, if we do, we may need rethink our approach. + // Could probably support inlines that don't introduce flow. + // + assert(!compIsForInlining()); + + // OSR is not yet supported for methods with explicit tail calls. + // + // But we also do not have to switch these methods to be optimized, as we should be + // able to avoid getting trapped in Tier0 code by normal call counting. + // So instead, just suppress adding patchpoints. + // + if (!compTailPrefixSeen) + { + // We only need to add patchpoints if the method can loop. + // + if (compHasBackwardJump) + { + assert(compCanHavePatchpoints()); + + // By default we use the "adaptive" strategy. + // + // This can create both source and target patchpoints within a given + // loop structure, which isn't ideal, but is not incorrect. We will + // just have some extra Tier0 overhead. + // + // Todo: implement support for mid-block patchpoints. If `block` + // is truly a backedge source (and not in a handler) then we should be + // able to find a stack empty point somewhere in the block. + // + const int patchpointStrategy = JitConfig.TC_PatchpointStrategy(); + bool addPatchpoint = false; + bool mustUseTargetPatchpoint = false; + + switch (patchpointStrategy) + { + default: + { + // Patchpoints at backedge sources, if possible, otherwise targets. + // + addPatchpoint = ((block->bbFlags & BBF_BACKWARD_JUMP_SOURCE) == BBF_BACKWARD_JUMP_SOURCE); + mustUseTargetPatchpoint = (verCurrentState.esStackDepth != 0) || block->hasHndIndex(); + break; + } + + case 1: + { + // Patchpoints at stackempty backedge targets. + // Note if we have loops where the IL stack is not empty on the backedge we can't patchpoint + // them. + // + // We should not have allowed OSR if there were backedges in handlers. + // + assert(!block->hasHndIndex()); + addPatchpoint = ((block->bbFlags & BBF_BACKWARD_JUMP_TARGET) == BBF_BACKWARD_JUMP_TARGET) && + (verCurrentState.esStackDepth == 0); + break; + } + + case 2: + { + // Adaptive strategy. + // + // Patchpoints at backedge targets if there are multiple backedges, + // otherwise at backedge sources, if possible. Note a block can be both; if so we + // just need one patchpoint. + // + if ((block->bbFlags & BBF_BACKWARD_JUMP_TARGET) == BBF_BACKWARD_JUMP_TARGET) + { + // We don't know backedge count, so just use ref count. + // + addPatchpoint = (block->bbRefs > 1) && (verCurrentState.esStackDepth == 0); + } + + if (!addPatchpoint && ((block->bbFlags & BBF_BACKWARD_JUMP_SOURCE) == BBF_BACKWARD_JUMP_SOURCE)) + { + addPatchpoint = true; + mustUseTargetPatchpoint = (verCurrentState.esStackDepth != 0) || block->hasHndIndex(); + + // Also force target patchpoint if target block has multiple (backedge) preds. + // + if (!mustUseTargetPatchpoint) + { + for (BasicBlock* const succBlock : block->Succs(this)) + { + if ((succBlock->bbNum <= block->bbNum) && (succBlock->bbRefs > 1)) + { + mustUseTargetPatchpoint = true; + break; + } + } + } + } + break; + } + } + + if (addPatchpoint) + { + if (mustUseTargetPatchpoint) + { + // We wanted a source patchpoint, but could not have one. + // So, add patchpoints to the backedge targets. + // + for (BasicBlock* const succBlock : block->Succs(this)) + { + if (succBlock->bbNum <= block->bbNum) + { + // The succBlock had better agree it's a target. + // + assert((succBlock->bbFlags & BBF_BACKWARD_JUMP_TARGET) == BBF_BACKWARD_JUMP_TARGET); + + // We may already have decided to put a patchpoint in succBlock. If not, add one. + // + if ((succBlock->bbFlags & BBF_PATCHPOINT) != 0) + { + // In some cases the target may not be stack-empty at entry. + // If so, we will bypass patchpoints for this backedge. + // + if (succBlock->bbStackDepthOnEntry() > 0) + { + JITDUMP("\nCan't set source patchpoint at " FMT_BB ", can't use target " FMT_BB + " as it has non-empty stack on entry.\n", + block->bbNum, succBlock->bbNum); + } + else + { + JITDUMP("\nCan't set source patchpoint at " FMT_BB ", using target " FMT_BB + " instead\n", + block->bbNum, succBlock->bbNum); + + assert(!succBlock->hasHndIndex()); + succBlock->bbFlags |= BBF_PATCHPOINT; + } + } + } + } + } + else + { + assert(!block->hasHndIndex()); + block->bbFlags |= BBF_PATCHPOINT; + } + + setMethodHasPatchpoint(); + } + } + else + { + // Should not see backward branch targets w/o backwards branches. + // So if !compHasBackwardsBranch, these flags should never be set. + // + assert((block->bbFlags & (BBF_BACKWARD_JUMP_TARGET | BBF_BACKWARD_JUMP_SOURCE)) == 0); + } + } + +#ifdef DEBUG + // As a stress test, we can place patchpoints at the start of any block + // that is a stack empty point and is not within a handler. + // + // Todo: enable for mid-block stack empty points too. + // + const int offsetOSR = JitConfig.JitOffsetOnStackReplacement(); + const int randomOSR = JitConfig.JitRandomOnStackReplacement(); + const bool tryOffsetOSR = offsetOSR >= 0; + const bool tryRandomOSR = randomOSR > 0; + + if (compCanHavePatchpoints() && (tryOffsetOSR || tryRandomOSR) && (verCurrentState.esStackDepth == 0) && + !block->hasHndIndex() && ((block->bbFlags & BBF_PATCHPOINT) == 0)) + { + // Block start can have a patchpoint. See if we should add one. + // + bool addPatchpoint = false; + + // Specific offset? + // + if (tryOffsetOSR) + { + if (impCurOpcOffs == (unsigned)offsetOSR) + { + addPatchpoint = true; + } + } + // Random? + // + else + { + // Reuse the random inliner's random state. + // Note m_inlineStrategy is always created, even if we're not inlining. + // + CLRRandom* const random = impInlineRoot()->m_inlineStrategy->GetRandom(randomOSR); + const int randomValue = (int)random->Next(100); - // int + long => gives long - // long + int => gives long - // we get this because in the IL the long isn't Int64, it's just IntPtr + addPatchpoint = (randomValue < randomOSR); + } - if (genActualType(op1->TypeGet()) != TYP_I_IMPL) - { - // insert an explicit upcast - op1 = *pOp1 = gtNewCastNode(TYP_I_IMPL, op1, fUnsigned, fUnsigned ? TYP_U_IMPL : TYP_I_IMPL); - } - else if (genActualType(op2->TypeGet()) != TYP_I_IMPL) - { - // insert an explicit upcast - op2 = *pOp2 = gtNewCastNode(TYP_I_IMPL, op2, fUnsigned, fUnsigned ? TYP_U_IMPL : TYP_I_IMPL); + if (addPatchpoint) + { + block->bbFlags |= BBF_PATCHPOINT; + setMethodHasPatchpoint(); + } + + JITDUMP("\n** %s patchpoint%s added to " FMT_BB " (il offset %u)\n", tryOffsetOSR ? "offset" : "random", + addPatchpoint ? "" : " not", block->bbNum, impCurOpcOffs); } - type = TYP_I_IMPL; +#endif // DEBUG } -#else // 32-bit TARGET - else if (genActualType(op1->TypeGet()) == TYP_LONG || genActualType(op2->TypeGet()) == TYP_LONG) - { - assert(!varTypeIsFloating(op1->gtType) && !varTypeIsFloating(op2->gtType)); - - // int + long => gives long - // long + int => gives long - type = TYP_LONG; - } -#endif // TARGET_64BIT - else + // Mark stack-empty rare blocks to be considered for partial compilation. + // + // Ideally these are conditionally executed blocks -- if the method is going + // to unconditionally throw, there's not as much to be gained by deferring jitting. + // For now, we just screen out the entry bb. + // + // In general we might want track all the IL stack empty points so we can + // propagate rareness back through flow and place the partial compilation patchpoints "earlier" + // so there are fewer overall. + // + // Note unlike OSR, it's ok to forgo these. + // + // Todo: stress mode... + // + if (opts.jitFlags->IsSet(JitFlags::JIT_FLAG_TIER0) && (JitConfig.TC_PartialCompilation() > 0) && + compCanHavePatchpoints() && !compTailPrefixSeen) { - // int + int => gives an int - assert(genActualType(op1->TypeGet()) != TYP_BYREF && genActualType(op2->TypeGet()) != TYP_BYREF); - - assert(genActualType(op1->TypeGet()) == genActualType(op2->TypeGet()) || - (varTypeIsFloating(op1->gtType) && varTypeIsFloating(op2->gtType))); - - type = genActualType(op1->gtType); - - // If both operands are TYP_FLOAT, then leave it as TYP_FLOAT. - // Otherwise, turn floats into doubles - if ((type == TYP_FLOAT) && (genActualType(op2->gtType) != TYP_FLOAT)) + // Is this block a good place for partial compilation? + // + if ((block != fgFirstBB) && block->isRunRarely() && (verCurrentState.esStackDepth == 0) && + ((block->bbFlags & BBF_PATCHPOINT) == 0) && !block->hasHndIndex()) { - assert(genActualType(op2->gtType) == TYP_DOUBLE); - type = TYP_DOUBLE; - } - } - - assert(type == TYP_BYREF || type == TYP_DOUBLE || type == TYP_FLOAT || type == TYP_LONG || type == TYP_INT); - return type; -} + JITDUMP("\nBlock " FMT_BB " will be a partial compilation patchpoint -- not importing\n", block->bbNum); + block->bbFlags |= BBF_PARTIAL_COMPILATION_PATCHPOINT; + setMethodHasPartialCompilationPatchpoint(); -//------------------------------------------------------------------------ -// impOptimizeCastClassOrIsInst: attempt to resolve a cast when jitting -// -// Arguments: -// op1 - value to cast -// pResolvedToken - resolved token for type to cast to -// isCastClass - true if this is a castclass, false if isinst -// -// Return Value: -// tree representing optimized cast, or null if no optimization possible + // Change block to BBJ_THROW so we won't trigger importation of successors. + // + block->bbJumpKind = BBJ_THROW; -GenTree* Compiler::impOptimizeCastClassOrIsInst(GenTree* op1, CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isCastClass) -{ - assert(op1->TypeGet() == TYP_REF); + // If this method has a explicit generic context, the only uses of it may be in + // the IL for this block. So assume it's used. + // + if (info.compMethodInfo->options & + (CORINFO_GENERICS_CTXT_FROM_METHODDESC | CORINFO_GENERICS_CTXT_FROM_METHODTABLE)) + { + lvaGenericsContextInUse = true; + } - // Don't optimize for minopts or debug codegen. - if (opts.OptimizationDisabled()) - { - return nullptr; + return; + } } - // See what we know about the type of the object being cast. - bool isExact = false; - bool isNonNull = false; - CORINFO_CLASS_HANDLE fromClass = gtGetClassHandle(op1, &isExact, &isNonNull); +#endif // FEATURE_ON_STACK_REPLACEMENT - if (fromClass != nullptr) - { - CORINFO_CLASS_HANDLE toClass = pResolvedToken->hClass; - JITDUMP("\nConsidering optimization of %s from %s%p (%s) to %p (%s)\n", isCastClass ? "castclass" : "isinst", - isExact ? "exact " : "", dspPtr(fromClass), eeGetClassName(fromClass), dspPtr(toClass), - eeGetClassName(toClass)); + /* Walk the opcodes that comprise the basic block */ - // Perhaps we know if the cast will succeed or fail. - TypeCompareState castResult = info.compCompHnd->compareTypesForCast(fromClass, toClass); + const BYTE* codeAddr = info.compCode + block->bbCodeOffs; + const BYTE* codeEndp = info.compCode + block->bbCodeOffsEnd; - if (castResult == TypeCompareState::Must) - { - // Cast will succeed, result is simply op1. - JITDUMP("Cast will succeed, optimizing to simply return input\n"); - return op1; - } - else if (castResult == TypeCompareState::MustNot) - { - // See if we can sharpen exactness by looking for final classes - if (!isExact) - { - isExact = impIsClassExact(fromClass); - } + IL_OFFSET opcodeOffs = block->bbCodeOffs; + IL_OFFSET lastSpillOffs = opcodeOffs; - // Cast to exact type will fail. Handle case where we have - // an exact type (that is, fromClass is not a subtype) - // and we're not going to throw on failure. - if (isExact && !isCastClass) - { - JITDUMP("Cast will fail, optimizing to return null\n"); - GenTree* result = gtNewIconNode(0, TYP_REF); + signed jmpDist; - // If the cast was fed by a box, we can remove that too. - if (op1->IsBoxedValue()) - { - JITDUMP("Also removing upstream box\n"); - gtTryRemoveBoxUpstreamEffects(op1); - } + /* remember the start of the delegate creation sequence (used for verification) */ + const BYTE* delegateCreateStart = nullptr; - return result; - } - else if (isExact) - { - JITDUMP("Not optimizing failing castclass (yet)\n"); - } - else - { - JITDUMP("Can't optimize since fromClass is inexact\n"); - } - } - else - { - JITDUMP("Result of cast unknown, must generate runtime test\n"); - } - } - else - { - JITDUMP("\nCan't optimize since fromClass is unknown\n"); - } + int prefixFlags = 0; + bool explicitTailCall, constraintCall, readonlyCall; - return nullptr; -} + typeInfo tiRetVal; -//------------------------------------------------------------------------ -// impCastClassOrIsInstToTree: build and import castclass/isinst -// -// Arguments: -// op1 - value to cast -// op2 - type handle for type to cast to -// pResolvedToken - resolved token from the cast operation -// isCastClass - true if this is castclass, false means isinst -// -// Return Value: -// Tree representing the cast -// -// Notes: -// May expand into a series of runtime checks or a helper call. + unsigned numArgs = info.compArgsCount; -GenTree* Compiler::impCastClassOrIsInstToTree( - GenTree* op1, GenTree* op2, CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isCastClass, IL_OFFSET ilOffset) -{ - assert(op1->TypeGet() == TYP_REF); + /* Now process all the opcodes in the block */ - // Optimistically assume the jit should expand this as an inline test - bool shouldExpandInline = true; - bool isClassExact = impIsClassExact(pResolvedToken->hClass); + var_types callTyp = TYP_COUNT; + OPCODE prevOpcode = CEE_ILLEGAL; - // Profitability check. - // - // Don't bother with inline expansion when jit is trying to generate code quickly - if (opts.OptimizationDisabled()) - { - // not worth the code expansion if jitting fast or in a rarely run block - shouldExpandInline = false; - } - else if ((op1->gtFlags & GTF_GLOB_EFFECT) && lvaHaveManyLocals()) - { - // not worth creating an untracked local variable - shouldExpandInline = false; - } - else if (opts.jitFlags->IsSet(JitFlags::JIT_FLAG_BBINSTR) && (JitConfig.JitProfileCasts() == 1)) + if (block->bbCatchTyp) { - // Optimizations are enabled but we're still instrumenting (including casts) - if (isCastClass && !isClassExact) + if (info.compStmtOffsetsImplicit & ICorDebugInfo::CALL_SITE_BOUNDARIES) { - // Usually, we make a speculative assumption that it makes sense to expand castclass - // even for non-sealed classes, but let's rely on PGO in this specific case - shouldExpandInline = false; + impCurStmtOffsSet(block->bbCodeOffs); } - } - if (shouldExpandInline && compCurBB->isRunRarely()) - { - // For cold blocks we only expand castclass against exact classes because it's cheap - shouldExpandInline = isCastClass && isClassExact; + // We will spill the GT_CATCH_ARG and the input of the BB_QMARK block + // to a temp. This is a trade off for code simplicity + impSpillSpecialSideEff(); } - // Pessimistically assume the jit cannot expand this as an inline test - bool canExpandInline = false; - bool partialExpand = false; - const CorInfoHelpFunc helper = info.compCompHnd->getCastingHelper(pResolvedToken, isCastClass); - - CORINFO_CLASS_HANDLE exactCls = NO_CLASS_HANDLE; - - // Legality check. - // - // Not all classclass/isinst operations can be inline expanded. - // Check legality only if an inline expansion is desirable. - if (shouldExpandInline) + while (codeAddr < codeEndp) { - if (isCastClass) +#ifdef FEATURE_READYTORUN + bool usingReadyToRunHelper = false; +#endif + CORINFO_RESOLVED_TOKEN resolvedToken; + CORINFO_RESOLVED_TOKEN constrainedResolvedToken = {}; + CORINFO_CALL_INFO callInfo; + CORINFO_FIELD_INFO fieldInfo; + + tiRetVal = typeInfo(); // Default type info + + //--------------------------------------------------------------------- + + /* We need to restrict the max tree depth as many of the Compiler + functions are recursive. We do this by spilling the stack */ + + if (verCurrentState.esStackDepth) { - // Jit can only inline expand CHKCASTCLASS and CHKCASTARRAY helpers. - canExpandInline = (helper == CORINFO_HELP_CHKCASTCLASS) || (helper == CORINFO_HELP_CHKCASTARRAY); + /* Has it been a while since we last saw a non-empty stack (which + guarantees that the tree depth isnt accumulating. */ + + if ((opcodeOffs - lastSpillOffs) > MAX_TREE_SIZE && impCanSpillNow(prevOpcode)) + { + impSpillStackEnsure(); + lastSpillOffs = opcodeOffs; + } } - else if ((helper == CORINFO_HELP_ISINSTANCEOFCLASS) || (helper == CORINFO_HELP_ISINSTANCEOFARRAY)) + else { - // If the class is exact, the jit can expand the IsInst check inline. - canExpandInline = isClassExact; + lastSpillOffs = opcodeOffs; + impBoxTempInUse = false; // nothing on the stack, box temp OK to use again } - // Check if this cast helper have some profile data - if (impIsCastHelperMayHaveProfileData(helper)) - { - const int maxLikelyClasses = 32; - LikelyClassMethodRecord likelyClasses[maxLikelyClasses]; - unsigned likelyClassCount = - getLikelyClasses(likelyClasses, maxLikelyClasses, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset); + /* Compute the current instr offset */ - if (likelyClassCount > 0) - { -#ifdef DEBUG - // Optional stress mode to pick a random known class, rather than - // the most likely known class. - if (JitConfig.JitRandomGuardedDevirtualization() != 0) - { - // Reuse the random inliner's random state. - CLRRandom* const random = - impInlineRoot()->m_inlineStrategy->GetRandom(JitConfig.JitRandomGuardedDevirtualization()); + opcodeOffs = (IL_OFFSET)(codeAddr - info.compCode); - unsigned index = static_cast(random->Next(static_cast(likelyClassCount))); - likelyClasses[0].handle = likelyClasses[index].handle; - likelyClasses[0].likelihood = 100; - likelyClassCount = 1; - } +#ifndef DEBUG + if (opts.compDbgInfo) #endif + { + nxtStmtOffs = + (nxtStmtIndex < info.compStmtOffsetsCount) ? info.compStmtOffsets[nxtStmtIndex] : BAD_IL_OFFSET; - LikelyClassMethodRecord likelyClass = likelyClasses[0]; - CORINFO_CLASS_HANDLE likelyCls = (CORINFO_CLASS_HANDLE)likelyClass.handle; + /* Have we reached the next stmt boundary ? */ - if ((likelyCls != NO_CLASS_HANDLE) && - (likelyClass.likelihood > (UINT32)JitConfig.JitGuardedDevirtualizationChainLikelihood())) + if (nxtStmtOffs != BAD_IL_OFFSET && opcodeOffs >= nxtStmtOffs) + { + assert(nxtStmtOffs == info.compStmtOffsets[nxtStmtIndex]); + + if (verCurrentState.esStackDepth != 0 && opts.compDbgCode) { - if ((info.compCompHnd->compareTypesForCast(likelyCls, pResolvedToken->hClass) == - TypeCompareState::Must)) - { - assert((info.compCompHnd->getClassAttribs(likelyCls) & - (CORINFO_FLG_INTERFACE | CORINFO_FLG_ABSTRACT)) == 0); - JITDUMP("Adding \"is %s (%X)\" check as a fast path for %s using PGO data.\n", - eeGetClassName(likelyCls), likelyCls, isCastClass ? "castclass" : "isinst"); + /* We need to provide accurate IP-mapping at this point. + So spill anything on the stack so that it will form + gtStmts with the correct stmt offset noted */ - canExpandInline = true; - partialExpand = true; - exactCls = likelyCls; - } + impSpillStackEnsure(true); } - } - } - } - const bool expandInline = canExpandInline && shouldExpandInline; + // Have we reported debug info for any tree? - if (!expandInline) - { - JITDUMP("\nExpanding %s as call because %s\n", isCastClass ? "castclass" : "isinst", - canExpandInline ? "want smaller code or faster jitting" : "inline expansion not legal"); + if (impCurStmtDI.IsValid() && opts.compDbgCode) + { + GenTree* placeHolder = new (this, GT_NO_OP) GenTree(GT_NO_OP, TYP_VOID); + impAppendTree(placeHolder, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); - // If we CSE this class handle we prevent assertionProp from making SubType assertions - // so instead we force the CSE logic to not consider CSE-ing this class handle. - // - op2->gtFlags |= GTF_DONT_CSE; + assert(!impCurStmtDI.IsValid()); + } - GenTreeCall* call = gtNewHelperCallNode(helper, TYP_REF, op2, op1); - if ((JitConfig.JitClassProfiling() > 0) && impIsCastHelperEligibleForClassProbe(call) && !isClassExact) - { - HandleHistogramProfileCandidateInfo* pInfo = new (this, CMK_Inlining) HandleHistogramProfileCandidateInfo; - pInfo->ilOffset = ilOffset; - pInfo->probeIndex = info.compHandleHistogramProbeCount++; - call->gtHandleHistogramProfileCandidateInfo = pInfo; - compCurBB->bbFlags |= BBF_HAS_HISTOGRAM_PROFILE; - } - return call; - } + if (!impCurStmtDI.IsValid()) + { + /* Make sure that nxtStmtIndex is in sync with opcodeOffs. + If opcodeOffs has gone past nxtStmtIndex, catch up */ - JITDUMP("\nExpanding %s inline\n", isCastClass ? "castclass" : "isinst"); + while ((nxtStmtIndex + 1) < info.compStmtOffsetsCount && + info.compStmtOffsets[nxtStmtIndex + 1] <= opcodeOffs) + { + nxtStmtIndex++; + } - impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("bubbling QMark2")); + /* Go to the new stmt */ - GenTree* temp; - GenTree* condMT; - // - // expand the methodtable match: - // - // condMT ==> GT_NE - // / \. - // GT_IND op2 (typically CNS_INT) - // | - // op1Copy - // + impCurStmtOffsSet(info.compStmtOffsets[nxtStmtIndex]); - // This can replace op1 with a GT_COMMA that evaluates op1 into a local - // - op1 = impCloneExpr(op1, &temp, NO_CLASS_HANDLE, CHECK_SPILL_ALL, nullptr DEBUGARG("CASTCLASS eval op1")); - // - // op1 is now known to be a non-complex tree - // thus we can use gtClone(op1) from now on - // + /* Update the stmt boundary index */ - GenTree* op2Var = op2; - if (isCastClass && !partialExpand) - { - op2Var = fgInsertCommaFormTemp(&op2); - lvaTable[op2Var->AsLclVarCommon()->GetLclNum()].lvIsCSE = true; - } - temp = gtNewMethodTableLookup(temp); - condMT = - gtNewOperNode(GT_NE, TYP_INT, temp, (exactCls != NO_CLASS_HANDLE) ? gtNewIconEmbClsHndNode(exactCls) : op2); + nxtStmtIndex++; + assert(nxtStmtIndex <= info.compStmtOffsetsCount); - GenTree* condNull; - // - // expand the null check: - // - // condNull ==> GT_EQ - // / \. - // op1Copy CNS_INT - // null - // - condNull = gtNewOperNode(GT_EQ, TYP_INT, gtClone(op1), gtNewIconNode(0, TYP_REF)); + /* Are there any more line# entries after this one? */ - // - // expand the true and false trees for the condMT - // - GenTree* condFalse = gtClone(op1); - GenTree* condTrue; - if (isCastClass) - { - assert((helper == CORINFO_HELP_CHKCASTCLASS) || (helper == CORINFO_HELP_CHKCASTARRAY) || - (helper == CORINFO_HELP_CHKCASTINTERFACE)); + if (nxtStmtIndex < info.compStmtOffsetsCount) + { + /* Remember where the next line# starts */ - CorInfoHelpFunc specialHelper = helper; - if ((helper == CORINFO_HELP_CHKCASTCLASS) && - ((exactCls == nullptr) || (exactCls == gtGetHelperArgClassHandle(op2)))) - { - // use the special helper that skips the cases checked by our inlined cast - specialHelper = CORINFO_HELP_CHKCASTCLASS_SPECIAL; - } - condTrue = gtNewHelperCallNode(specialHelper, TYP_REF, partialExpand ? op2 : op2Var, gtClone(op1)); - } - else if (partialExpand) - { - condTrue = gtNewHelperCallNode(helper, TYP_REF, op2, gtClone(op1)); - } - else - { - condTrue = gtNewIconNode(0, TYP_REF); - } + nxtStmtOffs = info.compStmtOffsets[nxtStmtIndex]; + } + else + { + /* No more line# entries */ - GenTree* qmarkMT; - // - // Generate first QMARK - COLON tree - // - // qmarkMT ==> GT_QMARK - // / \. - // condMT GT_COLON - // / \. - // condFalse condTrue - // - temp = new (this, GT_COLON) GenTreeColon(TYP_REF, condTrue, condFalse); - qmarkMT = gtNewQmarkNode(TYP_REF, condMT, temp->AsColon()); + nxtStmtOffs = BAD_IL_OFFSET; + } + } + } + else if ((info.compStmtOffsetsImplicit & ICorDebugInfo::STACK_EMPTY_BOUNDARIES) && + (verCurrentState.esStackDepth == 0)) + { + /* At stack-empty locations, we have already added the tree to + the stmt list with the last offset. We just need to update + impCurStmtDI + */ - if (isCastClass && isClassExact && condTrue->OperIs(GT_CALL)) - { - if (helper == CORINFO_HELP_CHKCASTCLASS) - { - // condTrue is used only for throwing InvalidCastException in case of casting to an exact class. - condTrue->AsCall()->gtCallMoreFlags |= GTF_CALL_M_DOES_NOT_RETURN; - } - } + impCurStmtOffsSet(opcodeOffs); + } + else if ((info.compStmtOffsetsImplicit & ICorDebugInfo::CALL_SITE_BOUNDARIES) && + impOpcodeIsCallSiteBoundary(prevOpcode)) + { + /* Make sure we have a type cached */ + assert(callTyp != TYP_COUNT); - GenTree* qmarkNull; - // - // Generate second QMARK - COLON tree - // - // qmarkNull ==> GT_QMARK - // / \. - // condNull GT_COLON - // / \. - // qmarkMT op1Copy - // - temp = new (this, GT_COLON) GenTreeColon(TYP_REF, gtClone(op1), qmarkMT); - qmarkNull = gtNewQmarkNode(TYP_REF, condNull, temp->AsColon()); - qmarkNull->gtFlags |= GTF_QMARK_CAST_INSTOF; + if (callTyp == TYP_VOID) + { + impCurStmtOffsSet(opcodeOffs); + } + else if (opts.compDbgCode) + { + impSpillStackEnsure(true); + impCurStmtOffsSet(opcodeOffs); + } + } + else if ((info.compStmtOffsetsImplicit & ICorDebugInfo::NOP_BOUNDARIES) && (prevOpcode == CEE_NOP)) + { + if (opts.compDbgCode) + { + impSpillStackEnsure(true); + } - // Make QMark node a top level node by spilling it. - unsigned tmp = lvaGrabTemp(true DEBUGARG("spilling QMark2")); - impAssignTempGen(tmp, qmarkNull, CHECK_SPILL_NONE); + impCurStmtOffsSet(opcodeOffs); + } - // TODO-CQ: Is it possible op1 has a better type? - // - // See also gtGetHelperCallClassHandle where we make the same - // determination for the helper call variants. - LclVarDsc* lclDsc = lvaGetDesc(tmp); - assert(lclDsc->lvSingleDef == 0); - lclDsc->lvSingleDef = 1; - JITDUMP("Marked V%02u as a single def temp\n", tmp); - lvaSetClass(tmp, pResolvedToken->hClass); - return gtNewLclvNode(tmp, TYP_REF); -} + assert(!impCurStmtDI.IsValid() || (nxtStmtOffs == BAD_IL_OFFSET) || + (impCurStmtDI.GetLocation().GetOffset() <= nxtStmtOffs)); + } -#ifndef DEBUG -#define assertImp(cond) ((void)0) -#else -#define assertImp(cond) \ - do \ - { \ - if (!(cond)) \ - { \ - const int cchAssertImpBuf = 600; \ - char* assertImpBuf = (char*)_alloca(cchAssertImpBuf); \ - _snprintf_s(assertImpBuf, cchAssertImpBuf, cchAssertImpBuf - 1, \ - "%s : Possibly bad IL with CEE_%s at offset %04Xh (op1=%s op2=%s stkDepth=%d)", #cond, \ - impCurOpcName, impCurOpcOffs, op1 ? varTypeName(op1->TypeGet()) : "NULL", \ - op2 ? varTypeName(op2->TypeGet()) : "NULL", verCurrentState.esStackDepth); \ - assertAbort(assertImpBuf, __FILE__, __LINE__); \ - } \ - } while (0) -#endif // DEBUG + CORINFO_CLASS_HANDLE clsHnd = DUMMY_INIT(NULL); + CORINFO_CLASS_HANDLE ldelemClsHnd = NO_CLASS_HANDLE; + CORINFO_CLASS_HANDLE stelemClsHnd = DUMMY_INIT(NULL); -//------------------------------------------------------------------------ -// impBlockIsInALoop: check if a block might be in a loop -// -// Arguments: -// block - block to check -// -// Returns: -// true if the block might be in a loop. -// -// Notes: -// Conservatively correct; may return true for some blocks that are -// not actually in loops. -// -bool Compiler::impBlockIsInALoop(BasicBlock* block) -{ - return (compIsForInlining() && ((impInlineInfo->iciBlock->bbFlags & BBF_BACKWARD_JUMP) != 0)) || - ((block->bbFlags & BBF_BACKWARD_JUMP) != 0); -} + var_types lclTyp, ovflType = TYP_UNKNOWN; + GenTree* op1 = DUMMY_INIT(NULL); + GenTree* op2 = DUMMY_INIT(NULL); + GenTree* newObjThisPtr = DUMMY_INIT(NULL); + bool uns = DUMMY_INIT(false); + bool isLocal = false; -#ifdef _PREFAST_ -#pragma warning(push) -#pragma warning(disable : 21000) // Suppress PREFast warning about overly large function -#endif -/***************************************************************************** - * Import the instr for the given basic block - */ -void Compiler::impImportBlockCode(BasicBlock* block) -{ -#define _impResolveToken(kind) impResolveToken(codeAddr, &resolvedToken, kind) + /* Get the next opcode and the size of its parameters */ -#ifdef DEBUG + OPCODE opcode = (OPCODE)getU1LittleEndian(codeAddr); + codeAddr += sizeof(__int8); - if (verbose) - { - printf("\nImporting " FMT_BB " (PC=%03u) of '%s'", block->bbNum, block->bbCodeOffs, info.compFullName); - } +#ifdef DEBUG + impCurOpcOffs = (IL_OFFSET)(codeAddr - info.compCode - 1); + JITDUMP("\n [%2u] %3u (0x%03x) ", verCurrentState.esStackDepth, impCurOpcOffs, impCurOpcOffs); #endif - unsigned nxtStmtIndex = impInitBlockLineInfo(); - IL_OFFSET nxtStmtOffs; - CorInfoHelpFunc helper; - CorInfoIsAccessAllowedResult accessAllowedResult; - CORINFO_HELPER_DESC calloutHelper; - const BYTE* lastLoadToken = nullptr; - - /* Get the tree list started */ + DECODE_OPCODE: - impBeginTreeList(); + // Return if any previous code has caused inline to fail. + if (compDonotInline()) + { + return; + } -#ifdef FEATURE_ON_STACK_REPLACEMENT + /* Get the size of additional parameters */ - bool enablePatchpoints = opts.jitFlags->IsSet(JitFlags::JIT_FLAG_TIER0) && (JitConfig.TC_OnStackReplacement() > 0); + signed int sz = opcodeSizes[opcode]; #ifdef DEBUG + clsHnd = NO_CLASS_HANDLE; + lclTyp = TYP_COUNT; + callTyp = TYP_COUNT; - // Optionally suppress patchpoints by method hash - // - static ConfigMethodRange JitEnablePatchpointRange; - JitEnablePatchpointRange.EnsureInit(JitConfig.JitEnablePatchpointRange()); - const unsigned hash = impInlineRoot()->info.compMethodHash(); - const bool inRange = JitEnablePatchpointRange.Contains(hash); - enablePatchpoints &= inRange; + impCurOpcOffs = (IL_OFFSET)(codeAddr - info.compCode - 1); + impCurOpcName = opcodeNames[opcode]; -#endif // DEBUG + if (verbose && (opcode != CEE_PREFIX1)) + { + printf("%s", impCurOpcName); + } - if (enablePatchpoints) - { - // We don't inline at Tier0, if we do, we may need rethink our approach. - // Could probably support inlines that don't introduce flow. - // - assert(!compIsForInlining()); + /* Use assertImp() to display the opcode */ - // OSR is not yet supported for methods with explicit tail calls. - // - // But we also do not have to switch these methods to be optimized, as we should be - // able to avoid getting trapped in Tier0 code by normal call counting. - // So instead, just suppress adding patchpoints. - // - if (!compTailPrefixSeen) - { - // We only need to add patchpoints if the method can loop. - // - if (compHasBackwardJump) - { - assert(compCanHavePatchpoints()); + op1 = op2 = nullptr; +#endif - // By default we use the "adaptive" strategy. - // - // This can create both source and target patchpoints within a given - // loop structure, which isn't ideal, but is not incorrect. We will - // just have some extra Tier0 overhead. - // - // Todo: implement support for mid-block patchpoints. If `block` - // is truly a backedge source (and not in a handler) then we should be - // able to find a stack empty point somewhere in the block. - // - const int patchpointStrategy = JitConfig.TC_PatchpointStrategy(); - bool addPatchpoint = false; - bool mustUseTargetPatchpoint = false; + /* See what kind of an opcode we have, then */ - switch (patchpointStrategy) - { - default: - { - // Patchpoints at backedge sources, if possible, otherwise targets. - // - addPatchpoint = ((block->bbFlags & BBF_BACKWARD_JUMP_SOURCE) == BBF_BACKWARD_JUMP_SOURCE); - mustUseTargetPatchpoint = (verCurrentState.esStackDepth != 0) || block->hasHndIndex(); - break; - } + unsigned mflags = 0; + unsigned clsFlags = 0; - case 1: - { - // Patchpoints at stackempty backedge targets. - // Note if we have loops where the IL stack is not empty on the backedge we can't patchpoint - // them. - // - // We should not have allowed OSR if there were backedges in handlers. - // - assert(!block->hasHndIndex()); - addPatchpoint = ((block->bbFlags & BBF_BACKWARD_JUMP_TARGET) == BBF_BACKWARD_JUMP_TARGET) && - (verCurrentState.esStackDepth == 0); - break; - } + switch (opcode) + { + unsigned lclNum; + var_types type; - case 2: - { - // Adaptive strategy. - // - // Patchpoints at backedge targets if there are multiple backedges, - // otherwise at backedge sources, if possible. Note a block can be both; if so we - // just need one patchpoint. - // - if ((block->bbFlags & BBF_BACKWARD_JUMP_TARGET) == BBF_BACKWARD_JUMP_TARGET) - { - // We don't know backedge count, so just use ref count. - // - addPatchpoint = (block->bbRefs > 1) && (verCurrentState.esStackDepth == 0); - } + GenTree* op3; + genTreeOps oper; + unsigned size; - if (!addPatchpoint && ((block->bbFlags & BBF_BACKWARD_JUMP_SOURCE) == BBF_BACKWARD_JUMP_SOURCE)) - { - addPatchpoint = true; - mustUseTargetPatchpoint = (verCurrentState.esStackDepth != 0) || block->hasHndIndex(); + int val; - // Also force target patchpoint if target block has multiple (backedge) preds. - // - if (!mustUseTargetPatchpoint) - { - for (BasicBlock* const succBlock : block->Succs(this)) - { - if ((succBlock->bbNum <= block->bbNum) && (succBlock->bbRefs > 1)) - { - mustUseTargetPatchpoint = true; - break; - } - } - } - } - break; - } - } + CORINFO_SIG_INFO sig; + IL_OFFSET jmpAddr; + bool ovfl, unordered, callNode; + CORINFO_CLASS_HANDLE tokenType; - if (addPatchpoint) - { - if (mustUseTargetPatchpoint) - { - // We wanted a source patchpoint, but could not have one. - // So, add patchpoints to the backedge targets. - // - for (BasicBlock* const succBlock : block->Succs(this)) - { - if (succBlock->bbNum <= block->bbNum) - { - // The succBlock had better agree it's a target. - // - assert((succBlock->bbFlags & BBF_BACKWARD_JUMP_TARGET) == BBF_BACKWARD_JUMP_TARGET); + union { + int intVal; + float fltVal; + __int64 lngVal; + double dblVal; + } cval; - // We may already have decided to put a patchpoint in succBlock. If not, add one. - // - if ((succBlock->bbFlags & BBF_PATCHPOINT) != 0) - { - // In some cases the target may not be stack-empty at entry. - // If so, we will bypass patchpoints for this backedge. - // - if (succBlock->bbStackDepthOnEntry() > 0) - { - JITDUMP("\nCan't set source patchpoint at " FMT_BB ", can't use target " FMT_BB - " as it has non-empty stack on entry.\n", - block->bbNum, succBlock->bbNum); - } - else - { - JITDUMP("\nCan't set source patchpoint at " FMT_BB ", using target " FMT_BB - " instead\n", - block->bbNum, succBlock->bbNum); + case CEE_PREFIX1: + opcode = (OPCODE)(getU1LittleEndian(codeAddr) + 256); + opcodeOffs = (IL_OFFSET)(codeAddr - info.compCode); + codeAddr += sizeof(__int8); + goto DECODE_OPCODE; - assert(!succBlock->hasHndIndex()); - succBlock->bbFlags |= BBF_PATCHPOINT; - } - } + SPILL_APPEND: + + // We need to call impSpillLclRefs() for a struct type lclVar. + // This is because there may be loads of that lclVar on the evaluation stack, and + // we need to ensure that those loads are completed before we modify it. + if ((op1->OperGet() == GT_ASG) && varTypeIsStruct(op1->gtGetOp1())) + { + GenTree* lhs = op1->gtGetOp1(); + GenTreeLclVarCommon* lclVar = nullptr; + if (lhs->gtOper == GT_LCL_VAR) + { + lclVar = lhs->AsLclVarCommon(); + } + else if (lhs->OperIsBlk()) + { + // Check if LHS address is within some struct local, to catch + // cases where we're updating the struct by something other than a stfld + GenTree* addr = lhs->AsBlk()->Addr(); + + // Catches ADDR(LCL_VAR), or ADD(ADDR(LCL_VAR),CNS_INT)) + lclVar = addr->IsLocalAddrExpr(); + + // Catches ADDR(FIELD(... ADDR(LCL_VAR))) + if (lclVar == nullptr) + { + GenTree* lclTree = nullptr; + if (impIsAddressInLocal(addr, &lclTree)) + { + lclVar = lclTree->AsLclVarCommon(); } } } - else + if (lclVar != nullptr) { - assert(!block->hasHndIndex()); - block->bbFlags |= BBF_PATCHPOINT; + impSpillLclRefs(lclVar->GetLclNum()); } - - setMethodHasPatchpoint(); } - } - else - { - // Should not see backward branch targets w/o backwards branches. - // So if !compHasBackwardsBranch, these flags should never be set. - // - assert((block->bbFlags & (BBF_BACKWARD_JUMP_TARGET | BBF_BACKWARD_JUMP_SOURCE)) == 0); - } - } + + /* Append 'op1' to the list of statements */ + impAppendTree(op1, (unsigned)CHECK_SPILL_ALL, impCurStmtDI); + goto DONE_APPEND; + + APPEND: + + /* Append 'op1' to the list of statements */ + + impAppendTree(op1, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); + goto DONE_APPEND; + + DONE_APPEND: #ifdef DEBUG - // As a stress test, we can place patchpoints at the start of any block - // that is a stack empty point and is not within a handler. - // - // Todo: enable for mid-block stack empty points too. - // - const int offsetOSR = JitConfig.JitOffsetOnStackReplacement(); - const int randomOSR = JitConfig.JitRandomOnStackReplacement(); - const bool tryOffsetOSR = offsetOSR >= 0; - const bool tryRandomOSR = randomOSR > 0; + // Remember at which BC offset the tree was finished + impNoteLastILoffs(); +#endif + break; - if (compCanHavePatchpoints() && (tryOffsetOSR || tryRandomOSR) && (verCurrentState.esStackDepth == 0) && - !block->hasHndIndex() && ((block->bbFlags & BBF_PATCHPOINT) == 0)) - { - // Block start can have a patchpoint. See if we should add one. - // - bool addPatchpoint = false; + case CEE_LDNULL: + impPushNullObjRefOnStack(); + break; - // Specific offset? - // - if (tryOffsetOSR) - { - if (impCurOpcOffs == (unsigned)offsetOSR) - { - addPatchpoint = true; - } - } - // Random? - // - else - { - // Reuse the random inliner's random state. - // Note m_inlineStrategy is always created, even if we're not inlining. - // - CLRRandom* const random = impInlineRoot()->m_inlineStrategy->GetRandom(randomOSR); - const int randomValue = (int)random->Next(100); + case CEE_LDC_I4_M1: + case CEE_LDC_I4_0: + case CEE_LDC_I4_1: + case CEE_LDC_I4_2: + case CEE_LDC_I4_3: + case CEE_LDC_I4_4: + case CEE_LDC_I4_5: + case CEE_LDC_I4_6: + case CEE_LDC_I4_7: + case CEE_LDC_I4_8: + cval.intVal = (opcode - CEE_LDC_I4_0); + assert(-1 <= cval.intVal && cval.intVal <= 8); + goto PUSH_I4CON; - addPatchpoint = (randomValue < randomOSR); - } + case CEE_LDC_I4_S: + cval.intVal = getI1LittleEndian(codeAddr); + goto PUSH_I4CON; + case CEE_LDC_I4: + cval.intVal = getI4LittleEndian(codeAddr); + goto PUSH_I4CON; + PUSH_I4CON: + JITDUMP(" %d", cval.intVal); + impPushOnStack(gtNewIconNode(cval.intVal), typeInfo(TI_INT)); + break; - if (addPatchpoint) - { - block->bbFlags |= BBF_PATCHPOINT; - setMethodHasPatchpoint(); - } + case CEE_LDC_I8: + cval.lngVal = getI8LittleEndian(codeAddr); + JITDUMP(" 0x%016llx", cval.lngVal); + impPushOnStack(gtNewLconNode(cval.lngVal), typeInfo(TI_LONG)); + break; - JITDUMP("\n** %s patchpoint%s added to " FMT_BB " (il offset %u)\n", tryOffsetOSR ? "offset" : "random", - addPatchpoint ? "" : " not", block->bbNum, impCurOpcOffs); - } + case CEE_LDC_R8: + cval.dblVal = getR8LittleEndian(codeAddr); + JITDUMP(" %#.17g", cval.dblVal); + impPushOnStack(gtNewDconNode(cval.dblVal), typeInfo(TI_DOUBLE)); + break; -#endif // DEBUG - } + case CEE_LDC_R4: + cval.dblVal = getR4LittleEndian(codeAddr); + JITDUMP(" %#.17g", cval.dblVal); + impPushOnStack(gtNewDconNode(cval.dblVal, TYP_FLOAT), typeInfo(TI_DOUBLE)); + break; - // Mark stack-empty rare blocks to be considered for partial compilation. - // - // Ideally these are conditionally executed blocks -- if the method is going - // to unconditionally throw, there's not as much to be gained by deferring jitting. - // For now, we just screen out the entry bb. - // - // In general we might want track all the IL stack empty points so we can - // propagate rareness back through flow and place the partial compilation patchpoints "earlier" - // so there are fewer overall. - // - // Note unlike OSR, it's ok to forgo these. - // - // Todo: stress mode... - // - if (opts.jitFlags->IsSet(JitFlags::JIT_FLAG_TIER0) && (JitConfig.TC_PartialCompilation() > 0) && - compCanHavePatchpoints() && !compTailPrefixSeen) - { - // Is this block a good place for partial compilation? - // - if ((block != fgFirstBB) && block->isRunRarely() && (verCurrentState.esStackDepth == 0) && - ((block->bbFlags & BBF_PATCHPOINT) == 0) && !block->hasHndIndex()) - { - JITDUMP("\nBlock " FMT_BB " will be a partial compilation patchpoint -- not importing\n", block->bbNum); - block->bbFlags |= BBF_PARTIAL_COMPILATION_PATCHPOINT; - setMethodHasPartialCompilationPatchpoint(); + case CEE_LDSTR: + val = getU4LittleEndian(codeAddr); + JITDUMP(" %08X", val); + impPushOnStack(gtNewSconNode(val, info.compScopeHnd), tiRetVal); + break; - // Change block to BBJ_THROW so we won't trigger importation of successors. - // - block->bbJumpKind = BBJ_THROW; + case CEE_LDARG: + lclNum = getU2LittleEndian(codeAddr); + JITDUMP(" %u", lclNum); + impLoadArg(lclNum, opcodeOffs + sz + 1); + break; - // If this method has a explicit generic context, the only uses of it may be in - // the IL for this block. So assume it's used. - // - if (info.compMethodInfo->options & - (CORINFO_GENERICS_CTXT_FROM_METHODDESC | CORINFO_GENERICS_CTXT_FROM_METHODTABLE)) - { - lvaGenericsContextInUse = true; - } + case CEE_LDARG_S: + lclNum = getU1LittleEndian(codeAddr); + JITDUMP(" %u", lclNum); + impLoadArg(lclNum, opcodeOffs + sz + 1); + break; - return; - } - } + case CEE_LDARG_0: + case CEE_LDARG_1: + case CEE_LDARG_2: + case CEE_LDARG_3: + lclNum = (opcode - CEE_LDARG_0); + assert(lclNum >= 0 && lclNum < 4); + impLoadArg(lclNum, opcodeOffs + sz + 1); + break; -#endif // FEATURE_ON_STACK_REPLACEMENT + case CEE_LDLOC: + lclNum = getU2LittleEndian(codeAddr); + JITDUMP(" %u", lclNum); + impLoadLoc(lclNum, opcodeOffs + sz + 1); + break; - /* Walk the opcodes that comprise the basic block */ + case CEE_LDLOC_S: + lclNum = getU1LittleEndian(codeAddr); + JITDUMP(" %u", lclNum); + impLoadLoc(lclNum, opcodeOffs + sz + 1); + break; - const BYTE* codeAddr = info.compCode + block->bbCodeOffs; - const BYTE* codeEndp = info.compCode + block->bbCodeOffsEnd; + case CEE_LDLOC_0: + case CEE_LDLOC_1: + case CEE_LDLOC_2: + case CEE_LDLOC_3: + lclNum = (opcode - CEE_LDLOC_0); + assert(lclNum >= 0 && lclNum < 4); + impLoadLoc(lclNum, opcodeOffs + sz + 1); + break; - IL_OFFSET opcodeOffs = block->bbCodeOffs; - IL_OFFSET lastSpillOffs = opcodeOffs; + case CEE_STARG: + lclNum = getU2LittleEndian(codeAddr); + goto STARG; - signed jmpDist; + case CEE_STARG_S: + lclNum = getU1LittleEndian(codeAddr); + STARG: + JITDUMP(" %u", lclNum); - /* remember the start of the delegate creation sequence (used for verification) */ - const BYTE* delegateCreateStart = nullptr; + if (compIsForInlining()) + { + op1 = impInlineFetchArg(lclNum, impInlineInfo->inlArgInfo, impInlineInfo->lclVarInfo); + noway_assert(op1->gtOper == GT_LCL_VAR); + lclNum = op1->AsLclVar()->GetLclNum(); - int prefixFlags = 0; - bool explicitTailCall, constraintCall, readonlyCall; + goto VAR_ST_VALID; + } - typeInfo tiRetVal; + lclNum = compMapILargNum(lclNum); // account for possible hidden param + assertImp(lclNum < numArgs); - unsigned numArgs = info.compArgsCount; + if (lclNum == info.compThisArg) + { + lclNum = lvaArg0Var; + } - /* Now process all the opcodes in the block */ + // We should have seen this arg write in the prescan + assert(lvaTable[lclNum].lvHasILStoreOp); - var_types callTyp = TYP_COUNT; - OPCODE prevOpcode = CEE_ILLEGAL; + goto VAR_ST; - if (block->bbCatchTyp) - { - if (info.compStmtOffsetsImplicit & ICorDebugInfo::CALL_SITE_BOUNDARIES) - { - impCurStmtOffsSet(block->bbCodeOffs); - } + case CEE_STLOC: + lclNum = getU2LittleEndian(codeAddr); + isLocal = true; + JITDUMP(" %u", lclNum); + goto LOC_ST; - // We will spill the GT_CATCH_ARG and the input of the BB_QMARK block - // to a temp. This is a trade off for code simplicity - impSpillSpecialSideEff(); - } + case CEE_STLOC_S: + lclNum = getU1LittleEndian(codeAddr); + isLocal = true; + JITDUMP(" %u", lclNum); + goto LOC_ST; - while (codeAddr < codeEndp) - { -#ifdef FEATURE_READYTORUN - bool usingReadyToRunHelper = false; -#endif - CORINFO_RESOLVED_TOKEN resolvedToken; - CORINFO_RESOLVED_TOKEN constrainedResolvedToken = {}; - CORINFO_CALL_INFO callInfo; - CORINFO_FIELD_INFO fieldInfo; + case CEE_STLOC_0: + case CEE_STLOC_1: + case CEE_STLOC_2: + case CEE_STLOC_3: + isLocal = true; + lclNum = (opcode - CEE_STLOC_0); + assert(lclNum >= 0 && lclNum < 4); - tiRetVal = typeInfo(); // Default type info + LOC_ST: + if (compIsForInlining()) + { + lclTyp = impInlineInfo->lclVarInfo[lclNum + impInlineInfo->argCnt].lclTypeInfo; - //--------------------------------------------------------------------- + /* Have we allocated a temp for this local? */ - /* We need to restrict the max tree depth as many of the Compiler - functions are recursive. We do this by spilling the stack */ + lclNum = impInlineFetchLocal(lclNum DEBUGARG("Inline stloc first use temp")); - if (verCurrentState.esStackDepth) - { - /* Has it been a while since we last saw a non-empty stack (which - guarantees that the tree depth isnt accumulating. */ + goto _PopValue; + } - if ((opcodeOffs - lastSpillOffs) > MAX_TREE_SIZE && impCanSpillNow(prevOpcode)) - { - impSpillStackEnsure(); - lastSpillOffs = opcodeOffs; - } - } - else - { - lastSpillOffs = opcodeOffs; - impBoxTempInUse = false; // nothing on the stack, box temp OK to use again - } + lclNum += numArgs; - /* Compute the current instr offset */ + VAR_ST: - opcodeOffs = (IL_OFFSET)(codeAddr - info.compCode); + if (lclNum >= info.compLocalsCount && lclNum != lvaArg0Var) + { + BADCODE("Bad IL"); + } -#ifndef DEBUG - if (opts.compDbgInfo) -#endif - { - nxtStmtOffs = - (nxtStmtIndex < info.compStmtOffsetsCount) ? info.compStmtOffsets[nxtStmtIndex] : BAD_IL_OFFSET; + VAR_ST_VALID: - /* Have we reached the next stmt boundary ? */ + /* if it is a struct assignment, make certain we don't overflow the buffer */ + assert(lclTyp != TYP_STRUCT || lvaLclSize(lclNum) >= info.compCompHnd->getClassSize(clsHnd)); - if (nxtStmtOffs != BAD_IL_OFFSET && opcodeOffs >= nxtStmtOffs) - { - assert(nxtStmtOffs == info.compStmtOffsets[nxtStmtIndex]); + if (lvaTable[lclNum].lvNormalizeOnLoad()) + { + lclTyp = lvaGetRealType(lclNum); + } + else + { + lclTyp = lvaGetActualType(lclNum); + } + + _PopValue: + /* Pop the value being assigned */ - if (verCurrentState.esStackDepth != 0 && opts.compDbgCode) { - /* We need to provide accurate IP-mapping at this point. - So spill anything on the stack so that it will form - gtStmts with the correct stmt offset noted */ + StackEntry se = impPopStack(); + clsHnd = se.seTypeInfo.GetClassHandle(); + op1 = se.val; + tiRetVal = se.seTypeInfo; + } - impSpillStackEnsure(true); +#ifdef FEATURE_SIMD + if (varTypeIsSIMD(lclTyp) && (lclTyp != op1->TypeGet())) + { + assert(op1->TypeGet() == TYP_STRUCT); + op1->gtType = lclTyp; } +#endif // FEATURE_SIMD - // Have we reported debug info for any tree? + op1 = impImplicitIorI4Cast(op1, lclTyp); - if (impCurStmtDI.IsValid() && opts.compDbgCode) +#ifdef TARGET_64BIT + // Downcast the TYP_I_IMPL into a 32-bit Int for x86 JIT compatiblity + if (varTypeIsI(op1->TypeGet()) && (genActualType(lclTyp) == TYP_INT)) { - GenTree* placeHolder = new (this, GT_NO_OP) GenTree(GT_NO_OP, TYP_VOID); - impAppendTree(placeHolder, CHECK_SPILL_NONE, impCurStmtDI); - - assert(!impCurStmtDI.IsValid()); + op1 = gtNewCastNode(TYP_INT, op1, false, TYP_INT); } +#endif // TARGET_64BIT - if (!impCurStmtDI.IsValid()) + // We had better assign it a value of the correct type + assertImp( + genActualType(lclTyp) == genActualType(op1->gtType) || + (genActualType(lclTyp) == TYP_I_IMPL && op1->IsLocalAddrExpr() != nullptr) || + (genActualType(lclTyp) == TYP_I_IMPL && (op1->gtType == TYP_BYREF || op1->gtType == TYP_REF)) || + (genActualType(op1->gtType) == TYP_I_IMPL && lclTyp == TYP_BYREF) || + (varTypeIsFloating(lclTyp) && varTypeIsFloating(op1->TypeGet())) || + ((genActualType(lclTyp) == TYP_BYREF) && genActualType(op1->TypeGet()) == TYP_REF)); + + /* If op1 is "&var" then its type is the transient "*" and it can + be used either as TYP_BYREF or TYP_I_IMPL */ + + if (op1->IsLocalAddrExpr() != nullptr) { - /* Make sure that nxtStmtIndex is in sync with opcodeOffs. - If opcodeOffs has gone past nxtStmtIndex, catch up */ + assertImp(genActualType(lclTyp) == TYP_I_IMPL || lclTyp == TYP_BYREF); - while ((nxtStmtIndex + 1) < info.compStmtOffsetsCount && - info.compStmtOffsets[nxtStmtIndex + 1] <= opcodeOffs) + /* When "&var" is created, we assume it is a byref. If it is + being assigned to a TYP_I_IMPL var, change the type to + prevent unnecessary GC info */ + + if (genActualType(lclTyp) == TYP_I_IMPL) { - nxtStmtIndex++; + op1->gtType = TYP_I_IMPL; } + } - /* Go to the new stmt */ + // If this is a local and the local is a ref type, see + // if we can improve type information based on the + // value being assigned. + if (isLocal && (lclTyp == TYP_REF)) + { + // We should have seen a stloc in our IL prescan. + assert(lvaTable[lclNum].lvHasILStoreOp); - impCurStmtOffsSet(info.compStmtOffsets[nxtStmtIndex]); + // Is there just one place this local is defined? + const bool isSingleDefLocal = lvaTable[lclNum].lvSingleDef; - /* Update the stmt boundary index */ + // Conservative check that there is just one + // definition that reaches this store. + const bool hasSingleReachingDef = (block->bbStackDepthOnEntry() == 0); - nxtStmtIndex++; - assert(nxtStmtIndex <= info.compStmtOffsetsCount); + if (isSingleDefLocal && hasSingleReachingDef) + { + lvaUpdateClass(lclNum, op1, clsHnd); + } + } - /* Are there any more line# entries after this one? */ + /* Filter out simple assignments to itself */ - if (nxtStmtIndex < info.compStmtOffsetsCount) + if (op1->gtOper == GT_LCL_VAR && lclNum == op1->AsLclVarCommon()->GetLclNum()) + { + if (opts.compDbgCode) { - /* Remember where the next line# starts */ - - nxtStmtOffs = info.compStmtOffsets[nxtStmtIndex]; + op1 = gtNewNothingNode(); + goto SPILL_APPEND; } else { - /* No more line# entries */ - - nxtStmtOffs = BAD_IL_OFFSET; + break; } } - } - else if ((info.compStmtOffsetsImplicit & ICorDebugInfo::STACK_EMPTY_BOUNDARIES) && - (verCurrentState.esStackDepth == 0)) - { - /* At stack-empty locations, we have already added the tree to - the stmt list with the last offset. We just need to update - impCurStmtDI - */ - impCurStmtOffsSet(opcodeOffs); - } - else if ((info.compStmtOffsetsImplicit & ICorDebugInfo::CALL_SITE_BOUNDARIES) && - impOpcodeIsCallSiteBoundary(prevOpcode)) - { - /* Make sure we have a type cached */ - assert(callTyp != TYP_COUNT); + /* Create the assignment node */ - if (callTyp == TYP_VOID) + op2 = gtNewLclvNode(lclNum, lclTyp DEBUGARG(opcodeOffs + sz + 1)); + + /* If the local is aliased or pinned, we need to spill calls and + indirections from the stack. */ + + if ((lvaTable[lclNum].IsAddressExposed() || lvaTable[lclNum].lvHasLdAddrOp || + lvaTable[lclNum].lvPinned) && + (verCurrentState.esStackDepth > 0)) { - impCurStmtOffsSet(opcodeOffs); + impSpillSideEffects(false, + (unsigned)CHECK_SPILL_ALL DEBUGARG("Local could be aliased or is pinned")); } - else if (opts.compDbgCode) + + /* Spill any refs to the local from the stack */ + + impSpillLclRefs(lclNum); + + // We can generate an assignment to a TYP_FLOAT from a TYP_DOUBLE + // We insert a cast to the dest 'op2' type + // + if ((op1->TypeGet() != op2->TypeGet()) && varTypeIsFloating(op1->gtType) && + varTypeIsFloating(op2->gtType)) { - impSpillStackEnsure(true); - impCurStmtOffsSet(opcodeOffs); + op1 = gtNewCastNode(op2->TypeGet(), op1, false, op2->TypeGet()); } - } - else if ((info.compStmtOffsetsImplicit & ICorDebugInfo::NOP_BOUNDARIES) && (prevOpcode == CEE_NOP)) - { - if (opts.compDbgCode) + + if (varTypeIsStruct(lclTyp)) { - impSpillStackEnsure(true); + op1 = impAssignStruct(op2, op1, clsHnd, (unsigned)CHECK_SPILL_ALL); + } + else + { + op1 = gtNewAssignNode(op2, op1); } - impCurStmtOffsSet(opcodeOffs); - } + goto SPILL_APPEND; - assert(!impCurStmtDI.IsValid() || (nxtStmtOffs == BAD_IL_OFFSET) || - (impCurStmtDI.GetLocation().GetOffset() <= nxtStmtOffs)); - } + case CEE_LDLOCA: + lclNum = getU2LittleEndian(codeAddr); + goto LDLOCA; - CORINFO_CLASS_HANDLE clsHnd = DUMMY_INIT(NULL); - CORINFO_CLASS_HANDLE ldelemClsHnd = NO_CLASS_HANDLE; - CORINFO_CLASS_HANDLE stelemClsHnd = DUMMY_INIT(NULL); + case CEE_LDLOCA_S: + lclNum = getU1LittleEndian(codeAddr); + LDLOCA: + JITDUMP(" %u", lclNum); + + if (compIsForInlining()) + { + // Get the local type + lclTyp = impInlineInfo->lclVarInfo[lclNum + impInlineInfo->argCnt].lclTypeInfo; + + /* Have we allocated a temp for this local? */ + + lclNum = impInlineFetchLocal(lclNum DEBUGARG("Inline ldloca(s) first use temp")); + + assert(!lvaGetDesc(lclNum)->lvNormalizeOnLoad()); + op1 = gtNewLclvNode(lclNum, lvaGetActualType(lclNum)); + + goto _PUSH_ADRVAR; + } + + lclNum += numArgs; + assertImp(lclNum < info.compLocalsCount); + goto ADRVAR; + + case CEE_LDARGA: + lclNum = getU2LittleEndian(codeAddr); + goto LDARGA; - var_types lclTyp, ovflType = TYP_UNKNOWN; - GenTree* op1 = DUMMY_INIT(NULL); - GenTree* op2 = DUMMY_INIT(NULL); - GenTree* newObjThisPtr = DUMMY_INIT(NULL); - bool uns = DUMMY_INIT(false); - bool isLocal = false; + case CEE_LDARGA_S: + lclNum = getU1LittleEndian(codeAddr); + LDARGA: + JITDUMP(" %u", lclNum); + Verify(lclNum < info.compILargsCount, "bad arg num"); - /* Get the next opcode and the size of its parameters */ + if (compIsForInlining()) + { + // In IL, LDARGA(_S) is used to load the byref managed pointer of struct argument, + // followed by a ldfld to load the field. - OPCODE opcode = (OPCODE)getU1LittleEndian(codeAddr); - codeAddr += sizeof(__int8); + op1 = impInlineFetchArg(lclNum, impInlineInfo->inlArgInfo, impInlineInfo->lclVarInfo); + if (op1->gtOper != GT_LCL_VAR) + { + compInlineResult->NoteFatal(InlineObservation::CALLSITE_LDARGA_NOT_LOCAL_VAR); + return; + } -#ifdef DEBUG - impCurOpcOffs = (IL_OFFSET)(codeAddr - info.compCode - 1); - JITDUMP("\n [%2u] %3u (0x%03x) ", verCurrentState.esStackDepth, impCurOpcOffs, impCurOpcOffs); -#endif + assert(op1->gtOper == GT_LCL_VAR); - DECODE_OPCODE: + goto _PUSH_ADRVAR; + } - // Return if any previous code has caused inline to fail. - if (compDonotInline()) - { - return; - } + lclNum = compMapILargNum(lclNum); // account for possible hidden param + assertImp(lclNum < numArgs); - /* Get the size of additional parameters */ + if (lclNum == info.compThisArg) + { + lclNum = lvaArg0Var; + } - signed int sz = opcodeSizes[opcode]; + goto ADRVAR; -#ifdef DEBUG - clsHnd = NO_CLASS_HANDLE; - lclTyp = TYP_COUNT; - callTyp = TYP_COUNT; + ADRVAR: - impCurOpcOffs = (IL_OFFSET)(codeAddr - info.compCode - 1); - impCurOpcName = opcodeNames[opcode]; + op1 = impCreateLocalNode(lclNum DEBUGARG(opcodeOffs + sz + 1)); - if (verbose && (opcode != CEE_PREFIX1)) - { - printf("%s", impCurOpcName); - } + _PUSH_ADRVAR: + assert(op1->gtOper == GT_LCL_VAR); - /* Use assertImp() to display the opcode */ + /* Note that this is supposed to create the transient type "*" + which may be used as a TYP_I_IMPL. However we catch places + where it is used as a TYP_I_IMPL and change the node if needed. + Thus we are pessimistic and may report byrefs in the GC info + where it was not absolutely needed, but it is safer this way. + */ + op1 = gtNewOperNode(GT_ADDR, TYP_BYREF, op1); - op1 = op2 = nullptr; -#endif + // &aliasedVar doesnt need GTF_GLOB_REF, though alisasedVar does + assert((op1->gtFlags & GTF_GLOB_REF) == 0); - /* See what kind of an opcode we have, then */ + tiRetVal = lvaTable[lclNum].lvVerTypeInfo; - unsigned mflags = 0; - unsigned clsFlags = 0; + impPushOnStack(op1, tiRetVal); + break; - switch (opcode) - { - unsigned lclNum; - var_types type; + case CEE_ARGLIST: - GenTree* op3; - genTreeOps oper; - unsigned size; + if (!info.compIsVarArgs) + { + BADCODE("arglist in non-vararg method"); + } - int val; + assertImp((info.compMethodInfo->args.callConv & CORINFO_CALLCONV_MASK) == CORINFO_CALLCONV_VARARG); - CORINFO_SIG_INFO sig; - IL_OFFSET jmpAddr; - bool ovfl, unordered, callNode; - CORINFO_CLASS_HANDLE tokenType; + /* The ARGLIST cookie is a hidden 'last' parameter, we have already + adjusted the arg count cos this is like fetching the last param */ + assertImp(0 < numArgs); + lclNum = lvaVarargsHandleArg; + op1 = gtNewLclvNode(lclNum, TYP_I_IMPL DEBUGARG(opcodeOffs + sz + 1)); + op1 = gtNewOperNode(GT_ADDR, TYP_BYREF, op1); + impPushOnStack(op1, tiRetVal); + break; - union { - int intVal; - float fltVal; - __int64 lngVal; - double dblVal; - } cval; + case CEE_ENDFINALLY: - case CEE_PREFIX1: - opcode = (OPCODE)(getU1LittleEndian(codeAddr) + 256); - opcodeOffs = (IL_OFFSET)(codeAddr - info.compCode); - codeAddr += sizeof(__int8); - goto DECODE_OPCODE; + if (compIsForInlining()) + { + assert(!"Shouldn't have exception handlers in the inliner!"); + compInlineResult->NoteFatal(InlineObservation::CALLEE_HAS_ENDFINALLY); + return; + } - SPILL_APPEND: - impAppendTree(op1, CHECK_SPILL_ALL, impCurStmtDI); - goto DONE_APPEND; + if (verCurrentState.esStackDepth > 0) + { + impEvalSideEffects(); + } - APPEND: - impAppendTree(op1, CHECK_SPILL_NONE, impCurStmtDI); - goto DONE_APPEND; + if (info.compXcptnsCount == 0) + { + BADCODE("endfinally outside finally"); + } - DONE_APPEND: -#ifdef DEBUG - // Remember at which BC offset the tree was finished - impNoteLastILoffs(); -#endif - break; + assert(verCurrentState.esStackDepth == 0); - case CEE_LDNULL: - impPushNullObjRefOnStack(); - break; + op1 = gtNewOperNode(GT_RETFILT, TYP_VOID, nullptr); + goto APPEND; - case CEE_LDC_I4_M1: - case CEE_LDC_I4_0: - case CEE_LDC_I4_1: - case CEE_LDC_I4_2: - case CEE_LDC_I4_3: - case CEE_LDC_I4_4: - case CEE_LDC_I4_5: - case CEE_LDC_I4_6: - case CEE_LDC_I4_7: - case CEE_LDC_I4_8: - cval.intVal = (opcode - CEE_LDC_I4_0); - assert(-1 <= cval.intVal && cval.intVal <= 8); - goto PUSH_I4CON; + case CEE_ENDFILTER: - case CEE_LDC_I4_S: - cval.intVal = getI1LittleEndian(codeAddr); - goto PUSH_I4CON; - case CEE_LDC_I4: - cval.intVal = getI4LittleEndian(codeAddr); - goto PUSH_I4CON; - PUSH_I4CON: - JITDUMP(" %d", cval.intVal); - impPushOnStack(gtNewIconNode(cval.intVal), typeInfo(TI_INT)); - break; + if (compIsForInlining()) + { + assert(!"Shouldn't have exception handlers in the inliner!"); + compInlineResult->NoteFatal(InlineObservation::CALLEE_HAS_ENDFILTER); + return; + } - case CEE_LDC_I8: - cval.lngVal = getI8LittleEndian(codeAddr); - JITDUMP(" 0x%016llx", cval.lngVal); - impPushOnStack(gtNewLconNode(cval.lngVal), typeInfo(TI_LONG)); - break; + block->bbSetRunRarely(); // filters are rare - case CEE_LDC_R8: - cval.dblVal = getR8LittleEndian(codeAddr); - JITDUMP(" %#.17g", cval.dblVal); - impPushOnStack(gtNewDconNode(cval.dblVal), typeInfo(TI_DOUBLE)); - break; + if (info.compXcptnsCount == 0) + { + BADCODE("endfilter outside filter"); + } - case CEE_LDC_R4: - cval.dblVal = getR4LittleEndian(codeAddr); - JITDUMP(" %#.17g", cval.dblVal); - impPushOnStack(gtNewDconNode(cval.dblVal, TYP_FLOAT), typeInfo(TI_DOUBLE)); - break; + op1 = impPopStack().val; + assertImp(op1->gtType == TYP_INT); + if (!bbInFilterILRange(block)) + { + BADCODE("EndFilter outside a filter handler"); + } - case CEE_LDSTR: - val = getU4LittleEndian(codeAddr); - JITDUMP(" %08X", val); - impPushOnStack(gtNewSconNode(val, info.compScopeHnd), tiRetVal); - break; + /* Mark current bb as end of filter */ - case CEE_LDARG: - lclNum = getU2LittleEndian(codeAddr); - JITDUMP(" %u", lclNum); - impLoadArg(lclNum, opcodeOffs + sz + 1); - break; + assert(compCurBB->bbFlags & BBF_DONT_REMOVE); + assert(compCurBB->bbJumpKind == BBJ_EHFILTERRET); - case CEE_LDARG_S: - lclNum = getU1LittleEndian(codeAddr); - JITDUMP(" %u", lclNum); - impLoadArg(lclNum, opcodeOffs + sz + 1); - break; + /* Mark catch handler as successor */ - case CEE_LDARG_0: - case CEE_LDARG_1: - case CEE_LDARG_2: - case CEE_LDARG_3: - lclNum = (opcode - CEE_LDARG_0); - assert(lclNum >= 0 && lclNum < 4); - impLoadArg(lclNum, opcodeOffs + sz + 1); - break; + op1 = gtNewOperNode(GT_RETFILT, op1->TypeGet(), op1); + if (verCurrentState.esStackDepth != 0) + { + verRaiseVerifyException(INDEBUG("stack must be 1 on end of filter") DEBUGARG(__FILE__) + DEBUGARG(__LINE__)); + } + goto APPEND; - case CEE_LDLOC: - lclNum = getU2LittleEndian(codeAddr); - JITDUMP(" %u", lclNum); - impLoadLoc(lclNum, opcodeOffs + sz + 1); - break; + case CEE_RET: + prefixFlags &= ~PREFIX_TAILCALL; // ret without call before it + RET: + if (!impReturnInstruction(prefixFlags, opcode)) + { + return; // abort + } + else + { + break; + } - case CEE_LDLOC_S: - lclNum = getU1LittleEndian(codeAddr); - JITDUMP(" %u", lclNum); - impLoadLoc(lclNum, opcodeOffs + sz + 1); - break; + case CEE_JMP: - case CEE_LDLOC_0: - case CEE_LDLOC_1: - case CEE_LDLOC_2: - case CEE_LDLOC_3: - lclNum = (opcode - CEE_LDLOC_0); - assert(lclNum >= 0 && lclNum < 4); - impLoadLoc(lclNum, opcodeOffs + sz + 1); - break; + assert(!compIsForInlining()); - case CEE_STARG: - lclNum = getU2LittleEndian(codeAddr); - goto STARG; + if ((info.compFlags & CORINFO_FLG_SYNCH) || block->hasTryIndex() || block->hasHndIndex()) + { + /* CEE_JMP does not make sense in some "protected" regions. */ - case CEE_STARG_S: - lclNum = getU1LittleEndian(codeAddr); - STARG: - JITDUMP(" %u", lclNum); + BADCODE("Jmp not allowed in protected region"); + } - if (compIsForInlining()) + if (opts.IsReversePInvoke()) { - op1 = impInlineFetchArg(lclNum, impInlineInfo->inlArgInfo, impInlineInfo->lclVarInfo); - noway_assert(op1->gtOper == GT_LCL_VAR); - lclNum = op1->AsLclVar()->GetLclNum(); + BADCODE("Jmp not allowed in reverse P/Invoke"); + } - goto VAR_ST_VALID; + if (verCurrentState.esStackDepth != 0) + { + BADCODE("Stack must be empty after CEE_JMPs"); } - lclNum = compMapILargNum(lclNum); // account for possible hidden param - assertImp(lclNum < numArgs); + _impResolveToken(CORINFO_TOKENKIND_Method); - if (lclNum == info.compThisArg) + JITDUMP(" %08X", resolvedToken.token); + + /* The signature of the target has to be identical to ours. + At least check that argCnt and returnType match */ + + eeGetMethodSig(resolvedToken.hMethod, &sig); + if (sig.numArgs != info.compMethodInfo->args.numArgs || + sig.retType != info.compMethodInfo->args.retType || + sig.callConv != info.compMethodInfo->args.callConv) { - lclNum = lvaArg0Var; + BADCODE("Incompatible target for CEE_JMPs"); } - // We should have seen this arg write in the prescan - assert(lvaTable[lclNum].lvHasILStoreOp); + op1 = new (this, GT_JMP) GenTreeVal(GT_JMP, TYP_VOID, (size_t)resolvedToken.hMethod); - goto VAR_ST; + /* Mark the basic block as being a JUMP instead of RETURN */ - case CEE_STLOC: - lclNum = getU2LittleEndian(codeAddr); - isLocal = true; - JITDUMP(" %u", lclNum); - goto LOC_ST; + block->bbFlags |= BBF_HAS_JMP; - case CEE_STLOC_S: - lclNum = getU1LittleEndian(codeAddr); - isLocal = true; - JITDUMP(" %u", lclNum); - goto LOC_ST; + /* Set this flag to make sure register arguments have a location assigned + * even if we don't use them inside the method */ - case CEE_STLOC_0: - case CEE_STLOC_1: - case CEE_STLOC_2: - case CEE_STLOC_3: - isLocal = true; - lclNum = (opcode - CEE_STLOC_0); - assert(lclNum >= 0 && lclNum < 4); + compJmpOpUsed = true; - LOC_ST: - if (compIsForInlining()) - { - lclTyp = impInlineInfo->lclVarInfo[lclNum + impInlineInfo->argCnt].lclTypeInfo; + fgNoStructPromotion = true; - /* Have we allocated a temp for this local? */ + goto APPEND; - lclNum = impInlineFetchLocal(lclNum DEBUGARG("Inline stloc first use temp")); + case CEE_LDELEMA: + assertImp(sz == sizeof(unsigned)); - goto _PopValue; + _impResolveToken(CORINFO_TOKENKIND_Class); + + JITDUMP(" %08X", resolvedToken.token); + + ldelemClsHnd = resolvedToken.hClass; + + // If it's a value class array we just do a simple address-of + if (eeIsValueClass(ldelemClsHnd)) + { + CorInfoType cit = info.compCompHnd->getTypeForPrimitiveValueClass(ldelemClsHnd); + if (cit == CORINFO_TYPE_UNDEF) + { + lclTyp = TYP_STRUCT; + } + else + { + lclTyp = JITtype2varType(cit); + } + goto ARR_LD; } - lclNum += numArgs; + // Similarly, if its a readonly access, we can do a simple address-of + // without doing a runtime type-check + if (prefixFlags & PREFIX_READONLY) + { + lclTyp = TYP_REF; + goto ARR_LD; + } - VAR_ST: + // Otherwise we need the full helper function with run-time type check + op1 = impTokenToHandle(&resolvedToken); + if (op1 == nullptr) + { // compDonotInline() + return; + } - if (lclNum >= info.compLocalsCount && lclNum != lvaArg0Var) { - BADCODE("Bad IL"); + GenTree* type = op1; + GenTree* index = impPopStack().val; + GenTree* arr = impPopStack().val; +#ifdef TARGET_64BIT + // The CLI Spec allows an array to be indexed by either an int32 or a native int. + // The array helper takes a native int for array length. + // So if we have an int, explicitly extend it to be a native int. + if (genActualType(index->TypeGet()) != TYP_I_IMPL) + { + if (index->IsIntegralConst()) + { + index->gtType = TYP_I_IMPL; + } + else + { + bool isUnsigned = false; + index = gtNewCastNode(TYP_I_IMPL, index, isUnsigned, TYP_I_IMPL); + } + } +#endif // TARGET_64BIT + op1 = gtNewHelperCallNode(CORINFO_HELP_LDELEMA_REF, TYP_BYREF, arr, index, type); } - VAR_ST_VALID: + impPushOnStack(op1, tiRetVal); + break; - /* if it is a struct assignment, make certain we don't overflow the buffer */ - assert(lclTyp != TYP_STRUCT || lvaLclSize(lclNum) >= info.compCompHnd->getClassSize(clsHnd)); + // ldelem for reference and value types + case CEE_LDELEM: + assertImp(sz == sizeof(unsigned)); - if (lvaTable[lclNum].lvNormalizeOnLoad()) + _impResolveToken(CORINFO_TOKENKIND_Class); + + JITDUMP(" %08X", resolvedToken.token); + + ldelemClsHnd = resolvedToken.hClass; + + // If it's a reference type or generic variable type + // then just generate code as though it's a ldelem.ref instruction + if (!eeIsValueClass(ldelemClsHnd)) { - lclTyp = lvaGetRealType(lclNum); + lclTyp = TYP_REF; + opcode = CEE_LDELEM_REF; } else { - lclTyp = lvaGetActualType(lclNum); + CorInfoType jitTyp = info.compCompHnd->asCorInfoType(ldelemClsHnd); + lclTyp = JITtype2varType(jitTyp); + tiRetVal = verMakeTypeInfo(ldelemClsHnd); // precise type always needed for struct + tiRetVal.NormaliseForStack(); } + goto ARR_LD; - _PopValue: - /* Pop the value being assigned */ + case CEE_LDELEM_I1: + lclTyp = TYP_BYTE; + goto ARR_LD; + case CEE_LDELEM_I2: + lclTyp = TYP_SHORT; + goto ARR_LD; + case CEE_LDELEM_I: + lclTyp = TYP_I_IMPL; + goto ARR_LD; + case CEE_LDELEM_U4: + lclTyp = TYP_INT; + goto ARR_LD; + case CEE_LDELEM_I4: + lclTyp = TYP_INT; + goto ARR_LD; + case CEE_LDELEM_I8: + lclTyp = TYP_LONG; + goto ARR_LD; + case CEE_LDELEM_REF: + lclTyp = TYP_REF; + goto ARR_LD; + case CEE_LDELEM_R4: + lclTyp = TYP_FLOAT; + goto ARR_LD; + case CEE_LDELEM_R8: + lclTyp = TYP_DOUBLE; + goto ARR_LD; + case CEE_LDELEM_U1: + lclTyp = TYP_UBYTE; + goto ARR_LD; + case CEE_LDELEM_U2: + lclTyp = TYP_USHORT; + goto ARR_LD; + + ARR_LD: + + op2 = impPopStack().val; // index + op1 = impPopStack().val; // array + assertImp(op1->TypeIs(TYP_REF)); + // Check for null pointer - in the inliner case we simply abort. + if (compIsForInlining() && op1->IsCnsIntOrI()) { - StackEntry se = impPopStack(); - clsHnd = se.seTypeInfo.GetClassHandle(); - op1 = se.val; - tiRetVal = se.seTypeInfo; + compInlineResult->NoteFatal(InlineObservation::CALLEE_HAS_NULL_FOR_LDELEM); + return; } -#ifdef FEATURE_SIMD - if (varTypeIsSIMD(lclTyp) && (lclTyp != op1->TypeGet())) + // Mark the block as containing an index expression. + + if (op1->OperIs(GT_LCL_VAR) && op2->OperIs(GT_LCL_VAR, GT_CNS_INT, GT_ADD)) { - assert(op1->TypeGet() == TYP_STRUCT); - op1->gtType = lclTyp; + block->bbFlags |= BBF_HAS_IDX_LEN; + optMethodFlags |= OMF_HAS_ARRAYREF; } -#endif // FEATURE_SIMD - op1 = impImplicitIorI4Cast(op1, lclTyp); + op1 = gtNewArrayIndexAddr(op1, op2, lclTyp, ldelemClsHnd); -#ifdef TARGET_64BIT - // Downcast the TYP_I_IMPL into a 32-bit Int for x86 JIT compatibility - if (varTypeIsI(op1->TypeGet()) && (genActualType(lclTyp) == TYP_INT)) + if (opcode != CEE_LDELEMA) { - op1 = gtNewCastNode(TYP_INT, op1, false, TYP_INT); + op1 = gtNewIndexIndir(op1->AsIndexAddr()); } -#endif // TARGET_64BIT - // We had better assign it a value of the correct type - assertImp( - genActualType(lclTyp) == genActualType(op1->gtType) || - (genActualType(lclTyp) == TYP_I_IMPL && op1->IsLocalAddrExpr() != nullptr) || - (genActualType(lclTyp) == TYP_I_IMPL && (op1->gtType == TYP_BYREF || op1->gtType == TYP_REF)) || - (genActualType(op1->gtType) == TYP_I_IMPL && lclTyp == TYP_BYREF) || - (varTypeIsFloating(lclTyp) && varTypeIsFloating(op1->TypeGet())) || - ((genActualType(lclTyp) == TYP_BYREF) && genActualType(op1->TypeGet()) == TYP_REF)); + impPushOnStack(op1, tiRetVal); + break; - /* If op1 is "&var" then its type is the transient "*" and it can - be used either as TYP_BYREF or TYP_I_IMPL */ + // stelem for reference and value types + case CEE_STELEM: - if (op1->IsLocalAddrExpr() != nullptr) - { - assertImp(genActualType(lclTyp) == TYP_I_IMPL || lclTyp == TYP_BYREF); + assertImp(sz == sizeof(unsigned)); - /* When "&var" is created, we assume it is a byref. If it is - being assigned to a TYP_I_IMPL var, change the type to - prevent unnecessary GC info */ + _impResolveToken(CORINFO_TOKENKIND_Class); - if (genActualType(lclTyp) == TYP_I_IMPL) - { - op1->gtType = TYP_I_IMPL; - } + JITDUMP(" %08X", resolvedToken.token); + + stelemClsHnd = resolvedToken.hClass; + + // If it's a reference type just behave as though it's a stelem.ref instruction + if (!eeIsValueClass(stelemClsHnd)) + { + goto STELEM_REF_POST_VERIFY; } - // If this is a local and the local is a ref type, see - // if we can improve type information based on the - // value being assigned. - if (isLocal && (lclTyp == TYP_REF)) + // Otherwise extract the type { - // We should have seen a stloc in our IL prescan. - assert(lvaTable[lclNum].lvHasILStoreOp); + CorInfoType jitTyp = info.compCompHnd->asCorInfoType(stelemClsHnd); + lclTyp = JITtype2varType(jitTyp); + goto ARR_ST; + } - // Is there just one place this local is defined? - const bool isSingleDefLocal = lvaTable[lclNum].lvSingleDef; + case CEE_STELEM_REF: + STELEM_REF_POST_VERIFY: - // Conservative check that there is just one - // definition that reaches this store. - const bool hasSingleReachingDef = (block->bbStackDepthOnEntry() == 0); + { + GenTree* value = impStackTop(0).val; + GenTree* index = impStackTop(1).val; + GenTree* array = impStackTop(2).val; - if (isSingleDefLocal && hasSingleReachingDef) + if (opts.OptimizationEnabled()) + { + // Is this a case where we can skip the covariant store check? + if (impCanSkipCovariantStoreCheck(value, array)) { - lvaUpdateClass(lclNum, op1, clsHnd); + lclTyp = TYP_REF; + goto ARR_ST; } } - /* Filter out simple assignments to itself */ + impPopStack(3); - if (op1->gtOper == GT_LCL_VAR && lclNum == op1->AsLclVarCommon()->GetLclNum()) +// Else call a helper function to do the assignment +#ifdef TARGET_64BIT + // The CLI Spec allows an array to be indexed by either an int32 or a native int. + // The array helper takes a native int for array length. + // So if we have an int, explicitly extend it to be a native int. + if (genActualType(index->TypeGet()) != TYP_I_IMPL) { - if (opts.compDbgCode) + if (index->IsIntegralConst()) { - op1 = gtNewNothingNode(); - goto SPILL_APPEND; + index->gtType = TYP_I_IMPL; } else { - break; + bool isUnsigned = false; + index = gtNewCastNode(TYP_I_IMPL, index, isUnsigned, TYP_I_IMPL); } } +#endif // TARGET_64BIT + op1 = gtNewHelperCallNode(CORINFO_HELP_ARRADDR_ST, TYP_VOID, array, index, value); + goto SPILL_APPEND; + } - op2 = gtNewLclvNode(lclNum, lclTyp DEBUGARG(opcodeOffs + sz + 1)); + case CEE_STELEM_I1: + lclTyp = TYP_BYTE; + goto ARR_ST; + case CEE_STELEM_I2: + lclTyp = TYP_SHORT; + goto ARR_ST; + case CEE_STELEM_I: + lclTyp = TYP_I_IMPL; + goto ARR_ST; + case CEE_STELEM_I4: + lclTyp = TYP_INT; + goto ARR_ST; + case CEE_STELEM_I8: + lclTyp = TYP_LONG; + goto ARR_ST; + case CEE_STELEM_R4: + lclTyp = TYP_FLOAT; + goto ARR_ST; + case CEE_STELEM_R8: + lclTyp = TYP_DOUBLE; + goto ARR_ST; + + ARR_ST: + // TODO-Review: this comment is no longer correct. + /* The strict order of evaluation is LHS-operands, RHS-operands, + range-check, and then assignment. However, codegen currently + does the range-check before evaluation the RHS-operands. So to + maintain strict ordering, we spill the stack. */ - // Stores to pinned locals can have the implicit side effect of "unpinning", so we must spill - // things that could depend on the pin. TODO-Bug: which can actually be anything, including - // unpinned unaliased locals, not just side-effecting trees. - if (lvaTable[lclNum].lvPinned) + if (impStackTop().val->gtFlags & GTF_SIDE_EFFECT) { - impSpillSideEffects(false, CHECK_SPILL_ALL DEBUGARG("Spill before store to pinned local")); + impSpillSideEffects(false, (unsigned)CHECK_SPILL_ALL DEBUGARG( + "Strict ordering of exceptions for Array store")); } - // We can generate an assignment to a TYP_FLOAT from a TYP_DOUBLE - // We insert a cast to the dest 'op2' type - // - if ((op1->TypeGet() != op2->TypeGet()) && varTypeIsFloating(op1->gtType) && - varTypeIsFloating(op2->gtType)) + // Pull the new value from the stack. + op2 = impPopStack().val; + impBashVarAddrsToI(op2); + + // Pull the index value. + op1 = impPopStack().val; + + // Pull the array address. + op3 = impPopStack().val; + assertImp(op3->TypeIs(TYP_REF)); + + // Mark the block as containing an index expression + if (op3->OperIs(GT_LCL_VAR) && op1->OperIs(GT_LCL_VAR, GT_CNS_INT, GT_ADD)) { - op1 = gtNewCastNode(op2->TypeGet(), op1, false, op2->TypeGet()); + block->bbFlags |= BBF_HAS_IDX_LEN; + optMethodFlags |= OMF_HAS_ARRAYREF; } - if (varTypeIsStruct(lclTyp)) + // Create the index node. + op1 = gtNewArrayIndexAddr(op3, op1, lclTyp, stelemClsHnd); + op1 = gtNewIndexIndir(op1->AsIndexAddr()); + + // Create the assignment node and append it. + if (varTypeIsStruct(op1)) { - op1 = impAssignStruct(op2, op1, CHECK_SPILL_ALL); + op1 = impAssignStruct(op1, op2, stelemClsHnd, (unsigned)CHECK_SPILL_ALL); } else { - op1 = gtNewAssignNode(op2, op1); + op2 = impImplicitR4orR8Cast(op2, op1->TypeGet()); + op1 = gtNewAssignNode(op1, op2); } goto SPILL_APPEND; - case CEE_LDLOCA: - lclNum = getU2LittleEndian(codeAddr); - goto LDLOCA; + case CEE_ADD: + oper = GT_ADD; + goto MATH_OP2; - case CEE_LDLOCA_S: - lclNum = getU1LittleEndian(codeAddr); - LDLOCA: - JITDUMP(" %u", lclNum); + case CEE_ADD_OVF: + uns = false; + goto ADD_OVF; + case CEE_ADD_OVF_UN: + uns = true; + goto ADD_OVF; - if (compIsForInlining()) - { - // Get the local type - lclTyp = impInlineInfo->lclVarInfo[lclNum + impInlineInfo->argCnt].lclTypeInfo; + ADD_OVF: + ovfl = true; + callNode = false; + oper = GT_ADD; + goto MATH_OP2_FLAGS; - /* Have we allocated a temp for this local? */ + case CEE_SUB: + oper = GT_SUB; + goto MATH_OP2; - lclNum = impInlineFetchLocal(lclNum DEBUGARG("Inline ldloca(s) first use temp")); + case CEE_SUB_OVF: + uns = false; + goto SUB_OVF; + case CEE_SUB_OVF_UN: + uns = true; + goto SUB_OVF; - assert(!lvaGetDesc(lclNum)->lvNormalizeOnLoad()); - op1 = gtNewLclVarAddrNode(lclNum, TYP_BYREF); - goto _PUSH_ADRVAR; - } + SUB_OVF: + ovfl = true; + callNode = false; + oper = GT_SUB; + goto MATH_OP2_FLAGS; - lclNum += numArgs; - assertImp(lclNum < info.compLocalsCount); - goto ADRVAR; + case CEE_MUL: + oper = GT_MUL; + goto MATH_MAYBE_CALL_NO_OVF; - case CEE_LDARGA: - lclNum = getU2LittleEndian(codeAddr); - goto LDARGA; + case CEE_MUL_OVF: + uns = false; + goto MUL_OVF; + case CEE_MUL_OVF_UN: + uns = true; + goto MUL_OVF; - case CEE_LDARGA_S: - lclNum = getU1LittleEndian(codeAddr); - LDARGA: - JITDUMP(" %u", lclNum); - Verify(lclNum < info.compILargsCount, "bad arg num"); + MUL_OVF: + ovfl = true; + oper = GT_MUL; + goto MATH_MAYBE_CALL_OVF; - if (compIsForInlining()) + // Other binary math operations + + case CEE_DIV: + oper = GT_DIV; + goto MATH_MAYBE_CALL_NO_OVF; + + case CEE_DIV_UN: + oper = GT_UDIV; + goto MATH_MAYBE_CALL_NO_OVF; + + case CEE_REM: + oper = GT_MOD; + goto MATH_MAYBE_CALL_NO_OVF; + + case CEE_REM_UN: + oper = GT_UMOD; + goto MATH_MAYBE_CALL_NO_OVF; + + MATH_MAYBE_CALL_NO_OVF: + ovfl = false; + MATH_MAYBE_CALL_OVF: + // Morpher has some complex logic about when to turn different + // typed nodes on different platforms into helper calls. We + // need to either duplicate that logic here, or just + // pessimistically make all the nodes large enough to become + // call nodes. Since call nodes aren't that much larger and + // these opcodes are infrequent enough I chose the latter. + callNode = true; + goto MATH_OP2_FLAGS; + + case CEE_AND: + oper = GT_AND; + goto MATH_OP2; + case CEE_OR: + oper = GT_OR; + goto MATH_OP2; + case CEE_XOR: + oper = GT_XOR; + goto MATH_OP2; + + MATH_OP2: // For default values of 'ovfl' and 'callNode' + + ovfl = false; + callNode = false; + + MATH_OP2_FLAGS: // If 'ovfl' and 'callNode' have already been set + + /* Pull two values and push back the result */ + + op2 = impPopStack().val; + op1 = impPopStack().val; + + /* Can't do arithmetic with references */ + assertImp(genActualType(op1->TypeGet()) != TYP_REF && genActualType(op2->TypeGet()) != TYP_REF); + + // Change both to TYP_I_IMPL (impBashVarAddrsToI won't change if its a true byref, only + // if it is in the stack) + impBashVarAddrsToI(op1, op2); + + type = impGetByRefResultType(oper, uns, &op1, &op2); + + assert(!ovfl || !varTypeIsFloating(op1->gtType)); + + /* Special case: "int+0", "int-0", "int*1", "int/1" */ + + if (op2->gtOper == GT_CNS_INT) { - // In IL, LDARGA(_S) is used to load the byref managed pointer of struct argument, - // followed by a ldfld to load the field. + if ((op2->IsIntegralConst(0) && (oper == GT_ADD || oper == GT_SUB)) || + (op2->IsIntegralConst(1) && (oper == GT_MUL || oper == GT_DIV))) - op1 = impInlineFetchArg(lclNum, impInlineInfo->inlArgInfo, impInlineInfo->lclVarInfo); - if (op1->gtOper != GT_LCL_VAR) { - compInlineResult->NoteFatal(InlineObservation::CALLSITE_LDARGA_NOT_LOCAL_VAR); - return; + impPushOnStack(op1, tiRetVal); + break; } + } - op1->ChangeType(TYP_BYREF); - op1->SetOper(GT_LCL_VAR_ADDR); - goto _PUSH_ADRVAR; + // We can generate a TYP_FLOAT operation that has a TYP_DOUBLE operand + // + if (varTypeIsFloating(type) && varTypeIsFloating(op1->gtType) && varTypeIsFloating(op2->gtType)) + { + if (op1->TypeGet() != type) + { + // We insert a cast of op1 to 'type' + op1 = gtNewCastNode(type, op1, false, type); + } + if (op2->TypeGet() != type) + { + // We insert a cast of op2 to 'type' + op2 = gtNewCastNode(type, op2, false, type); + } } - lclNum = compMapILargNum(lclNum); // account for possible hidden param - assertImp(lclNum < numArgs); + if (callNode) + { + /* These operators can later be transformed into 'GT_CALL' */ - if (lclNum == info.compThisArg) + assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_MUL]); +#ifndef TARGET_ARM + assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_DIV]); + assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_UDIV]); + assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_MOD]); + assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_UMOD]); +#endif + // It's tempting to use LargeOpOpcode() here, but this logic is *not* saying + // that we'll need to transform into a general large node, but rather specifically + // to a call: by doing it this way, things keep working if there are multiple sizes, + // and a CALL is no longer the largest. + // That said, as of now it *is* a large node, so we'll do this with an assert rather + // than an "if". + assert(GenTree::s_gtNodeSizes[GT_CALL] == TREE_NODE_SZ_LARGE); + op1 = new (this, GT_CALL) GenTreeOp(oper, type, op1, op2 DEBUGARG(/*largeNode*/ true)); + } + else { - lclNum = lvaArg0Var; + op1 = gtNewOperNode(oper, type, op1, op2); } - goto ADRVAR; + /* Special case: integer/long division may throw an exception */ - ADRVAR: - // Note that this is supposed to create the transient type "*" - // which may be used as a TYP_I_IMPL. However we catch places - // where it is used as a TYP_I_IMPL and change the node if needed. - // Thus we are pessimistic and may report byrefs in the GC info - // where it was not absolutely needed, but doing otherwise would - // require careful rethinking of the importer routines which use - // the IL validity model (e. g. "impGetByRefResultType"). - op1 = gtNewLclVarAddrNode(lclNum, TYP_BYREF); + if (varTypeIsIntegral(op1->TypeGet()) && op1->OperMayThrow(this)) + { + op1->gtFlags |= GTF_EXCEPT; + } - _PUSH_ADRVAR: - assert(op1->OperIs(GT_LCL_VAR_ADDR)); + if (ovfl) + { + assert(oper == GT_ADD || oper == GT_SUB || oper == GT_MUL); + if (ovflType != TYP_UNKNOWN) + { + op1->gtType = ovflType; + } + op1->gtFlags |= (GTF_EXCEPT | GTF_OVERFLOW); + if (uns) + { + op1->gtFlags |= GTF_UNSIGNED; + } + } - tiRetVal = typeInfo(TI_BYTE).MakeByRef(); impPushOnStack(op1, tiRetVal); break; - case CEE_ARGLIST: + case CEE_SHL: + oper = GT_LSH; + goto CEE_SH_OP2; - if (!info.compIsVarArgs) - { - BADCODE("arglist in non-vararg method"); - } + case CEE_SHR: + oper = GT_RSH; + goto CEE_SH_OP2; + case CEE_SHR_UN: + oper = GT_RSZ; + goto CEE_SH_OP2; - assertImp((info.compMethodInfo->args.callConv & CORINFO_CALLCONV_MASK) == CORINFO_CALLCONV_VARARG); + CEE_SH_OP2: + op2 = impPopStack().val; + op1 = impPopStack().val; // operand to be shifted + impBashVarAddrsToI(op1, op2); + + type = genActualType(op1->TypeGet()); + op1 = gtNewOperNode(oper, type, op1, op2); - // The ARGLIST cookie is a hidden 'last' parameter, we have already - // adjusted the arg count cos this is like fetching the last param. - assertImp(numArgs > 0); - op1 = gtNewLclVarAddrNode(lvaVarargsHandleArg, TYP_BYREF); impPushOnStack(op1, tiRetVal); break; - case CEE_ENDFINALLY: + case CEE_NOT: + op1 = impPopStack().val; + impBashVarAddrsToI(op1, nullptr); + type = genActualType(op1->TypeGet()); + impPushOnStack(gtNewOperNode(GT_NOT, type, op1), tiRetVal); + break; - if (compIsForInlining()) - { - assert(!"Shouldn't have exception handlers in the inliner!"); - compInlineResult->NoteFatal(InlineObservation::CALLEE_HAS_ENDFINALLY); - return; - } + case CEE_CKFINITE: + op1 = impPopStack().val; + type = op1->TypeGet(); + op1 = gtNewOperNode(GT_CKFINITE, type, op1); + op1->gtFlags |= GTF_EXCEPT; - if (verCurrentState.esStackDepth > 0) - { - impEvalSideEffects(); - } + impPushOnStack(op1, tiRetVal); + break; - if (info.compXcptnsCount == 0) - { - BADCODE("endfinally outside finally"); - } + case CEE_LEAVE: - assert(verCurrentState.esStackDepth == 0); + val = getI4LittleEndian(codeAddr); // jump distance + jmpAddr = (IL_OFFSET)((codeAddr - info.compCode + sizeof(__int32)) + val); + goto LEAVE; - op1 = gtNewOperNode(GT_RETFILT, TYP_VOID, nullptr); - goto APPEND; + case CEE_LEAVE_S: + val = getI1LittleEndian(codeAddr); // jump distance + jmpAddr = (IL_OFFSET)((codeAddr - info.compCode + sizeof(__int8)) + val); - case CEE_ENDFILTER: + LEAVE: if (compIsForInlining()) { - assert(!"Shouldn't have exception handlers in the inliner!"); - compInlineResult->NoteFatal(InlineObservation::CALLEE_HAS_ENDFILTER); + compInlineResult->NoteFatal(InlineObservation::CALLEE_HAS_LEAVE); return; } - block->bbSetRunRarely(); // filters are rare - - if (info.compXcptnsCount == 0) - { - BADCODE("endfilter outside filter"); - } - - op1 = impPopStack().val; - assertImp(op1->gtType == TYP_INT); - if (!bbInFilterILRange(block)) + JITDUMP(" %04X", jmpAddr); + if (block->bbJumpKind != BBJ_LEAVE) { - BADCODE("EndFilter outside a filter handler"); + impResetLeaveBlock(block, jmpAddr); } - /* Mark current bb as end of filter */ - - assert(compCurBB->bbFlags & BBF_DONT_REMOVE); - assert(compCurBB->bbJumpKind == BBJ_EHFILTERRET); + assert(jmpAddr == block->bbJumpDest->bbCodeOffs); + impImportLeave(block); + impNoteBranchOffs(); - /* Mark catch handler as successor */ + break; - op1 = gtNewOperNode(GT_RETFILT, op1->TypeGet(), op1); - if (verCurrentState.esStackDepth != 0) - { - verRaiseVerifyException(INDEBUG("stack must be 1 on end of filter") DEBUGARG(__FILE__) - DEBUGARG(__LINE__)); - } - goto APPEND; + case CEE_BR: + case CEE_BR_S: + jmpDist = (sz == 1) ? getI1LittleEndian(codeAddr) : getI4LittleEndian(codeAddr); - case CEE_RET: - prefixFlags &= ~PREFIX_TAILCALL; // ret without call before it - RET: - if (!impReturnInstruction(prefixFlags, opcode)) - { - return; // abort - } - else + if (compIsForInlining() && jmpDist == 0) { - break; + break; /* NOP */ } - case CEE_JMP: + impNoteBranchOffs(); + break; - assert(!compIsForInlining()); + case CEE_BRTRUE: + case CEE_BRTRUE_S: + case CEE_BRFALSE: + case CEE_BRFALSE_S: - if ((info.compFlags & CORINFO_FLG_SYNCH) || block->hasTryIndex() || block->hasHndIndex()) - { - /* CEE_JMP does not make sense in some "protected" regions. */ + /* Pop the comparand (now there's a neat term) from the stack */ - BADCODE("Jmp not allowed in protected region"); - } + op1 = impPopStack().val; + type = op1->TypeGet(); - if (opts.IsReversePInvoke()) + // Per Ecma-355, brfalse and brtrue are only specified for nint, ref, and byref. + // + // We've historically been a bit more permissive, so here we allow + // any type that gtNewZeroConNode can handle. + if (!varTypeIsArithmetic(type) && !varTypeIsGC(type)) { - BADCODE("Jmp not allowed in reverse P/Invoke"); + BADCODE("invalid type for brtrue/brfalse"); } - if (verCurrentState.esStackDepth != 0) + if (opts.OptimizationEnabled() && (block->bbJumpDest == block->bbNext)) { - BADCODE("Stack must be empty after CEE_JMPs"); - } - - _impResolveToken(CORINFO_TOKENKIND_Method); - - JITDUMP(" %08X", resolvedToken.token); - - /* The signature of the target has to be identical to ours. - At least check that argCnt and returnType match */ + block->bbJumpKind = BBJ_NONE; - eeGetMethodSig(resolvedToken.hMethod, &sig); - if (sig.numArgs != info.compMethodInfo->args.numArgs || - sig.retType != info.compMethodInfo->args.retType || - sig.callConv != info.compMethodInfo->args.callConv) - { - BADCODE("Incompatible target for CEE_JMPs"); + if (op1->gtFlags & GTF_GLOB_EFFECT) + { + op1 = gtUnusedValNode(op1); + goto SPILL_APPEND; + } + else + { + break; + } } - op1 = new (this, GT_JMP) GenTreeVal(GT_JMP, TYP_VOID, (size_t)resolvedToken.hMethod); - - /* Mark the basic block as being a JUMP instead of RETURN */ - - block->bbFlags |= BBF_HAS_JMP; - - /* Set this flag to make sure register arguments have a location assigned - * even if we don't use them inside the method */ - - compJmpOpUsed = true; + if (op1->OperIsCompare()) + { + if (opcode == CEE_BRFALSE || opcode == CEE_BRFALSE_S) + { + // Flip the sense of the compare - fgNoStructPromotion = true; + op1 = gtReverseCond(op1); + } + } + else + { + // We'll compare against an equally-sized integer 0 + // For small types, we always compare against int + op2 = gtNewZeroConNode(genActualType(op1->gtType)); - goto APPEND; + // Create the comparison operator and try to fold it + oper = (opcode == CEE_BRTRUE || opcode == CEE_BRTRUE_S) ? GT_NE : GT_EQ; + op1 = gtNewOperNode(oper, TYP_INT, op1, op2); + } - case CEE_LDELEMA: - { - assertImp(sz == sizeof(unsigned)); + // fall through - _impResolveToken(CORINFO_TOKENKIND_Class); + COND_JUMP: - JITDUMP(" %08X", resolvedToken.token); + /* Fold comparison if we can */ - ldelemClsHnd = resolvedToken.hClass; - lclTyp = JITtype2varType(info.compCompHnd->asCorInfoType(ldelemClsHnd)); + op1 = gtFoldExpr(op1); - // If it's a value class / pointer array, or a readonly access, we don't need a type check. - // TODO-CQ: adapt "impCanSkipCovariantStoreCheck" to handle "ldelema"s and call it here to - // skip using the helper in more cases. - if ((lclTyp != TYP_REF) || ((prefixFlags & PREFIX_READONLY) != 0)) - { - goto ARR_LD; - } + /* Try to fold the really simple cases like 'iconst *, ifne/ifeq'*/ + /* Don't make any blocks unreachable in import only mode */ - // Otherwise we need the full helper function with run-time type check - GenTree* type = impTokenToHandle(&resolvedToken); - if (type == nullptr) + if ((op1->gtOper == GT_CNS_INT) && !compIsForImportOnly()) { - assert(compDonotInline()); - return; - } + /* gtFoldExpr() should prevent this as we don't want to make any blocks + unreachable under compDbgCode */ + assert(!opts.compDbgCode); - GenTree* index = impPopStack().val; - GenTree* arr = impPopStack().val; + BBjumpKinds foldedJumpKind = (BBjumpKinds)(op1->AsIntCon()->gtIconVal ? BBJ_ALWAYS : BBJ_NONE); + assertImp((block->bbJumpKind == BBJ_COND) // normal case + || (block->bbJumpKind == foldedJumpKind)); // this can happen if we are reimporting the + // block for the second time -#ifdef TARGET_64BIT - // The CLI Spec allows an array to be indexed by either an int32 or a native int. - // The array helper takes a native int for array length. - // So if we have an int, explicitly extend it to be a native int. - if (genActualType(index->TypeGet()) != TYP_I_IMPL) - { - if (index->IsIntegralConst()) - { - index->gtType = TYP_I_IMPL; - } - else + block->bbJumpKind = foldedJumpKind; +#ifdef DEBUG + if (verbose) { - bool isUnsigned = false; - index = gtNewCastNode(TYP_I_IMPL, index, isUnsigned, TYP_I_IMPL); + if (op1->AsIntCon()->gtIconVal) + { + printf("\nThe conditional jump becomes an unconditional jump to " FMT_BB "\n", + block->bbJumpDest->bbNum); + } + else + { + printf("\nThe block falls through into the next " FMT_BB "\n", block->bbNext->bbNum); + } } +#endif + break; } -#endif // TARGET_64BIT - - op1 = gtNewHelperCallNode(CORINFO_HELP_LDELEMA_REF, TYP_BYREF, arr, index, type); - impPushOnStack(op1, tiRetVal); - } - break; - - // ldelem for reference and value types - case CEE_LDELEM: - assertImp(sz == sizeof(unsigned)); - _impResolveToken(CORINFO_TOKENKIND_Class); + op1 = gtNewOperNode(GT_JTRUE, TYP_VOID, op1); - JITDUMP(" %08X", resolvedToken.token); + /* GT_JTRUE is handled specially for non-empty stacks. See 'addStmt' + in impImportBlock(block). For correct line numbers, spill stack. */ - ldelemClsHnd = resolvedToken.hClass; - lclTyp = JITtype2varType(info.compCompHnd->asCorInfoType(ldelemClsHnd)); - tiRetVal = verMakeTypeInfo(ldelemClsHnd); - tiRetVal.NormaliseForStack(); - goto ARR_LD; + if (opts.compDbgCode && impCurStmtDI.IsValid()) + { + impSpillStackEnsure(true); + } - case CEE_LDELEM_I1: - lclTyp = TYP_BYTE; - goto ARR_LD; - case CEE_LDELEM_I2: - lclTyp = TYP_SHORT; - goto ARR_LD; - case CEE_LDELEM_I: - lclTyp = TYP_I_IMPL; - goto ARR_LD; - case CEE_LDELEM_U4: - lclTyp = TYP_INT; - goto ARR_LD; - case CEE_LDELEM_I4: - lclTyp = TYP_INT; - goto ARR_LD; - case CEE_LDELEM_I8: - lclTyp = TYP_LONG; - goto ARR_LD; - case CEE_LDELEM_REF: - lclTyp = TYP_REF; - goto ARR_LD; - case CEE_LDELEM_R4: - lclTyp = TYP_FLOAT; - goto ARR_LD; - case CEE_LDELEM_R8: - lclTyp = TYP_DOUBLE; - goto ARR_LD; - case CEE_LDELEM_U1: - lclTyp = TYP_UBYTE; - goto ARR_LD; - case CEE_LDELEM_U2: - lclTyp = TYP_USHORT; - goto ARR_LD; + goto SPILL_APPEND; - ARR_LD: + case CEE_CEQ: + oper = GT_EQ; + uns = false; + goto CMP_2_OPs; + case CEE_CGT_UN: + oper = GT_GT; + uns = true; + goto CMP_2_OPs; + case CEE_CGT: + oper = GT_GT; + uns = false; + goto CMP_2_OPs; + case CEE_CLT_UN: + oper = GT_LT; + uns = true; + goto CMP_2_OPs; + case CEE_CLT: + oper = GT_LT; + uns = false; + goto CMP_2_OPs; - op2 = impPopStack().val; // index - op1 = impPopStack().val; // array - assertImp(op1->TypeIs(TYP_REF)); + CMP_2_OPs: + op2 = impPopStack().val; + op1 = impPopStack().val; - // Check for null pointer - in the inliner case we simply abort. - if (compIsForInlining() && op1->IsCnsIntOrI()) + // Recognize the IL idiom of CGT_UN(op1, 0) and normalize + // it so that downstream optimizations don't have to. + if ((opcode == CEE_CGT_UN) && op2->IsIntegralConst(0)) { - compInlineResult->NoteFatal(InlineObservation::CALLEE_HAS_NULL_FOR_LDELEM); - return; + oper = GT_NE; + uns = false; } - // Mark the block as containing an index expression. - - if (op1->OperIs(GT_LCL_VAR) && op2->OperIs(GT_LCL_VAR, GT_CNS_INT, GT_ADD)) +#ifdef TARGET_64BIT + // TODO-Casts: create a helper that upcasts int32 -> native int when necessary. + // See also identical code in impGetByRefResultType and STSFLD import. + if (varTypeIsI(op1) && (genActualType(op2) == TYP_INT)) { - block->bbFlags |= BBF_HAS_IDX_LEN; - optMethodFlags |= OMF_HAS_ARRAYREF; + op2 = gtNewCastNode(TYP_I_IMPL, op2, uns, TYP_I_IMPL); + } + else if (varTypeIsI(op2) && (genActualType(op1) == TYP_INT)) + { + op1 = gtNewCastNode(TYP_I_IMPL, op1, uns, TYP_I_IMPL); } +#endif // TARGET_64BIT - op1 = gtNewArrayIndexAddr(op1, op2, lclTyp, ldelemClsHnd); + assertImp(genActualType(op1) == genActualType(op2) || (varTypeIsI(op1) && varTypeIsI(op2)) || + (varTypeIsFloating(op1) && varTypeIsFloating(op2))); - if (opcode != CEE_LDELEMA) + // Create the comparison node. + + op1 = gtNewOperNode(oper, TYP_INT, op1, op2); + + // TODO: setting both flags when only one is appropriate. + if (uns) { - op1 = gtNewIndexIndir(op1->AsIndexAddr()); + op1->gtFlags |= GTF_RELOP_NAN_UN | GTF_UNSIGNED; } + // Fold result, if possible. + op1 = gtFoldExpr(op1); + impPushOnStack(op1, tiRetVal); break; - // stelem for reference and value types - case CEE_STELEM: - - assertImp(sz == sizeof(unsigned)); + case CEE_BEQ_S: + case CEE_BEQ: + oper = GT_EQ; + goto CMP_2_OPs_AND_BR; - _impResolveToken(CORINFO_TOKENKIND_Class); + case CEE_BGE_S: + case CEE_BGE: + oper = GT_GE; + goto CMP_2_OPs_AND_BR; - JITDUMP(" %08X", resolvedToken.token); + case CEE_BGE_UN_S: + case CEE_BGE_UN: + oper = GT_GE; + goto CMP_2_OPs_AND_BR_UN; - stelemClsHnd = resolvedToken.hClass; - lclTyp = JITtype2varType(info.compCompHnd->asCorInfoType(stelemClsHnd)); + case CEE_BGT_S: + case CEE_BGT: + oper = GT_GT; + goto CMP_2_OPs_AND_BR; - if (lclTyp != TYP_REF) - { - goto ARR_ST; - } - FALLTHROUGH; + case CEE_BGT_UN_S: + case CEE_BGT_UN: + oper = GT_GT; + goto CMP_2_OPs_AND_BR_UN; - case CEE_STELEM_REF: - { - GenTree* value = impStackTop(0).val; - GenTree* index = impStackTop(1).val; - GenTree* array = impStackTop(2).val; + case CEE_BLE_S: + case CEE_BLE: + oper = GT_LE; + goto CMP_2_OPs_AND_BR; - if (opts.OptimizationEnabled()) - { - // Is this a case where we can skip the covariant store check? - if (impCanSkipCovariantStoreCheck(value, array)) - { - lclTyp = TYP_REF; - goto ARR_ST; - } - } + case CEE_BLE_UN_S: + case CEE_BLE_UN: + oper = GT_LE; + goto CMP_2_OPs_AND_BR_UN; - impPopStack(3); + case CEE_BLT_S: + case CEE_BLT: + oper = GT_LT; + goto CMP_2_OPs_AND_BR; -// Else call a helper function to do the assignment -#ifdef TARGET_64BIT - // The CLI Spec allows an array to be indexed by either an int32 or a native int. - // The array helper takes a native int for array length. - // So if we have an int, explicitly extend it to be a native int. - if (genActualType(index->TypeGet()) != TYP_I_IMPL) - { - if (index->IsIntegralConst()) - { - index->gtType = TYP_I_IMPL; - } - else - { - bool isUnsigned = false; - index = gtNewCastNode(TYP_I_IMPL, index, isUnsigned, TYP_I_IMPL); - } - } -#endif // TARGET_64BIT - op1 = gtNewHelperCallNode(CORINFO_HELP_ARRADDR_ST, TYP_VOID, array, index, value); - goto SPILL_APPEND; - } + case CEE_BLT_UN_S: + case CEE_BLT_UN: + oper = GT_LT; + goto CMP_2_OPs_AND_BR_UN; - case CEE_STELEM_I1: - lclTyp = TYP_BYTE; - goto ARR_ST; - case CEE_STELEM_I2: - lclTyp = TYP_SHORT; - goto ARR_ST; - case CEE_STELEM_I: - lclTyp = TYP_I_IMPL; - goto ARR_ST; - case CEE_STELEM_I4: - lclTyp = TYP_INT; - goto ARR_ST; - case CEE_STELEM_I8: - lclTyp = TYP_LONG; - goto ARR_ST; - case CEE_STELEM_R4: - lclTyp = TYP_FLOAT; - goto ARR_ST; - case CEE_STELEM_R8: - lclTyp = TYP_DOUBLE; - goto ARR_ST; + case CEE_BNE_UN_S: + case CEE_BNE_UN: + oper = GT_NE; + goto CMP_2_OPs_AND_BR_UN; - ARR_ST: - // TODO-Review: this comment is no longer correct. - /* The strict order of evaluation is LHS-operands, RHS-operands, - range-check, and then assignment. However, codegen currently - does the range-check before evaluation the RHS-operands. So to - maintain strict ordering, we spill the stack. */ + CMP_2_OPs_AND_BR_UN: + uns = true; + unordered = true; + goto CMP_2_OPs_AND_BR_ALL; + CMP_2_OPs_AND_BR: + uns = false; + unordered = false; + goto CMP_2_OPs_AND_BR_ALL; + CMP_2_OPs_AND_BR_ALL: + /* Pull two values */ + op2 = impPopStack().val; + op1 = impPopStack().val; - if (impStackTop().val->gtFlags & GTF_SIDE_EFFECT) +#ifdef TARGET_64BIT + if ((op1->TypeGet() == TYP_I_IMPL) && (genActualType(op2->TypeGet()) == TYP_INT)) { - impSpillSideEffects(false, - CHECK_SPILL_ALL DEBUGARG("Strict ordering of exceptions for Array store")); + op2 = gtNewCastNode(TYP_I_IMPL, op2, uns, uns ? TYP_U_IMPL : TYP_I_IMPL); + } + else if ((op2->TypeGet() == TYP_I_IMPL) && (genActualType(op1->TypeGet()) == TYP_INT)) + { + op1 = gtNewCastNode(TYP_I_IMPL, op1, uns, uns ? TYP_U_IMPL : TYP_I_IMPL); } +#endif // TARGET_64BIT - // Pull the new value from the stack. - op2 = impPopStack().val; - impBashVarAddrsToI(op2); + assertImp(genActualType(op1->TypeGet()) == genActualType(op2->TypeGet()) || + (varTypeIsI(op1->TypeGet()) && varTypeIsI(op2->TypeGet())) || + (varTypeIsFloating(op1->gtType) && varTypeIsFloating(op2->gtType))); - // Pull the index value. - op1 = impPopStack().val; + if (opts.OptimizationEnabled() && (block->bbJumpDest == block->bbNext)) + { + block->bbJumpKind = BBJ_NONE; - // Pull the array address. - op3 = impPopStack().val; - assertImp(op3->TypeIs(TYP_REF)); + if (op1->gtFlags & GTF_GLOB_EFFECT) + { + impSpillSideEffects(false, (unsigned)CHECK_SPILL_ALL DEBUGARG( + "Branch to next Optimization, op1 side effect")); + impAppendTree(gtUnusedValNode(op1), (unsigned)CHECK_SPILL_NONE, impCurStmtDI); + } + if (op2->gtFlags & GTF_GLOB_EFFECT) + { + impSpillSideEffects(false, (unsigned)CHECK_SPILL_ALL DEBUGARG( + "Branch to next Optimization, op2 side effect")); + impAppendTree(gtUnusedValNode(op2), (unsigned)CHECK_SPILL_NONE, impCurStmtDI); + } - // Mark the block as containing an index expression - if (op3->OperIs(GT_LCL_VAR) && op1->OperIs(GT_LCL_VAR, GT_CNS_INT, GT_ADD)) +#ifdef DEBUG + if ((op1->gtFlags | op2->gtFlags) & GTF_GLOB_EFFECT) + { + impNoteLastILoffs(); + } +#endif + break; + } + + // We can generate an compare of different sized floating point op1 and op2 + // We insert a cast + // + if (varTypeIsFloating(op1->TypeGet())) { - block->bbFlags |= BBF_HAS_IDX_LEN; - optMethodFlags |= OMF_HAS_ARRAYREF; + if (op1->TypeGet() != op2->TypeGet()) + { + assert(varTypeIsFloating(op2->TypeGet())); + + // say op1=double, op2=float. To avoid loss of precision + // while comparing, op2 is converted to double and double + // comparison is done. + if (op1->TypeGet() == TYP_DOUBLE) + { + // We insert a cast of op2 to TYP_DOUBLE + op2 = gtNewCastNode(TYP_DOUBLE, op2, false, TYP_DOUBLE); + } + else if (op2->TypeGet() == TYP_DOUBLE) + { + // We insert a cast of op1 to TYP_DOUBLE + op1 = gtNewCastNode(TYP_DOUBLE, op1, false, TYP_DOUBLE); + } + } } - // Create the index node. - op1 = gtNewArrayIndexAddr(op3, op1, lclTyp, stelemClsHnd); - op1 = gtNewIndexIndir(op1->AsIndexAddr()); + /* Create and append the operator */ - // Create the assignment node and append it. - if (varTypeIsStruct(op1)) + op1 = gtNewOperNode(oper, TYP_INT, op1, op2); + + if (uns) { - op1 = impAssignStruct(op1, op2, CHECK_SPILL_ALL); + op1->gtFlags |= GTF_UNSIGNED; } - else + + if (unordered) { - op2 = impImplicitR4orR8Cast(op2, op1->TypeGet()); - op1 = gtNewAssignNode(op1, op2); + op1->gtFlags |= GTF_RELOP_NAN_UN; } - goto SPILL_APPEND; - case CEE_ADD: - oper = GT_ADD; - goto MATH_OP2; + goto COND_JUMP; - case CEE_ADD_OVF: - uns = false; - goto ADD_OVF; - case CEE_ADD_OVF_UN: - uns = true; - goto ADD_OVF; + case CEE_SWITCH: + /* Pop the switch value off the stack */ + op1 = impPopStack().val; + assertImp(genActualTypeIsIntOrI(op1->TypeGet())); - ADD_OVF: - ovfl = true; - callNode = false; - oper = GT_ADD; - goto MATH_OP2_FLAGS; + /* We can create a switch node */ - case CEE_SUB: - oper = GT_SUB; - goto MATH_OP2; + op1 = gtNewOperNode(GT_SWITCH, TYP_VOID, op1); - case CEE_SUB_OVF: - uns = false; - goto SUB_OVF; - case CEE_SUB_OVF_UN: - uns = true; - goto SUB_OVF; + val = (int)getU4LittleEndian(codeAddr); + codeAddr += 4 + val * 4; // skip over the switch-table - SUB_OVF: - ovfl = true; - callNode = false; - oper = GT_SUB; - goto MATH_OP2_FLAGS; + goto SPILL_APPEND; - case CEE_MUL: - oper = GT_MUL; - goto MATH_MAYBE_CALL_NO_OVF; + /************************** Casting OPCODES ***************************/ - case CEE_MUL_OVF: - uns = false; - goto MUL_OVF; - case CEE_MUL_OVF_UN: + case CEE_CONV_OVF_I1: + lclTyp = TYP_BYTE; + goto CONV_OVF; + case CEE_CONV_OVF_I2: + lclTyp = TYP_SHORT; + goto CONV_OVF; + case CEE_CONV_OVF_I: + lclTyp = TYP_I_IMPL; + goto CONV_OVF; + case CEE_CONV_OVF_I4: + lclTyp = TYP_INT; + goto CONV_OVF; + case CEE_CONV_OVF_I8: + lclTyp = TYP_LONG; + goto CONV_OVF; + + case CEE_CONV_OVF_U1: + lclTyp = TYP_UBYTE; + goto CONV_OVF; + case CEE_CONV_OVF_U2: + lclTyp = TYP_USHORT; + goto CONV_OVF; + case CEE_CONV_OVF_U: + lclTyp = TYP_U_IMPL; + goto CONV_OVF; + case CEE_CONV_OVF_U4: + lclTyp = TYP_UINT; + goto CONV_OVF; + case CEE_CONV_OVF_U8: + lclTyp = TYP_ULONG; + goto CONV_OVF; + + case CEE_CONV_OVF_I1_UN: + lclTyp = TYP_BYTE; + goto CONV_OVF_UN; + case CEE_CONV_OVF_I2_UN: + lclTyp = TYP_SHORT; + goto CONV_OVF_UN; + case CEE_CONV_OVF_I_UN: + lclTyp = TYP_I_IMPL; + goto CONV_OVF_UN; + case CEE_CONV_OVF_I4_UN: + lclTyp = TYP_INT; + goto CONV_OVF_UN; + case CEE_CONV_OVF_I8_UN: + lclTyp = TYP_LONG; + goto CONV_OVF_UN; + + case CEE_CONV_OVF_U1_UN: + lclTyp = TYP_UBYTE; + goto CONV_OVF_UN; + case CEE_CONV_OVF_U2_UN: + lclTyp = TYP_USHORT; + goto CONV_OVF_UN; + case CEE_CONV_OVF_U_UN: + lclTyp = TYP_U_IMPL; + goto CONV_OVF_UN; + case CEE_CONV_OVF_U4_UN: + lclTyp = TYP_UINT; + goto CONV_OVF_UN; + case CEE_CONV_OVF_U8_UN: + lclTyp = TYP_ULONG; + goto CONV_OVF_UN; + + CONV_OVF_UN: uns = true; - goto MUL_OVF; + goto CONV_OVF_COMMON; + CONV_OVF: + uns = false; + goto CONV_OVF_COMMON; - MUL_OVF: + CONV_OVF_COMMON: ovfl = true; - oper = GT_MUL; - goto MATH_MAYBE_CALL_OVF; - - // Other binary math operations + goto _CONV; - case CEE_DIV: - oper = GT_DIV; - goto MATH_MAYBE_CALL_NO_OVF; + case CEE_CONV_I1: + lclTyp = TYP_BYTE; + goto CONV; + case CEE_CONV_I2: + lclTyp = TYP_SHORT; + goto CONV; + case CEE_CONV_I: + lclTyp = TYP_I_IMPL; + goto CONV; + case CEE_CONV_I4: + lclTyp = TYP_INT; + goto CONV; + case CEE_CONV_I8: + lclTyp = TYP_LONG; + goto CONV; - case CEE_DIV_UN: - oper = GT_UDIV; - goto MATH_MAYBE_CALL_NO_OVF; + case CEE_CONV_U1: + lclTyp = TYP_UBYTE; + goto CONV; + case CEE_CONV_U2: + lclTyp = TYP_USHORT; + goto CONV; +#if (REGSIZE_BYTES == 8) + case CEE_CONV_U: + lclTyp = TYP_U_IMPL; + goto CONV_UN; +#else + case CEE_CONV_U: + lclTyp = TYP_U_IMPL; + goto CONV; +#endif + case CEE_CONV_U4: + lclTyp = TYP_UINT; + goto CONV; + case CEE_CONV_U8: + lclTyp = TYP_ULONG; + goto CONV_UN; - case CEE_REM: - oper = GT_MOD; - goto MATH_MAYBE_CALL_NO_OVF; + case CEE_CONV_R4: + lclTyp = TYP_FLOAT; + goto CONV; + case CEE_CONV_R8: + lclTyp = TYP_DOUBLE; + goto CONV; - case CEE_REM_UN: - oper = GT_UMOD; - goto MATH_MAYBE_CALL_NO_OVF; + case CEE_CONV_R_UN: + lclTyp = TYP_DOUBLE; + goto CONV_UN; - MATH_MAYBE_CALL_NO_OVF: + CONV_UN: + uns = true; ovfl = false; - MATH_MAYBE_CALL_OVF: - // Morpher has some complex logic about when to turn different - // typed nodes on different platforms into helper calls. We - // need to either duplicate that logic here, or just - // pessimistically make all the nodes large enough to become - // call nodes. Since call nodes aren't that much larger and - // these opcodes are infrequent enough I chose the latter. - callNode = true; - goto MATH_OP2_FLAGS; + goto _CONV; - case CEE_AND: - oper = GT_AND; - goto MATH_OP2; - case CEE_OR: - oper = GT_OR; - goto MATH_OP2; - case CEE_XOR: - oper = GT_XOR; - goto MATH_OP2; + CONV: + uns = false; + ovfl = false; + goto _CONV; - MATH_OP2: // For default values of 'ovfl' and 'callNode' + _CONV: + // only converts from FLOAT or DOUBLE to an integer type + // and converts from ULONG (or LONG on ARM) to DOUBLE are morphed to calls - ovfl = false; - callNode = false; + if (varTypeIsFloating(lclTyp)) + { + callNode = varTypeIsLong(impStackTop().val) || uns // uint->dbl gets turned into uint->long->dbl +#ifdef TARGET_64BIT + // TODO-ARM64-Bug?: This was AMD64; I enabled it for ARM64 also. OK? + // TYP_BYREF could be used as TYP_I_IMPL which is long. + // TODO-CQ: remove this when we lower casts long/ulong --> float/double + // and generate SSE2 code instead of going through helper calls. + || (impStackTop().val->TypeGet() == TYP_BYREF) +#endif + ; + } + else + { + callNode = varTypeIsFloating(impStackTop().val->TypeGet()); + } - MATH_OP2_FLAGS: // If 'ovfl' and 'callNode' have already been set + op1 = impPopStack().val; - /* Pull two values and push back the result */ + impBashVarAddrsToI(op1); - op2 = impPopStack().val; - op1 = impPopStack().val; + // Casts from floating point types must not have GTF_UNSIGNED set. + if (varTypeIsFloating(op1)) + { + uns = false; + } - /* Can't do arithmetic with references */ - assertImp(genActualType(op1->TypeGet()) != TYP_REF && genActualType(op2->TypeGet()) != TYP_REF); + // At this point uns, ovf, callNode are all set. - // Change both to TYP_I_IMPL (impBashVarAddrsToI won't change if its a true byref, only - // if it is in the stack) - impBashVarAddrsToI(op1, op2); + if (varTypeIsSmall(lclTyp) && !ovfl && op1->gtType == TYP_INT && op1->gtOper == GT_AND) + { + op2 = op1->AsOp()->gtOp2; - type = impGetByRefResultType(oper, uns, &op1, &op2); + if (op2->gtOper == GT_CNS_INT) + { + ssize_t ival = op2->AsIntCon()->gtIconVal; + ssize_t mask, umask; - assert(!ovfl || !varTypeIsFloating(op1->gtType)); + switch (lclTyp) + { + case TYP_BYTE: + case TYP_UBYTE: + mask = 0x00FF; + umask = 0x007F; + break; + case TYP_USHORT: + case TYP_SHORT: + mask = 0xFFFF; + umask = 0x7FFF; + break; - /* Special case: "int+0", "int-0", "int*1", "int/1" */ + default: + assert(!"unexpected type"); + return; + } - if (op2->gtOper == GT_CNS_INT) - { - if ((op2->IsIntegralConst(0) && (oper == GT_ADD || oper == GT_SUB)) || - (op2->IsIntegralConst(1) && (oper == GT_MUL || oper == GT_DIV))) + if (((ival & umask) == ival) || ((ival & mask) == ival && uns)) + { + /* Toss the cast, it's a waste of time */ - { - impPushOnStack(op1, tiRetVal); - break; - } - } + impPushOnStack(op1, tiRetVal); + break; + } + else if (ival == mask) + { + /* Toss the masking, it's a waste of time, since + we sign-extend from the small value anyways */ - // We can generate a TYP_FLOAT operation that has a TYP_DOUBLE operand - // - if (varTypeIsFloating(type) && varTypeIsFloating(op1->gtType) && varTypeIsFloating(op2->gtType)) - { - if (op1->TypeGet() != type) - { - // We insert a cast of op1 to 'type' - op1 = gtNewCastNode(type, op1, false, type); - } - if (op2->TypeGet() != type) - { - // We insert a cast of op2 to 'type' - op2 = gtNewCastNode(type, op2, false, type); + op1 = op1->AsOp()->gtOp1; + } } } - if (callNode) - { - /* These operators can later be transformed into 'GT_CALL' */ - - assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_MUL]); -#ifndef TARGET_ARM - assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_DIV]); - assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_UDIV]); - assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_MOD]); - assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_UMOD]); -#endif - // It's tempting to use LargeOpOpcode() here, but this logic is *not* saying - // that we'll need to transform into a general large node, but rather specifically - // to a call: by doing it this way, things keep working if there are multiple sizes, - // and a CALL is no longer the largest. - // That said, as of now it *is* a large node, so we'll do this with an assert rather - // than an "if". - assert(GenTree::s_gtNodeSizes[GT_CALL] == TREE_NODE_SZ_LARGE); - op1 = new (this, GT_CALL) GenTreeOp(oper, type, op1, op2 DEBUGARG(/*largeNode*/ true)); - } - else - { - op1 = gtNewOperNode(oper, type, op1, op2); - } + /* The 'op2' sub-operand of a cast is the 'real' type number, + since the result of a cast to one of the 'small' integer + types is an integer. + */ - /* Special case: integer/long division may throw an exception */ + type = genActualType(lclTyp); - if (varTypeIsIntegral(op1->TypeGet()) && op1->OperMayThrow(this)) + // If this is a no-op cast, just use op1. + if (!ovfl && (type == op1->TypeGet()) && (genTypeSize(type) == genTypeSize(lclTyp))) { - op1->gtFlags |= GTF_EXCEPT; + // Nothing needs to change } - - if (ovfl) + // Work is evidently required, add cast node + else { - assert(oper == GT_ADD || oper == GT_SUB || oper == GT_MUL); - if (ovflType != TYP_UNKNOWN) + if (callNode) { - op1->gtType = ovflType; + op1 = gtNewCastNodeL(type, op1, uns, lclTyp); } - op1->gtFlags |= (GTF_EXCEPT | GTF_OVERFLOW); - if (uns) + else { - op1->gtFlags |= GTF_UNSIGNED; + op1 = gtNewCastNode(type, op1, uns, lclTyp); } - } - - impPushOnStack(op1, tiRetVal); - break; - - case CEE_SHL: - oper = GT_LSH; - goto CEE_SH_OP2; - - case CEE_SHR: - oper = GT_RSH; - goto CEE_SH_OP2; - case CEE_SHR_UN: - oper = GT_RSZ; - goto CEE_SH_OP2; - CEE_SH_OP2: - op2 = impPopStack().val; - op1 = impPopStack().val; // operand to be shifted - impBashVarAddrsToI(op1, op2); + if (ovfl) + { + op1->gtFlags |= (GTF_OVERFLOW | GTF_EXCEPT); + } - type = genActualType(op1->TypeGet()); - op1 = gtNewOperNode(oper, type, op1, op2); + if (op1->gtGetOp1()->OperIsConst() && opts.OptimizationEnabled()) + { + // Try and fold the introduced cast + op1 = gtFoldExprConst(op1); + } + } impPushOnStack(op1, tiRetVal); break; - case CEE_NOT: + case CEE_NEG: op1 = impPopStack().val; impBashVarAddrsToI(op1, nullptr); - type = genActualType(op1->TypeGet()); - impPushOnStack(gtNewOperNode(GT_NOT, type, op1), tiRetVal); - break; - - case CEE_CKFINITE: - op1 = impPopStack().val; - type = op1->TypeGet(); - op1 = gtNewOperNode(GT_CKFINITE, type, op1); - op1->gtFlags |= GTF_EXCEPT; - - impPushOnStack(op1, tiRetVal); + impPushOnStack(gtNewOperNode(GT_NEG, genActualType(op1->gtType), op1), tiRetVal); break; - case CEE_LEAVE: - - val = getI4LittleEndian(codeAddr); // jump distance - jmpAddr = (IL_OFFSET)((codeAddr - info.compCode + sizeof(__int32)) + val); - goto LEAVE; - - case CEE_LEAVE_S: - val = getI1LittleEndian(codeAddr); // jump distance - jmpAddr = (IL_OFFSET)((codeAddr - info.compCode + sizeof(__int8)) + val); - - LEAVE: - - if (compIsForInlining()) - { - compInlineResult->NoteFatal(InlineObservation::CALLEE_HAS_LEAVE); - return; - } - - JITDUMP(" %04X", jmpAddr); - if (block->bbJumpKind != BBJ_LEAVE) - { - impResetLeaveBlock(block, jmpAddr); - } + case CEE_POP: + { + /* Pull the top value from the stack */ - assert(jmpAddr == block->bbJumpDest->bbCodeOffs); - impImportLeave(block); - impNoteBranchOffs(); + StackEntry se = impPopStack(); + clsHnd = se.seTypeInfo.GetClassHandle(); + op1 = se.val; - break; + /* Get hold of the type of the value being duplicated */ - case CEE_BR: - case CEE_BR_S: - jmpDist = (sz == 1) ? getI1LittleEndian(codeAddr) : getI4LittleEndian(codeAddr); + lclTyp = genActualType(op1->gtType); - if (compIsForInlining() && jmpDist == 0) + /* Does the value have any side effects? */ + + if ((op1->gtFlags & GTF_SIDE_EFFECT) || opts.compDbgCode) { - break; /* NOP */ - } + // Since we are throwing away the value, just normalize + // it to its address. This is more efficient. - impNoteBranchOffs(); - break; + if (varTypeIsStruct(op1)) + { + JITDUMP("\n ... CEE_POP struct ...\n"); + DISPTREE(op1); +#ifdef UNIX_AMD64_ABI + // Non-calls, such as obj or ret_expr, have to go through this. + // Calls with large struct return value have to go through this. + // Helper calls with small struct return value also have to go + // through this since they do not follow Unix calling convention. + if (op1->gtOper != GT_CALL || + !IsMultiRegReturnedType(clsHnd, op1->AsCall()->GetUnmanagedCallConv()) || + op1->AsCall()->gtCallType == CT_HELPER) +#endif // UNIX_AMD64_ABI + { + // If the value being produced comes from loading + // via an underlying address, just null check the address. + if (op1->OperIs(GT_FIELD, GT_IND, GT_OBJ)) + { + gtChangeOperToNullCheck(op1, block); + } + else + { + op1 = impGetStructAddr(op1, clsHnd, (unsigned)CHECK_SPILL_ALL, false); + } - case CEE_BRTRUE: - case CEE_BRTRUE_S: - case CEE_BRFALSE: - case CEE_BRFALSE_S: + JITDUMP("\n ... optimized to ...\n"); + DISPTREE(op1); + } + } - /* Pop the comparand (now there's a neat term) from the stack */ + // If op1 is non-overflow cast, throw it away since it is useless. + // Another reason for throwing away the useless cast is in the context of + // implicit tail calls when the operand of pop is GT_CAST(GT_CALL(..)). + // The cast gets added as part of importing GT_CALL, which gets in the way + // of fgMorphCall() on the forms of tail call nodes that we assert. + if ((op1->gtOper == GT_CAST) && !op1->gtOverflow()) + { + op1 = op1->AsOp()->gtOp1; + } - op1 = impPopStack().val; - type = op1->TypeGet(); + if (op1->gtOper != GT_CALL) + { + if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0) + { + op1 = gtUnusedValNode(op1); + } + else + { + // Can't bash to NOP here because op1 can be referenced from `currentBlock->bbEntryState`, + // if we ever need to reimport we need a valid LCL_VAR on it. + op1 = gtNewNothingNode(); + } + } - // Per Ecma-355, brfalse and brtrue are only specified for nint, ref, and byref. - // - // We've historically been a bit more permissive, so here we allow - // any type that gtNewZeroConNode can handle. - if (!varTypeIsArithmetic(type) && !varTypeIsGC(type)) - { - BADCODE("invalid type for brtrue/brfalse"); + /* Append the value to the tree list */ + goto SPILL_APPEND; } - if (opts.OptimizationEnabled() && (block->bbJumpDest == block->bbNext)) - { - block->bbJumpKind = BBJ_NONE; + /* No side effects - just throw the thing away */ + } + break; - if (op1->gtFlags & GTF_GLOB_EFFECT) + case CEE_DUP: + { + StackEntry se = impPopStack(); + GenTree* tree = se.val; + tiRetVal = se.seTypeInfo; + op1 = tree; + + // If the expression to dup is simple, just clone it. + // Otherwise spill it to a temp, and reload the temp twice. + bool cloneExpr = false; + + if (!opts.compDbgCode) + { + // Duplicate 0 and +0.0 + if (op1->IsIntegralConst(0) || op1->IsFloatPositiveZero()) { - op1 = gtUnusedValNode(op1); - goto SPILL_APPEND; + cloneExpr = true; } - else + // Duplicate locals and addresses of them + else if (op1->IsLocal()) { - break; + cloneExpr = true; } - } - - if (op1->OperIsCompare()) - { - if (opcode == CEE_BRFALSE || opcode == CEE_BRFALSE_S) + else if (op1->TypeIs(TYP_BYREF) && op1->OperIs(GT_ADDR) && op1->gtGetOp1()->IsLocal() && + (OPCODE)impGetNonPrefixOpcode(codeAddr + sz, codeEndp) != CEE_INITOBJ) { - // Flip the sense of the compare - - op1 = gtReverseCond(op1); + cloneExpr = true; } } else { - // We'll compare against an equally-sized integer 0 - // For small types, we always compare against int - op2 = gtNewZeroConNode(genActualType(op1->gtType)); + // Always clone for debug mode + cloneExpr = true; + } - // Create the comparison operator and try to fold it - oper = (opcode == CEE_BRTRUE || opcode == CEE_BRTRUE_S) ? GT_NE : GT_EQ; - op1 = gtNewOperNode(oper, TYP_INT, op1, op2); + if (!cloneExpr) + { + const unsigned tmpNum = lvaGrabTemp(true DEBUGARG("dup spill")); + impAssignTempGen(tmpNum, op1, tiRetVal.GetClassHandle(), (unsigned)CHECK_SPILL_ALL); + var_types type = genActualType(lvaTable[tmpNum].TypeGet()); + op1 = gtNewLclvNode(tmpNum, type); + + // Propagate type info to the temp from the stack and the original tree + if (type == TYP_REF) + { + assert(lvaTable[tmpNum].lvSingleDef == 0); + lvaTable[tmpNum].lvSingleDef = 1; + JITDUMP("Marked V%02u as a single def local\n", tmpNum); + lvaSetClass(tmpNum, tree, tiRetVal.GetClassHandle()); + } } - // fall through + op1 = impCloneExpr(op1, &op2, tiRetVal.GetClassHandle(), (unsigned)CHECK_SPILL_ALL, + nullptr DEBUGARG("DUP instruction")); - COND_JUMP: + assert(!(op1->gtFlags & GTF_GLOB_EFFECT) && !(op2->gtFlags & GTF_GLOB_EFFECT)); + impPushOnStack(op1, tiRetVal); + impPushOnStack(op2, tiRetVal); + } + break; - /* Fold comparison if we can */ + case CEE_STIND_I1: + lclTyp = TYP_BYTE; + goto STIND; + case CEE_STIND_I2: + lclTyp = TYP_SHORT; + goto STIND; + case CEE_STIND_I4: + lclTyp = TYP_INT; + goto STIND; + case CEE_STIND_I8: + lclTyp = TYP_LONG; + goto STIND; + case CEE_STIND_I: + lclTyp = TYP_I_IMPL; + goto STIND; + case CEE_STIND_REF: + lclTyp = TYP_REF; + goto STIND; + case CEE_STIND_R4: + lclTyp = TYP_FLOAT; + goto STIND; + case CEE_STIND_R8: + lclTyp = TYP_DOUBLE; + goto STIND; + STIND: - op1 = gtFoldExpr(op1); + op2 = impPopStack().val; // value to store + op1 = impPopStack().val; // address to store to - /* Try to fold the really simple cases like 'iconst *, ifne/ifeq'*/ - /* Don't make any blocks unreachable in import only mode */ + // you can indirect off of a TYP_I_IMPL (if we are in C) or a BYREF + assertImp(genActualType(op1->gtType) == TYP_I_IMPL || op1->gtType == TYP_BYREF); - if ((op1->gtOper == GT_CNS_INT) && !compIsForImportOnly()) - { - /* gtFoldExpr() should prevent this as we don't want to make any blocks - unreachable under compDbgCode */ - assert(!opts.compDbgCode); + impBashVarAddrsToI(op1, op2); - BBjumpKinds foldedJumpKind = (BBjumpKinds)(op1->AsIntCon()->gtIconVal ? BBJ_ALWAYS : BBJ_NONE); - assertImp((block->bbJumpKind == BBJ_COND) // normal case - || (block->bbJumpKind == foldedJumpKind)); // this can happen if we are reimporting the - // block for the second time + op2 = impImplicitR4orR8Cast(op2, lclTyp); - block->bbJumpKind = foldedJumpKind; -#ifdef DEBUG - if (verbose) +#ifdef TARGET_64BIT + // Automatic upcast for a GT_CNS_INT into TYP_I_IMPL + if ((op2->OperGet() == GT_CNS_INT) && varTypeIsI(lclTyp) && !varTypeIsI(op2->gtType)) + { + op2->gtType = TYP_I_IMPL; + } + else + { + // Allow a downcast of op2 from TYP_I_IMPL into a 32-bit Int for x86 JIT compatiblity + // + if (varTypeIsI(op2->gtType) && (genActualType(lclTyp) == TYP_INT)) { - if (op1->AsIntCon()->gtIconVal) - { - printf("\nThe conditional jump becomes an unconditional jump to " FMT_BB "\n", - block->bbJumpDest->bbNum); - } - else - { - printf("\nThe block falls through into the next " FMT_BB "\n", block->bbNext->bbNum); - } + op2 = gtNewCastNode(TYP_INT, op2, false, TYP_INT); + } + // Allow an upcast of op2 from a 32-bit Int into TYP_I_IMPL for x86 JIT compatiblity + // + if (varTypeIsI(lclTyp) && (genActualType(op2->gtType) == TYP_INT)) + { + op2 = gtNewCastNode(TYP_I_IMPL, op2, false, TYP_I_IMPL); } -#endif - break; } +#endif // TARGET_64BIT - op1 = gtNewOperNode(GT_JTRUE, TYP_VOID, op1); - - /* GT_JTRUE is handled specially for non-empty stacks. See 'addStmt' - in impImportBlock(block). For correct line numbers, spill stack. */ - - if (opts.compDbgCode && impCurStmtDI.IsValid()) + if (opcode == CEE_STIND_REF) { - impSpillStackEnsure(true); + // STIND_REF can be used to store TYP_INT, TYP_I_IMPL, TYP_REF, or TYP_BYREF + assertImp(varTypeIsIntOrI(op2->gtType) || varTypeIsGC(op2->gtType)); + lclTyp = genActualType(op2->TypeGet()); } - goto SPILL_APPEND; - - case CEE_CEQ: - oper = GT_EQ; - uns = false; - goto CMP_2_OPs; - case CEE_CGT_UN: - oper = GT_GT; - uns = true; - goto CMP_2_OPs; - case CEE_CGT: - oper = GT_GT; - uns = false; - goto CMP_2_OPs; - case CEE_CLT_UN: - oper = GT_LT; - uns = true; - goto CMP_2_OPs; - case CEE_CLT: - oper = GT_LT; - uns = false; - goto CMP_2_OPs; +// Check target type. +#ifdef DEBUG + if (op2->gtType == TYP_BYREF || lclTyp == TYP_BYREF) + { + if (op2->gtType == TYP_BYREF) + { + assertImp(lclTyp == TYP_BYREF || lclTyp == TYP_I_IMPL); + } + else if (lclTyp == TYP_BYREF) + { + assertImp(op2->gtType == TYP_BYREF || varTypeIsIntOrI(op2->gtType)); + } + } + else + { + assertImp(genActualType(op2->gtType) == genActualType(lclTyp) || + ((lclTyp == TYP_I_IMPL) && (genActualType(op2->gtType) == TYP_INT)) || + (varTypeIsFloating(op2->gtType) && varTypeIsFloating(lclTyp))); + } +#endif - CMP_2_OPs: - op2 = impPopStack().val; - op1 = impPopStack().val; + op1 = gtNewOperNode(GT_IND, lclTyp, op1); - // Recognize the IL idiom of CGT_UN(op1, 0) and normalize - // it so that downstream optimizations don't have to. - if ((opcode == CEE_CGT_UN) && op2->IsIntegralConst(0)) + if (prefixFlags & PREFIX_VOLATILE) { - oper = GT_NE; - uns = false; + assert(op1->OperGet() == GT_IND); + op1->gtFlags |= GTF_ORDER_SIDEEFF; // Prevent this from being reordered + op1->gtFlags |= GTF_IND_VOLATILE; } -#ifdef TARGET_64BIT - // TODO-Casts: create a helper that upcasts int32 -> native int when necessary. - // See also identical code in impGetByRefResultType and STSFLD import. - if (varTypeIsI(op1) && (genActualType(op2) == TYP_INT)) + if ((prefixFlags & PREFIX_UNALIGNED) && !varTypeIsByte(lclTyp)) { - op2 = gtNewCastNode(TYP_I_IMPL, op2, uns, TYP_I_IMPL); + assert(op1->OperGet() == GT_IND); + op1->gtFlags |= GTF_IND_UNALIGNED; } - else if (varTypeIsI(op2) && (genActualType(op1) == TYP_INT)) + + op1 = gtNewAssignNode(op1, op2); + op1->gtFlags |= GTF_EXCEPT | GTF_GLOB_REF; + + // Spill side-effects AND global-data-accesses + if (verCurrentState.esStackDepth > 0) { - op1 = gtNewCastNode(TYP_I_IMPL, op1, uns, TYP_I_IMPL); + impSpillSideEffects(true, (unsigned)CHECK_SPILL_ALL DEBUGARG("spill side effects before STIND")); } -#endif // TARGET_64BIT - assertImp(genActualType(op1) == genActualType(op2) || (varTypeIsI(op1) && varTypeIsI(op2)) || - (varTypeIsFloating(op1) && varTypeIsFloating(op2))); + goto APPEND; + + case CEE_LDIND_I1: + lclTyp = TYP_BYTE; + goto LDIND; + case CEE_LDIND_I2: + lclTyp = TYP_SHORT; + goto LDIND; + case CEE_LDIND_U4: + case CEE_LDIND_I4: + lclTyp = TYP_INT; + goto LDIND; + case CEE_LDIND_I8: + lclTyp = TYP_LONG; + goto LDIND; + case CEE_LDIND_REF: + lclTyp = TYP_REF; + goto LDIND; + case CEE_LDIND_I: + lclTyp = TYP_I_IMPL; + goto LDIND; + case CEE_LDIND_R4: + lclTyp = TYP_FLOAT; + goto LDIND; + case CEE_LDIND_R8: + lclTyp = TYP_DOUBLE; + goto LDIND; + case CEE_LDIND_U1: + lclTyp = TYP_UBYTE; + goto LDIND; + case CEE_LDIND_U2: + lclTyp = TYP_USHORT; + goto LDIND; + LDIND: + + op1 = impPopStack().val; // address to load from + impBashVarAddrsToI(op1); - if ((op1->TypeGet() != op2->TypeGet()) && varTypeIsFloating(op1)) +#ifdef TARGET_64BIT + // Allow an upcast of op1 from a 32-bit Int into TYP_I_IMPL for x86 JIT compatiblity + // + if (genActualType(op1->gtType) == TYP_INT) { - op1 = impImplicitR4orR8Cast(op1, TYP_DOUBLE); - op2 = impImplicitR4orR8Cast(op2, TYP_DOUBLE); + op1 = gtNewCastNode(TYP_I_IMPL, op1, false, TYP_I_IMPL); } +#endif - // Create the comparison node. - op1 = gtNewOperNode(oper, TYP_INT, op1, op2); + assertImp(genActualType(op1->gtType) == TYP_I_IMPL || op1->gtType == TYP_BYREF); - // TODO: setting both flags when only one is appropriate. - if (uns) + op1 = gtNewOperNode(GT_IND, lclTyp, op1); + + // ldind could point anywhere, example a boxed class static int + op1->gtFlags |= (GTF_EXCEPT | GTF_GLOB_REF); + + if (prefixFlags & PREFIX_VOLATILE) { - op1->gtFlags |= GTF_RELOP_NAN_UN | GTF_UNSIGNED; + assert(op1->OperGet() == GT_IND); + op1->gtFlags |= GTF_ORDER_SIDEEFF; // Prevent this from being reordered + op1->gtFlags |= GTF_IND_VOLATILE; } - // Fold result, if possible. - op1 = gtFoldExpr(op1); + if ((prefixFlags & PREFIX_UNALIGNED) && !varTypeIsByte(lclTyp)) + { + assert(op1->OperGet() == GT_IND); + op1->gtFlags |= GTF_IND_UNALIGNED; + } impPushOnStack(op1, tiRetVal); + break; - case CEE_BEQ_S: - case CEE_BEQ: - oper = GT_EQ; - goto CMP_2_OPs_AND_BR; + case CEE_UNALIGNED: - case CEE_BGE_S: - case CEE_BGE: - oper = GT_GE; - goto CMP_2_OPs_AND_BR; + assert(sz == 1); + val = getU1LittleEndian(codeAddr); + ++codeAddr; + JITDUMP(" %u", val); + if ((val != 1) && (val != 2) && (val != 4)) + { + BADCODE("Alignment unaligned. must be 1, 2, or 4"); + } - case CEE_BGE_UN_S: - case CEE_BGE_UN: - oper = GT_GE; - goto CMP_2_OPs_AND_BR_UN; + Verify(!(prefixFlags & PREFIX_UNALIGNED), "Multiple unaligned. prefixes"); + prefixFlags |= PREFIX_UNALIGNED; - case CEE_BGT_S: - case CEE_BGT: - oper = GT_GT; - goto CMP_2_OPs_AND_BR; + impValidateMemoryAccessOpcode(codeAddr, codeEndp, false); - case CEE_BGT_UN_S: - case CEE_BGT_UN: - oper = GT_GT; - goto CMP_2_OPs_AND_BR_UN; + PREFIX: + opcode = (OPCODE)getU1LittleEndian(codeAddr); + opcodeOffs = (IL_OFFSET)(codeAddr - info.compCode); + codeAddr += sizeof(__int8); + goto DECODE_OPCODE; - case CEE_BLE_S: - case CEE_BLE: - oper = GT_LE; - goto CMP_2_OPs_AND_BR; + case CEE_VOLATILE: - case CEE_BLE_UN_S: - case CEE_BLE_UN: - oper = GT_LE; - goto CMP_2_OPs_AND_BR_UN; + Verify(!(prefixFlags & PREFIX_VOLATILE), "Multiple volatile. prefixes"); + prefixFlags |= PREFIX_VOLATILE; - case CEE_BLT_S: - case CEE_BLT: - oper = GT_LT; - goto CMP_2_OPs_AND_BR; + impValidateMemoryAccessOpcode(codeAddr, codeEndp, true); - case CEE_BLT_UN_S: - case CEE_BLT_UN: - oper = GT_LT; - goto CMP_2_OPs_AND_BR_UN; + assert(sz == 0); + goto PREFIX; - case CEE_BNE_UN_S: - case CEE_BNE_UN: - oper = GT_NE; - goto CMP_2_OPs_AND_BR_UN; + case CEE_LDFTN: + { + // Need to do a lookup here so that we perform an access check + // and do a NOWAY if protections are violated + _impResolveToken(CORINFO_TOKENKIND_Method); - CMP_2_OPs_AND_BR_UN: - uns = true; - unordered = true; - goto CMP_2_OPs_AND_BR_ALL; - CMP_2_OPs_AND_BR: - uns = false; - unordered = false; - goto CMP_2_OPs_AND_BR_ALL; - CMP_2_OPs_AND_BR_ALL: - /* Pull two values */ - op2 = impPopStack().val; - op1 = impPopStack().val; + JITDUMP(" %08X", resolvedToken.token); -#ifdef TARGET_64BIT - if ((op1->TypeGet() == TYP_I_IMPL) && (genActualType(op2->TypeGet()) == TYP_INT)) + eeGetCallInfo(&resolvedToken, (prefixFlags & PREFIX_CONSTRAINED) ? &constrainedResolvedToken : nullptr, + combine(CORINFO_CALLINFO_SECURITYCHECKS, CORINFO_CALLINFO_LDFTN), &callInfo); + + // This check really only applies to intrinsic Array.Address methods + if (callInfo.sig.callConv & CORINFO_CALLCONV_PARAMTYPE) { - op2 = gtNewCastNode(TYP_I_IMPL, op2, uns, uns ? TYP_U_IMPL : TYP_I_IMPL); + NO_WAY("Currently do not support LDFTN of Parameterized functions"); } - else if ((op2->TypeGet() == TYP_I_IMPL) && (genActualType(op1->TypeGet()) == TYP_INT)) + + // Do this before DO_LDFTN since CEE_LDVIRTFN does it on its own. + impHandleAccessAllowed(callInfo.accessAllowed, &callInfo.callsiteCalloutHelper); + + DO_LDFTN: + op1 = impMethodPointer(&resolvedToken, &callInfo); + + if (compDonotInline()) { - op1 = gtNewCastNode(TYP_I_IMPL, op1, uns, uns ? TYP_U_IMPL : TYP_I_IMPL); + return; } -#endif // TARGET_64BIT - assertImp(genActualType(op1->TypeGet()) == genActualType(op2->TypeGet()) || - (varTypeIsI(op1->TypeGet()) && varTypeIsI(op2->TypeGet())) || - (varTypeIsFloating(op1->gtType) && varTypeIsFloating(op2->gtType))); + // Call info may have more precise information about the function than + // the resolved token. + mdToken constrainedToken = prefixFlags & PREFIX_CONSTRAINED ? constrainedResolvedToken.token : 0; + methodPointerInfo* heapToken = impAllocateMethodPointerInfo(resolvedToken, constrainedToken); + assert(callInfo.hMethod != nullptr); + heapToken->m_token.hMethod = callInfo.hMethod; + impPushOnStack(op1, typeInfo(heapToken)); - if (opts.OptimizationEnabled() && (block->bbJumpDest == block->bbNext)) + break; + } + + case CEE_LDVIRTFTN: + { + /* Get the method token */ + + _impResolveToken(CORINFO_TOKENKIND_Method); + + JITDUMP(" %08X", resolvedToken.token); + + eeGetCallInfo(&resolvedToken, nullptr /* constraint typeRef */, + combine(combine(CORINFO_CALLINFO_SECURITYCHECKS, CORINFO_CALLINFO_LDFTN), + CORINFO_CALLINFO_CALLVIRT), + &callInfo); + + // This check really only applies to intrinsic Array.Address methods + if (callInfo.sig.callConv & CORINFO_CALLCONV_PARAMTYPE) { - block->bbJumpKind = BBJ_NONE; + NO_WAY("Currently do not support LDFTN of Parameterized functions"); + } - if (op1->gtFlags & GTF_GLOB_EFFECT) - { - impSpillSideEffects(false, - CHECK_SPILL_ALL DEBUGARG("Branch to next Optimization, op1 side effect")); - impAppendTree(gtUnusedValNode(op1), CHECK_SPILL_NONE, impCurStmtDI); - } - if (op2->gtFlags & GTF_GLOB_EFFECT) - { - impSpillSideEffects(false, - CHECK_SPILL_ALL DEBUGARG("Branch to next Optimization, op2 side effect")); - impAppendTree(gtUnusedValNode(op2), CHECK_SPILL_NONE, impCurStmtDI); - } + mflags = callInfo.methodFlags; -#ifdef DEBUG - if ((op1->gtFlags | op2->gtFlags) & GTF_GLOB_EFFECT) + impHandleAccessAllowed(callInfo.accessAllowed, &callInfo.callsiteCalloutHelper); + + if (compIsForInlining()) + { + if (mflags & (CORINFO_FLG_FINAL | CORINFO_FLG_STATIC) || !(mflags & CORINFO_FLG_VIRTUAL)) { - impNoteLastILoffs(); + compInlineResult->NoteFatal(InlineObservation::CALLSITE_LDVIRTFN_ON_NON_VIRTUAL); + return; } -#endif - break; } - // We can generate an compare of different sized floating point op1 and op2 - // We insert a cast - // - if (varTypeIsFloating(op1->TypeGet())) + CORINFO_SIG_INFO& ftnSig = callInfo.sig; + + /* Get the object-ref */ + op1 = impPopStack().val; + assertImp(op1->gtType == TYP_REF); + + if (opts.IsReadyToRun()) { - if (op1->TypeGet() != op2->TypeGet()) + if (callInfo.kind != CORINFO_VIRTUALCALL_LDVIRTFTN) { - assert(varTypeIsFloating(op2->TypeGet())); - - // say op1=double, op2=float. To avoid loss of precision - // while comparing, op2 is converted to double and double - // comparison is done. - if (op1->TypeGet() == TYP_DOUBLE) - { - // We insert a cast of op2 to TYP_DOUBLE - op2 = gtNewCastNode(TYP_DOUBLE, op2, false, TYP_DOUBLE); - } - else if (op2->TypeGet() == TYP_DOUBLE) + if (op1->gtFlags & GTF_SIDE_EFFECT) { - // We insert a cast of op1 to TYP_DOUBLE - op1 = gtNewCastNode(TYP_DOUBLE, op1, false, TYP_DOUBLE); + op1 = gtUnusedValNode(op1); + impAppendTree(op1, (unsigned)CHECK_SPILL_ALL, impCurStmtDI); } + goto DO_LDFTN; } } - - /* Create and append the operator */ - - op1 = gtNewOperNode(oper, TYP_INT, op1, op2); - - if (uns) + else if (mflags & (CORINFO_FLG_FINAL | CORINFO_FLG_STATIC) || !(mflags & CORINFO_FLG_VIRTUAL)) { - op1->gtFlags |= GTF_UNSIGNED; + if (op1->gtFlags & GTF_SIDE_EFFECT) + { + op1 = gtUnusedValNode(op1); + impAppendTree(op1, (unsigned)CHECK_SPILL_ALL, impCurStmtDI); + } + goto DO_LDFTN; } - if (unordered) + GenTree* fptr = impImportLdvirtftn(op1, &resolvedToken, &callInfo); + if (compDonotInline()) { - op1->gtFlags |= GTF_RELOP_NAN_UN; + return; } - goto COND_JUMP; + methodPointerInfo* heapToken = impAllocateMethodPointerInfo(resolvedToken, 0); - case CEE_SWITCH: - /* Pop the switch value off the stack */ - op1 = impPopStack().val; - assertImp(genActualTypeIsIntOrI(op1->TypeGet())); + assert(heapToken->m_token.tokenType == CORINFO_TOKENKIND_Method); + assert(callInfo.hMethod != nullptr); - /* We can create a switch node */ + heapToken->m_token.tokenType = CORINFO_TOKENKIND_Ldvirtftn; + heapToken->m_token.hMethod = callInfo.hMethod; + impPushOnStack(fptr, typeInfo(heapToken)); - op1 = gtNewOperNode(GT_SWITCH, TYP_VOID, op1); + break; + } - val = (int)getU4LittleEndian(codeAddr); - codeAddr += 4 + val * 4; // skip over the switch-table + case CEE_CONSTRAINED: - goto SPILL_APPEND; + assertImp(sz == sizeof(unsigned)); + impResolveToken(codeAddr, &constrainedResolvedToken, CORINFO_TOKENKIND_Constrained); + codeAddr += sizeof(unsigned); // prefix instructions must increment codeAddr manually + JITDUMP(" (%08X) ", constrainedResolvedToken.token); - /************************** Casting OPCODES ***************************/ + Verify(!(prefixFlags & PREFIX_CONSTRAINED), "Multiple constrained. prefixes"); + prefixFlags |= PREFIX_CONSTRAINED; - case CEE_CONV_OVF_I1: - lclTyp = TYP_BYTE; - goto CONV_OVF; - case CEE_CONV_OVF_I2: - lclTyp = TYP_SHORT; - goto CONV_OVF; - case CEE_CONV_OVF_I: - lclTyp = TYP_I_IMPL; - goto CONV_OVF; - case CEE_CONV_OVF_I4: - lclTyp = TYP_INT; - goto CONV_OVF; - case CEE_CONV_OVF_I8: - lclTyp = TYP_LONG; - goto CONV_OVF; + { + OPCODE actualOpcode = impGetNonPrefixOpcode(codeAddr, codeEndp); + if (actualOpcode != CEE_CALLVIRT && actualOpcode != CEE_CALL && actualOpcode != CEE_LDFTN) + { + BADCODE("constrained. has to be followed by callvirt, call or ldftn"); + } + } - case CEE_CONV_OVF_U1: - lclTyp = TYP_UBYTE; - goto CONV_OVF; - case CEE_CONV_OVF_U2: - lclTyp = TYP_USHORT; - goto CONV_OVF; - case CEE_CONV_OVF_U: - lclTyp = TYP_U_IMPL; - goto CONV_OVF; - case CEE_CONV_OVF_U4: - lclTyp = TYP_UINT; - goto CONV_OVF; - case CEE_CONV_OVF_U8: - lclTyp = TYP_ULONG; - goto CONV_OVF; + goto PREFIX; - case CEE_CONV_OVF_I1_UN: - lclTyp = TYP_BYTE; - goto CONV_OVF_UN; - case CEE_CONV_OVF_I2_UN: - lclTyp = TYP_SHORT; - goto CONV_OVF_UN; - case CEE_CONV_OVF_I_UN: - lclTyp = TYP_I_IMPL; - goto CONV_OVF_UN; - case CEE_CONV_OVF_I4_UN: - lclTyp = TYP_INT; - goto CONV_OVF_UN; - case CEE_CONV_OVF_I8_UN: - lclTyp = TYP_LONG; - goto CONV_OVF_UN; + case CEE_READONLY: + JITDUMP(" readonly."); - case CEE_CONV_OVF_U1_UN: - lclTyp = TYP_UBYTE; - goto CONV_OVF_UN; - case CEE_CONV_OVF_U2_UN: - lclTyp = TYP_USHORT; - goto CONV_OVF_UN; - case CEE_CONV_OVF_U_UN: - lclTyp = TYP_U_IMPL; - goto CONV_OVF_UN; - case CEE_CONV_OVF_U4_UN: - lclTyp = TYP_UINT; - goto CONV_OVF_UN; - case CEE_CONV_OVF_U8_UN: - lclTyp = TYP_ULONG; - goto CONV_OVF_UN; + Verify(!(prefixFlags & PREFIX_READONLY), "Multiple readonly. prefixes"); + prefixFlags |= PREFIX_READONLY; - CONV_OVF_UN: - uns = true; - goto CONV_OVF_COMMON; - CONV_OVF: - uns = false; - goto CONV_OVF_COMMON; + { + OPCODE actualOpcode = impGetNonPrefixOpcode(codeAddr, codeEndp); + if (actualOpcode != CEE_LDELEMA && !impOpcodeIsCallOpcode(actualOpcode)) + { + BADCODE("readonly. has to be followed by ldelema or call"); + } + } - CONV_OVF_COMMON: - ovfl = true; - goto _CONV; + assert(sz == 0); + goto PREFIX; - case CEE_CONV_I1: - lclTyp = TYP_BYTE; - goto CONV; - case CEE_CONV_I2: - lclTyp = TYP_SHORT; - goto CONV; - case CEE_CONV_I: - lclTyp = TYP_I_IMPL; - goto CONV; - case CEE_CONV_I4: - lclTyp = TYP_INT; - goto CONV; - case CEE_CONV_I8: - lclTyp = TYP_LONG; - goto CONV; + case CEE_TAILCALL: + JITDUMP(" tail."); - case CEE_CONV_U1: - lclTyp = TYP_UBYTE; - goto CONV; - case CEE_CONV_U2: - lclTyp = TYP_USHORT; - goto CONV; -#if (REGSIZE_BYTES == 8) - case CEE_CONV_U: - lclTyp = TYP_U_IMPL; - goto CONV_UN; -#else - case CEE_CONV_U: - lclTyp = TYP_U_IMPL; - goto CONV; -#endif - case CEE_CONV_U4: - lclTyp = TYP_UINT; - goto CONV; - case CEE_CONV_U8: - lclTyp = TYP_ULONG; - goto CONV_UN; + Verify(!(prefixFlags & PREFIX_TAILCALL_EXPLICIT), "Multiple tailcall. prefixes"); + prefixFlags |= PREFIX_TAILCALL_EXPLICIT; - case CEE_CONV_R4: - lclTyp = TYP_FLOAT; - goto CONV; - case CEE_CONV_R8: - lclTyp = TYP_DOUBLE; - goto CONV; + { + OPCODE actualOpcode = impGetNonPrefixOpcode(codeAddr, codeEndp); + if (!impOpcodeIsCallOpcode(actualOpcode)) + { + BADCODE("tailcall. has to be followed by call, callvirt or calli"); + } + } + assert(sz == 0); + goto PREFIX; - case CEE_CONV_R_UN: - lclTyp = TYP_DOUBLE; - goto CONV_UN; + case CEE_NEWOBJ: - CONV_UN: - uns = true; - ovfl = false; - goto _CONV; + /* Since we will implicitly insert newObjThisPtr at the start of the + argument list, spill any GTF_ORDER_SIDEEFF */ + impSpillSpecialSideEff(); - CONV: - uns = false; - ovfl = false; - goto _CONV; + /* NEWOBJ does not respond to TAIL */ + prefixFlags &= ~PREFIX_TAILCALL_EXPLICIT; - _CONV: - // only converts from FLOAT or DOUBLE to an integer type - // and converts from ULONG (or LONG on ARM) to DOUBLE are morphed to calls + /* NEWOBJ does not respond to CONSTRAINED */ + prefixFlags &= ~PREFIX_CONSTRAINED; - if (varTypeIsFloating(lclTyp)) - { - callNode = varTypeIsLong(impStackTop().val) || uns // uint->dbl gets turned into uint->long->dbl -#ifdef TARGET_64BIT - // TODO-ARM64-Bug?: This was AMD64; I enabled it for ARM64 also. OK? - // TYP_BYREF could be used as TYP_I_IMPL which is long. - // TODO-CQ: remove this when we lower casts long/ulong --> float/double - // and generate SSE2 code instead of going through helper calls. - || (impStackTop().val->TypeGet() == TYP_BYREF) -#endif - ; - } - else + _impResolveToken(CORINFO_TOKENKIND_NewObj); + + eeGetCallInfo(&resolvedToken, nullptr /* constraint typeRef*/, + combine(CORINFO_CALLINFO_SECURITYCHECKS, CORINFO_CALLINFO_ALLOWINSTPARAM), &callInfo); + + mflags = callInfo.methodFlags; + + if ((mflags & (CORINFO_FLG_STATIC | CORINFO_FLG_ABSTRACT)) != 0) { - callNode = varTypeIsFloating(impStackTop().val->TypeGet()); + BADCODE("newobj on static or abstract method"); } - op1 = impPopStack().val; + // Insert the security callout before any actual code is generated + impHandleAccessAllowed(callInfo.accessAllowed, &callInfo.callsiteCalloutHelper); - impBashVarAddrsToI(op1); + // There are three different cases for new. + // Object size is variable (depends on arguments). + // 1) Object is an array (arrays treated specially by the EE) + // 2) Object is some other variable sized object (e.g. String) + // 3) Class Size can be determined beforehand (normal case) + // In the first case, we need to call a NEWOBJ helper (multinewarray). + // In the second case we call the constructor with a '0' this pointer. + // In the third case we alloc the memory, then call the constructor. - // Casts from floating point types must not have GTF_UNSIGNED set. - if (varTypeIsFloating(op1)) + clsFlags = callInfo.classFlags; + if (clsFlags & CORINFO_FLG_ARRAY) { - uns = false; - } + // Arrays need to call the NEWOBJ helper. + assertImp(clsFlags & CORINFO_FLG_VAROBJSIZE); - // At this point uns, ovf, callNode are all set. + impImportNewObjArray(&resolvedToken, &callInfo); + if (compDonotInline()) + { + return; + } - if (varTypeIsSmall(lclTyp) && !ovfl && op1->gtType == TYP_INT && op1->gtOper == GT_AND) + callTyp = TYP_REF; + break; + } + // At present this can only be String + else if (clsFlags & CORINFO_FLG_VAROBJSIZE) { - op2 = op1->AsOp()->gtOp2; + // Skip this thisPtr argument + newObjThisPtr = nullptr; - if (op2->gtOper == GT_CNS_INT) + /* Remember that this basic block contains 'new' of an object */ + block->bbFlags |= BBF_HAS_NEWOBJ; + optMethodFlags |= OMF_HAS_NEWOBJ; + } + else + { + // This is the normal case where the size of the object is + // fixed. Allocate the memory and call the constructor. + + // Note: We cannot add a peep to avoid use of temp here + // becase we don't have enough interference info to detect when + // sources and destination interfere, example: s = new S(ref); + + // TODO: We find the correct place to introduce a general + // reverse copy prop for struct return values from newobj or + // any function returning structs. + + /* get a temporary for the new object */ + lclNum = lvaGrabTemp(true DEBUGARG("NewObj constructor temp")); + if (compDonotInline()) { - ssize_t ival = op2->AsIntCon()->gtIconVal; - ssize_t mask, umask; + // Fail fast if lvaGrabTemp fails with CALLSITE_TOO_MANY_LOCALS. + assert(compInlineResult->GetObservation() == InlineObservation::CALLSITE_TOO_MANY_LOCALS); + return; + } - switch (lclTyp) + // In the value class case we only need clsHnd for size calcs. + // + // The lookup of the code pointer will be handled by CALL in this case + if (clsFlags & CORINFO_FLG_VALUECLASS) + { + if (compIsForInlining()) { - case TYP_BYTE: - case TYP_UBYTE: - mask = 0x00FF; - umask = 0x007F; - break; - case TYP_USHORT: - case TYP_SHORT: - mask = 0xFFFF; - umask = 0x7FFF; - break; + // If value class has GC fields, inform the inliner. It may choose to + // bail out on the inline. + DWORD typeFlags = info.compCompHnd->getClassAttribs(resolvedToken.hClass); + if ((typeFlags & CORINFO_FLG_CONTAINS_GC_PTR) != 0) + { + compInlineResult->Note(InlineObservation::CALLEE_HAS_GC_STRUCT); + if (compInlineResult->IsFailure()) + { + return; + } - default: - assert(!"unexpected type"); - return; + // Do further notification in the case where the call site is rare; + // some policies do not track the relative hotness of call sites for + // "always" inline cases. + if (impInlineInfo->iciBlock->isRunRarely()) + { + compInlineResult->Note(InlineObservation::CALLSITE_RARE_GC_STRUCT); + if (compInlineResult->IsFailure()) + { + return; + } + } + } } - if (((ival & umask) == ival) || ((ival & mask) == ival && uns)) - { - /* Toss the cast, it's a waste of time */ + CorInfoType jitTyp = info.compCompHnd->asCorInfoType(resolvedToken.hClass); - impPushOnStack(op1, tiRetVal); - break; + if (impIsPrimitive(jitTyp)) + { + lvaTable[lclNum].lvType = JITtype2varType(jitTyp); } - else if (ival == mask) + else { - /* Toss the masking, it's a waste of time, since - we sign-extend from the small value anyways */ - - op1 = op1->AsOp()->gtOp1; + // The local variable itself is the allocated space. + // Here we need unsafe value cls check, since the address of struct is taken for further use + // and potentially exploitable. + lvaSetStruct(lclNum, resolvedToken.hClass, true /* unsafe value cls check */); } - } - } - /* The 'op2' sub-operand of a cast is the 'real' type number, - since the result of a cast to one of the 'small' integer - types is an integer. - */ + bool bbInALoop = impBlockIsInALoop(block); + bool bbIsReturn = (block->bbJumpKind == BBJ_RETURN) && + (!compIsForInlining() || (impInlineInfo->iciBlock->bbJumpKind == BBJ_RETURN)); + LclVarDsc* const lclDsc = lvaGetDesc(lclNum); + if (fgVarNeedsExplicitZeroInit(lclNum, bbInALoop, bbIsReturn)) + { + // Append a tree to zero-out the temp + newObjThisPtr = gtNewLclvNode(lclNum, lclDsc->TypeGet()); - type = genActualType(lclTyp); + newObjThisPtr = gtNewBlkOpNode(newObjThisPtr, // Dest + gtNewIconNode(0), // Value + false, // isVolatile + false); // not copyBlock + impAppendTree(newObjThisPtr, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); + } + else + { + JITDUMP("\nSuppressing zero-init for V%02u -- expect to zero in prolog\n", lclNum); + lclDsc->lvSuppressedZeroInit = 1; + compSuppressedZeroInit = true; + } - // If this is a no-op cast, just use op1. - if (!ovfl && (type == op1->TypeGet()) && (genTypeSize(type) == genTypeSize(lclTyp))) - { - // Nothing needs to change - } - // Work is evidently required, add cast node - else - { - if (callNode) - { - op1 = gtNewCastNodeL(type, op1, uns, lclTyp); + // Obtain the address of the temp + newObjThisPtr = + gtNewOperNode(GT_ADDR, TYP_BYREF, gtNewLclvNode(lclNum, lvaTable[lclNum].TypeGet())); } else { - op1 = gtNewCastNode(type, op1, uns, lclTyp); - } + // If we're newing up a finalizable object, spill anything that can cause exceptions. + // + bool hasSideEffects = false; + CorInfoHelpFunc newHelper = + info.compCompHnd->getNewHelper(&resolvedToken, info.compMethodHnd, &hasSideEffects); - if (ovfl) - { - op1->gtFlags |= (GTF_OVERFLOW | GTF_EXCEPT); - } + if (hasSideEffects) + { + JITDUMP("\nSpilling stack for finalizable newobj\n"); + impSpillSideEffects(true, (unsigned)CHECK_SPILL_ALL DEBUGARG("finalizable newobj spill")); + } - if (op1->gtGetOp1()->OperIsConst() && opts.OptimizationEnabled()) - { - // Try and fold the introduced cast - op1 = gtFoldExprConst(op1); + const bool useParent = true; + op1 = gtNewAllocObjNode(&resolvedToken, useParent); + if (op1 == nullptr) + { + return; + } + + // Remember that this basic block contains 'new' of an object + block->bbFlags |= BBF_HAS_NEWOBJ; + optMethodFlags |= OMF_HAS_NEWOBJ; + + // Append the assignment to the temp/local. Dont need to spill + // at all as we are just calling an EE-Jit helper which can only + // cause an (async) OutOfMemoryException. + + // We assign the newly allocated object (by a GT_ALLOCOBJ node) + // to a temp. Note that the pattern "temp = allocObj" is required + // by ObjectAllocator phase to be able to determine GT_ALLOCOBJ nodes + // without exhaustive walk over all expressions. + + impAssignTempGen(lclNum, op1, (unsigned)CHECK_SPILL_NONE); + + assert(lvaTable[lclNum].lvSingleDef == 0); + lvaTable[lclNum].lvSingleDef = 1; + JITDUMP("Marked V%02u as a single def local\n", lclNum); + lvaSetClass(lclNum, resolvedToken.hClass, true /* is Exact */); + + newObjThisPtr = gtNewLclvNode(lclNum, TYP_REF); } } + goto CALL; - impPushOnStack(op1, tiRetVal); - break; + case CEE_CALLI: - case CEE_NEG: - op1 = impPopStack().val; - impBashVarAddrsToI(op1, nullptr); - impPushOnStack(gtNewOperNode(GT_NEG, genActualType(op1->gtType), op1), tiRetVal); - break; + /* CALLI does not respond to CONSTRAINED */ + prefixFlags &= ~PREFIX_CONSTRAINED; - case CEE_POP: - { - /* Pull the top value from the stack */ + FALLTHROUGH; - StackEntry se = impPopStack(); - clsHnd = se.seTypeInfo.GetClassHandle(); - op1 = se.val; + case CEE_CALLVIRT: + case CEE_CALL: - /* Get hold of the type of the value being duplicated */ + // We can't call getCallInfo on the token from a CALLI, but we need it in + // many other places. We unfortunately embed that knowledge here. + if (opcode != CEE_CALLI) + { + _impResolveToken(CORINFO_TOKENKIND_Method); - lclTyp = genActualType(op1->gtType); + eeGetCallInfo(&resolvedToken, + (prefixFlags & PREFIX_CONSTRAINED) ? &constrainedResolvedToken : nullptr, + // this is how impImportCall invokes getCallInfo + combine(combine(CORINFO_CALLINFO_ALLOWINSTPARAM, CORINFO_CALLINFO_SECURITYCHECKS), + (opcode == CEE_CALLVIRT) ? CORINFO_CALLINFO_CALLVIRT : CORINFO_CALLINFO_NONE), + &callInfo); + } + else + { + // Suppress uninitialized use warning. + memset(&resolvedToken, 0, sizeof(resolvedToken)); + memset(&callInfo, 0, sizeof(callInfo)); + + resolvedToken.token = getU4LittleEndian(codeAddr); + resolvedToken.tokenContext = impTokenLookupContextHandle; + resolvedToken.tokenScope = info.compScopeHnd; + } + + CALL: // memberRef should be set. + // newObjThisPtr should be set for CEE_NEWOBJ + + JITDUMP(" %08X", resolvedToken.token); + constraintCall = (prefixFlags & PREFIX_CONSTRAINED) != 0; + + bool newBBcreatedForTailcallStress; + bool passedStressModeValidation; - /* Does the value have any side effects? */ + newBBcreatedForTailcallStress = false; + passedStressModeValidation = true; - if ((op1->gtFlags & GTF_SIDE_EFFECT) || opts.compDbgCode) + if (compIsForInlining()) { - // Since we are throwing away the value, just normalize - // it to its address. This is more efficient. - - if (varTypeIsStruct(op1)) + if (compDonotInline()) { - JITDUMP("\n ... CEE_POP struct ...\n"); - DISPTREE(op1); -#ifdef UNIX_AMD64_ABI - // Non-calls, such as obj or ret_expr, have to go through this. - // Calls with large struct return value have to go through this. - // Helper calls with small struct return value also have to go - // through this since they do not follow Unix calling convention. - if (op1->gtOper != GT_CALL || - !IsMultiRegReturnedType(clsHnd, op1->AsCall()->GetUnmanagedCallConv()) || - op1->AsCall()->gtCallType == CT_HELPER) -#endif // UNIX_AMD64_ABI + return; + } + // We rule out inlinees with explicit tail calls in fgMakeBasicBlocks. + assert((prefixFlags & PREFIX_TAILCALL_EXPLICIT) == 0); + } + else + { + if (compTailCallStress()) + { + // Have we created a new BB after the "call" instruction in fgMakeBasicBlocks()? + // Tail call stress only recognizes call+ret patterns and forces them to be + // explicit tail prefixed calls. Also fgMakeBasicBlocks() under tail call stress + // doesn't import 'ret' opcode following the call into the basic block containing + // the call instead imports it to a new basic block. Note that fgMakeBasicBlocks() + // is already checking that there is an opcode following call and hence it is + // safe here to read next opcode without bounds check. + newBBcreatedForTailcallStress = + impOpcodeIsCallOpcode(opcode) && // Current opcode is a CALL, (not a CEE_NEWOBJ). So, don't + // make it jump to RET. + (OPCODE)getU1LittleEndian(codeAddr + sz) == CEE_RET; // Next opcode is a CEE_RET + + bool hasTailPrefix = (prefixFlags & PREFIX_TAILCALL_EXPLICIT); + if (newBBcreatedForTailcallStress && !hasTailPrefix) { - // If the value being produced comes from loading - // via an underlying address, just null check the address. - if (op1->OperIs(GT_FIELD, GT_IND, GT_OBJ)) + // Do a more detailed evaluation of legality + const bool returnFalseIfInvalid = true; + const bool passedConstraintCheck = + verCheckTailCallConstraint(opcode, &resolvedToken, + constraintCall ? &constrainedResolvedToken : nullptr, + returnFalseIfInvalid); + + if (passedConstraintCheck) { - gtChangeOperToNullCheck(op1, block); + // Now check with the runtime + CORINFO_METHOD_HANDLE declaredCalleeHnd = callInfo.hMethod; + bool isVirtual = (callInfo.kind == CORINFO_VIRTUALCALL_STUB) || + (callInfo.kind == CORINFO_VIRTUALCALL_VTABLE); + CORINFO_METHOD_HANDLE exactCalleeHnd = isVirtual ? nullptr : declaredCalleeHnd; + if (info.compCompHnd->canTailCall(info.compMethodHnd, declaredCalleeHnd, exactCalleeHnd, + hasTailPrefix)) // Is it legal to do tailcall? + { + // Stress the tailcall. + JITDUMP(" (Tailcall stress: prefixFlags |= PREFIX_TAILCALL_EXPLICIT)"); + prefixFlags |= PREFIX_TAILCALL_EXPLICIT; + prefixFlags |= PREFIX_TAILCALL_STRESS; + } + else + { + // Runtime disallows this tail call + JITDUMP(" (Tailcall stress: runtime preventing tailcall)"); + passedStressModeValidation = false; + } } else { - op1 = impGetStructAddr(op1, clsHnd, CHECK_SPILL_ALL, false); + // Constraints disallow this tail call + JITDUMP(" (Tailcall stress: constraint check failed)"); + passedStressModeValidation = false; } - - JITDUMP("\n ... optimized to ...\n"); - DISPTREE(op1); - } - } - - // If op1 is non-overflow cast, throw it away since it is useless. - // Another reason for throwing away the useless cast is in the context of - // implicit tail calls when the operand of pop is GT_CAST(GT_CALL(..)). - // The cast gets added as part of importing GT_CALL, which gets in the way - // of fgMorphCall() on the forms of tail call nodes that we assert. - if ((op1->gtOper == GT_CAST) && !op1->gtOverflow()) - { - op1 = op1->AsOp()->gtOp1; - } - - if (op1->gtOper != GT_CALL) - { - if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0) - { - op1 = gtUnusedValNode(op1); - } - else - { - // Can't bash to NOP here because op1 can be referenced from `currentBlock->bbEntryState`, - // if we ever need to reimport we need a valid LCL_VAR on it. - op1 = gtNewNothingNode(); } } - - /* Append the value to the tree list */ - goto SPILL_APPEND; } - /* No side effects - just throw the thing away */ - } - break; - - case CEE_DUP: - { - StackEntry se = impPopStack(); - GenTree* tree = se.val; - tiRetVal = se.seTypeInfo; - op1 = tree; - - // In unoptimized code we leave the decision of - // cloning/creating temps up to impCloneExpr, while in - // optimized code we prefer temps except for some cases we know - // are profitable. + // This is split up to avoid goto flow warnings. + bool isRecursive; + isRecursive = !compIsForInlining() && (callInfo.hMethod == info.compMethodHnd); - if (opts.OptimizationEnabled()) + // If we've already disqualified this call as a tail call under tail call stress, + // don't consider it for implicit tail calling either. + // + // When not running under tail call stress, we may mark this call as an implicit + // tail call candidate. We'll do an "equivalent" validation during impImportCall. + // + // Note that when running under tail call stress, a call marked as explicit + // tail prefixed will not be considered for implicit tail calling. + if (passedStressModeValidation && + impIsImplicitTailCallCandidate(opcode, codeAddr + sz, codeEndp, prefixFlags, isRecursive)) { - bool clone = false; - // Duplicate 0 and +0.0 - if (op1->IsIntegralConst(0) || op1->IsFloatPositiveZero()) - { - clone = true; - } - // Duplicate locals and addresses of them - else if (op1->IsLocal()) + if (compIsForInlining()) { - clone = true; +#if FEATURE_TAILCALL_OPT_SHARED_RETURN + // Are we inlining at an implicit tail call site? If so the we can flag + // implicit tail call sites in the inline body. These call sites + // often end up in non BBJ_RETURN blocks, so only flag them when + // we're able to handle shared returns. + if (impInlineInfo->iciCall->IsImplicitTailCall()) + { + JITDUMP("\n (Inline Implicit Tail call: prefixFlags |= PREFIX_TAILCALL_IMPLICIT)"); + prefixFlags |= PREFIX_TAILCALL_IMPLICIT; + } +#endif // FEATURE_TAILCALL_OPT_SHARED_RETURN } - else if (op1->TypeIs(TYP_BYREF, TYP_I_IMPL) && impIsAddressInLocal(op1)) + else { - // We mark implicit byrefs with GTF_GLOB_REF (see gtNewFieldRef for why). - // Avoid cloning for these. - clone = (op1->gtFlags & GTF_GLOB_REF) == 0; + JITDUMP("\n (Implicit Tail call: prefixFlags |= PREFIX_TAILCALL_IMPLICIT)"); + prefixFlags |= PREFIX_TAILCALL_IMPLICIT; } + } - if (clone) - { - op2 = gtCloneExpr(op1); - } - else - { - const unsigned tmpNum = lvaGrabTemp(true DEBUGARG("dup spill")); - impAssignTempGen(tmpNum, op1, tiRetVal.GetClassHandle(), CHECK_SPILL_ALL); - var_types type = genActualType(lvaTable[tmpNum].TypeGet()); + // Treat this call as tail call for verification only if "tail" prefixed (i.e. explicit tail call). + explicitTailCall = (prefixFlags & PREFIX_TAILCALL_EXPLICIT) != 0; + readonlyCall = (prefixFlags & PREFIX_READONLY) != 0; - // Propagate type info to the temp from the stack and the original tree - if (type == TYP_REF) - { - assert(lvaTable[tmpNum].lvSingleDef == 0); - lvaTable[tmpNum].lvSingleDef = 1; - JITDUMP("Marked V%02u as a single def local\n", tmpNum); - lvaSetClass(tmpNum, tree, tiRetVal.GetClassHandle()); - } + if (opcode != CEE_CALLI && opcode != CEE_NEWOBJ) + { + // All calls and delegates need a security callout. + // For delegates, this is the call to the delegate constructor, not the access check on the + // LD(virt)FTN. + impHandleAccessAllowed(callInfo.accessAllowed, &callInfo.callsiteCalloutHelper); + } - op1 = gtNewLclvNode(tmpNum, type); - op2 = gtNewLclvNode(tmpNum, type); - } + callTyp = impImportCall(opcode, &resolvedToken, constraintCall ? &constrainedResolvedToken : nullptr, + newObjThisPtr, prefixFlags, &callInfo, opcodeOffs); + if (compDonotInline()) + { + // We do not check fails after lvaGrabTemp. It is covered with CoreCLR_13272 issue. + assert((callTyp == TYP_UNDEF) || + (compInlineResult->GetObservation() == InlineObservation::CALLSITE_TOO_MANY_LOCALS)); + return; } - else + + if (explicitTailCall || newBBcreatedForTailcallStress) // If newBBcreatedForTailcallStress is true, we + // have created a new BB after the "call" + // instruction in fgMakeBasicBlocks(). So we need to jump to RET regardless. { - op1 = impCloneExpr(op1, &op2, tiRetVal.GetClassHandle(), CHECK_SPILL_ALL, - nullptr DEBUGARG("DUP instruction")); + assert(!compIsForInlining()); + goto RET; } - assert(!(op1->gtFlags & GTF_GLOB_EFFECT) && !(op2->gtFlags & GTF_GLOB_EFFECT)); - impPushOnStack(op1, tiRetVal); - impPushOnStack(op2, tiRetVal); - } - break; + break; - case CEE_STIND_I1: - lclTyp = TYP_BYTE; - goto STIND; - case CEE_STIND_I2: - lclTyp = TYP_SHORT; - goto STIND; - case CEE_STIND_I4: - lclTyp = TYP_INT; - goto STIND; - case CEE_STIND_I8: - lclTyp = TYP_LONG; - goto STIND; - case CEE_STIND_I: - lclTyp = TYP_I_IMPL; - goto STIND; - case CEE_STIND_REF: - lclTyp = TYP_REF; - goto STIND; - case CEE_STIND_R4: - lclTyp = TYP_FLOAT; - goto STIND; - case CEE_STIND_R8: - lclTyp = TYP_DOUBLE; - goto STIND; + case CEE_LDFLD: + case CEE_LDSFLD: + case CEE_LDFLDA: + case CEE_LDSFLDA: + { - STIND: - op2 = impPopStack().val; // value to store + bool isLoadAddress = (opcode == CEE_LDFLDA || opcode == CEE_LDSFLDA); + bool isLoadStatic = (opcode == CEE_LDSFLD || opcode == CEE_LDSFLDA); - STIND_VALUE: - op1 = impPopStack().val; // address to store to + /* Get the CP_Fieldref index */ + assertImp(sz == sizeof(unsigned)); - // you can indirect off of a TYP_I_IMPL (if we are in C) or a BYREF - assertImp(genActualType(op1->gtType) == TYP_I_IMPL || op1->gtType == TYP_BYREF); + _impResolveToken(CORINFO_TOKENKIND_Field); - impBashVarAddrsToI(op1, op2); + JITDUMP(" %08X", resolvedToken.token); - op2 = impImplicitR4orR8Cast(op2, lclTyp); + int aflags = isLoadAddress ? CORINFO_ACCESS_ADDRESS : CORINFO_ACCESS_GET; -#ifdef TARGET_64BIT - // Automatic upcast for a GT_CNS_INT into TYP_I_IMPL - if ((op2->OperGet() == GT_CNS_INT) && varTypeIsI(lclTyp) && !varTypeIsI(op2->gtType)) - { - op2->gtType = TYP_I_IMPL; - } - else + GenTree* obj = nullptr; + typeInfo* tiObj = nullptr; + CORINFO_CLASS_HANDLE objType = nullptr; // used for fields + + if (opcode == CEE_LDFLD || opcode == CEE_LDFLDA) { - // Allow a downcast of op2 from TYP_I_IMPL into a 32-bit Int for x86 JIT compatibility - // - if (varTypeIsI(op2->gtType) && (genActualType(lclTyp) == TYP_INT)) - { - op2 = gtNewCastNode(TYP_INT, op2, false, TYP_INT); - } - // Allow an upcast of op2 from a 32-bit Int into TYP_I_IMPL for x86 JIT compatibility - // - if (varTypeIsI(lclTyp) && (genActualType(op2->gtType) == TYP_INT)) + tiObj = &impStackTop().seTypeInfo; + StackEntry se = impPopStack(); + objType = se.seTypeInfo.GetClassHandle(); + obj = se.val; + + if (impIsThis(obj)) { - op2 = gtNewCastNode(TYP_I_IMPL, op2, false, TYP_I_IMPL); + aflags |= CORINFO_ACCESS_THIS; } } -#endif // TARGET_64BIT - if (opcode == CEE_STIND_REF) - { - // STIND_REF can be used to store TYP_INT, TYP_I_IMPL, TYP_REF, or TYP_BYREF - assertImp(varTypeIsIntOrI(op2->gtType) || varTypeIsGC(op2->gtType)); - lclTyp = genActualType(op2->TypeGet()); - } + eeGetFieldInfo(&resolvedToken, (CORINFO_ACCESS_FLAGS)aflags, &fieldInfo); -// Check target type. -#ifdef DEBUG - if (op2->gtType == TYP_BYREF || lclTyp == TYP_BYREF) + // Figure out the type of the member. We always call canAccessField, so you always need this + // handle + CorInfoType ciType = fieldInfo.fieldType; + clsHnd = fieldInfo.structType; + + lclTyp = JITtype2varType(ciType); + + if (compIsForInlining()) { - if (op2->gtType == TYP_BYREF) + switch (fieldInfo.fieldAccessor) { - assertImp(lclTyp == TYP_BYREF || lclTyp == TYP_I_IMPL); + case CORINFO_FIELD_INSTANCE_HELPER: + case CORINFO_FIELD_INSTANCE_ADDR_HELPER: + case CORINFO_FIELD_STATIC_ADDR_HELPER: + case CORINFO_FIELD_STATIC_TLS: + + compInlineResult->NoteFatal(InlineObservation::CALLEE_LDFLD_NEEDS_HELPER); + return; + + case CORINFO_FIELD_STATIC_GENERICS_STATIC_HELPER: + case CORINFO_FIELD_STATIC_READYTORUN_HELPER: + /* We may be able to inline the field accessors in specific instantiations of generic + * methods */ + compInlineResult->NoteFatal(InlineObservation::CALLSITE_LDFLD_NEEDS_HELPER); + return; + + default: + break; } - else if (lclTyp == TYP_BYREF) + + if (!isLoadAddress && (fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC) && lclTyp == TYP_STRUCT && + clsHnd) { - assertImp(op2->gtType == TYP_BYREF || varTypeIsIntOrI(op2->gtType)); + if ((info.compCompHnd->getTypeForPrimitiveValueClass(clsHnd) == CORINFO_TYPE_UNDEF) && + !(info.compFlags & CORINFO_FLG_FORCEINLINE)) + { + // Loading a static valuetype field usually will cause a JitHelper to be called + // for the static base. This will bloat the code. + compInlineResult->Note(InlineObservation::CALLEE_LDFLD_STATIC_VALUECLASS); + + if (compInlineResult->IsFailure()) + { + return; + } + } } } - else - { - assertImp(genActualType(op2->gtType) == genActualType(lclTyp) || - ((lclTyp == TYP_I_IMPL) && (genActualType(op2->gtType) == TYP_INT)) || - (varTypeIsFloating(op2->gtType) && varTypeIsFloating(lclTyp))); - } -#endif - - op1 = gtNewOperNode(GT_IND, lclTyp, op1); - op1->gtFlags |= GTF_EXCEPT | GTF_GLOB_REF; - if (prefixFlags & PREFIX_VOLATILE) + tiRetVal = verMakeTypeInfo(ciType, clsHnd); + if (isLoadAddress) { - assert(op1->OperGet() == GT_IND); - op1->gtFlags |= GTF_ORDER_SIDEEFF; // Prevent this from being reordered - op1->gtFlags |= GTF_IND_VOLATILE; + tiRetVal.MakeByRef(); } - - if ((prefixFlags & PREFIX_UNALIGNED) && !varTypeIsByte(lclTyp)) + else { - assert(op1->OperGet() == GT_IND); - op1->gtFlags |= GTF_IND_UNALIGNED; + tiRetVal.NormaliseForStack(); } - op1 = gtNewAssignNode(op1, op2); - goto SPILL_APPEND; - - case CEE_LDIND_I1: - lclTyp = TYP_BYTE; - goto LDIND; - case CEE_LDIND_I2: - lclTyp = TYP_SHORT; - goto LDIND; - case CEE_LDIND_U4: - case CEE_LDIND_I4: - lclTyp = TYP_INT; - goto LDIND; - case CEE_LDIND_I8: - lclTyp = TYP_LONG; - goto LDIND; - case CEE_LDIND_REF: - lclTyp = TYP_REF; - goto LDIND; - case CEE_LDIND_I: - lclTyp = TYP_I_IMPL; - goto LDIND; - case CEE_LDIND_R4: - lclTyp = TYP_FLOAT; - goto LDIND; - case CEE_LDIND_R8: - lclTyp = TYP_DOUBLE; - goto LDIND; - case CEE_LDIND_U1: - lclTyp = TYP_UBYTE; - goto LDIND; - case CEE_LDIND_U2: - lclTyp = TYP_USHORT; - goto LDIND; - LDIND: - - op1 = impPopStack().val; // address to load from - impBashVarAddrsToI(op1); + // Perform this check always to ensure that we get field access exceptions even with + // SkipVerification. + impHandleAccessAllowed(fieldInfo.accessAllowed, &fieldInfo.accessCalloutHelper); -#ifdef TARGET_64BIT - // Allow an upcast of op1 from a 32-bit Int into TYP_I_IMPL for x86 JIT compatibility - // - if (genActualType(op1->gtType) == TYP_INT) + // Raise InvalidProgramException if static load accesses non-static field + if (isLoadStatic && ((fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC) == 0)) { - op1 = gtNewCastNode(TYP_I_IMPL, op1, false, TYP_I_IMPL); + BADCODE("static access on an instance field"); } -#endif - - assertImp(genActualType(op1->gtType) == TYP_I_IMPL || op1->gtType == TYP_BYREF); - - op1 = gtNewOperNode(GT_IND, lclTyp, op1); - - // ldind could point anywhere, example a boxed class static int - op1->gtFlags |= (GTF_EXCEPT | GTF_GLOB_REF); - if (prefixFlags & PREFIX_VOLATILE) + // We are using ldfld/a on a static field. We allow it, but need to get side-effect from obj. + if ((fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC) && obj != nullptr) { - assert(op1->OperGet() == GT_IND); - op1->gtFlags |= GTF_ORDER_SIDEEFF; // Prevent this from being reordered - op1->gtFlags |= GTF_IND_VOLATILE; + if (obj->gtFlags & GTF_SIDE_EFFECT) + { + obj = gtUnusedValNode(obj); + impAppendTree(obj, (unsigned)CHECK_SPILL_ALL, impCurStmtDI); + } + obj = nullptr; } - if ((prefixFlags & PREFIX_UNALIGNED) && !varTypeIsByte(lclTyp)) + /* Preserve 'small' int types */ + if (!varTypeIsSmall(lclTyp)) { - assert(op1->OperGet() == GT_IND); - op1->gtFlags |= GTF_IND_UNALIGNED; + lclTyp = genActualType(lclTyp); } - impPushOnStack(op1, tiRetVal); - - break; - - case CEE_UNALIGNED: + bool usesHelper = false; - assert(sz == 1); - val = getU1LittleEndian(codeAddr); - ++codeAddr; - JITDUMP(" %u", val); - if ((val != 1) && (val != 2) && (val != 4)) + switch (fieldInfo.fieldAccessor) { - BADCODE("Alignment unaligned. must be 1, 2, or 4"); - } - - Verify(!(prefixFlags & PREFIX_UNALIGNED), "Multiple unaligned. prefixes"); - prefixFlags |= PREFIX_UNALIGNED; - - impValidateMemoryAccessOpcode(codeAddr, codeEndp, false); - - PREFIX: - opcode = (OPCODE)getU1LittleEndian(codeAddr); - opcodeOffs = (IL_OFFSET)(codeAddr - info.compCode); - codeAddr += sizeof(__int8); - goto DECODE_OPCODE; + case CORINFO_FIELD_INSTANCE: +#ifdef FEATURE_READYTORUN + case CORINFO_FIELD_INSTANCE_WITH_BASE: +#endif + { + // If the object is a struct, what we really want is + // for the field to operate on the address of the struct. + if (!varTypeGCtype(obj->TypeGet()) && impIsValueType(tiObj)) + { + assert(opcode == CEE_LDFLD && objType != nullptr); - case CEE_VOLATILE: + obj = impGetStructAddr(obj, objType, (unsigned)CHECK_SPILL_ALL, true); + } - Verify(!(prefixFlags & PREFIX_VOLATILE), "Multiple volatile. prefixes"); - prefixFlags |= PREFIX_VOLATILE; + /* Create the data member node */ + op1 = gtNewFieldRef(lclTyp, resolvedToken.hField, obj, fieldInfo.offset); - impValidateMemoryAccessOpcode(codeAddr, codeEndp, true); +#ifdef FEATURE_READYTORUN + if (fieldInfo.fieldAccessor == CORINFO_FIELD_INSTANCE_WITH_BASE) + { + op1->AsField()->gtFieldLookup = fieldInfo.fieldLookup; + } +#endif - assert(sz == 0); - goto PREFIX; + if (fgAddrCouldBeNull(obj)) + { + op1->gtFlags |= GTF_EXCEPT; + } - case CEE_LDFTN: - { - // Need to do a lookup here so that we perform an access check - // and do a NOWAY if protections are violated - _impResolveToken(CORINFO_TOKENKIND_Method); + DWORD typeFlags = info.compCompHnd->getClassAttribs(resolvedToken.hClass); + if (StructHasOverlappingFields(typeFlags)) + { + op1->AsField()->gtFldMayOverlap = true; + } - JITDUMP(" %08X", resolvedToken.token); + // wrap it in a address of operator if necessary + if (isLoadAddress) + { + op1 = gtNewOperNode(GT_ADDR, + (var_types)(varTypeIsGC(obj->TypeGet()) ? TYP_BYREF : TYP_I_IMPL), op1); + } + else + { + if (compIsForInlining() && + impInlineIsGuaranteedThisDerefBeforeAnySideEffects(nullptr, nullptr, obj, + impInlineInfo->inlArgInfo)) + { + impInlineInfo->thisDereferencedFirst = true; + } + } + } + break; - eeGetCallInfo(&resolvedToken, (prefixFlags & PREFIX_CONSTRAINED) ? &constrainedResolvedToken : nullptr, - combine(CORINFO_CALLINFO_SECURITYCHECKS, CORINFO_CALLINFO_LDFTN), &callInfo); + case CORINFO_FIELD_STATIC_TLS: +#ifdef TARGET_X86 + // Legacy TLS access is implemented as intrinsic on x86 only - // This check really only applies to intrinsic Array.Address methods - if (callInfo.sig.callConv & CORINFO_CALLCONV_PARAMTYPE) - { - NO_WAY("Currently do not support LDFTN of Parameterized functions"); - } + /* Create the data member node */ + op1 = gtNewFieldRef(lclTyp, resolvedToken.hField, NULL, fieldInfo.offset); + op1->gtFlags |= GTF_IND_TLS_REF; // fgMorphField will handle the transformation - // Do this before DO_LDFTN since CEE_LDVIRTFN does it on its own. - impHandleAccessAllowed(callInfo.accessAllowed, &callInfo.callsiteCalloutHelper); + if (isLoadAddress) + { + op1 = gtNewOperNode(GT_ADDR, (var_types)TYP_I_IMPL, op1); + } + break; +#else + fieldInfo.fieldAccessor = CORINFO_FIELD_STATIC_ADDR_HELPER; - DO_LDFTN: - op1 = impMethodPointer(&resolvedToken, &callInfo); + FALLTHROUGH; +#endif - if (compDonotInline()) - { - return; - } + case CORINFO_FIELD_STATIC_ADDR_HELPER: + case CORINFO_FIELD_INSTANCE_HELPER: + case CORINFO_FIELD_INSTANCE_ADDR_HELPER: + op1 = gtNewRefCOMfield(obj, &resolvedToken, (CORINFO_ACCESS_FLAGS)aflags, &fieldInfo, lclTyp, + clsHnd, nullptr); + usesHelper = true; + break; - // Call info may have more precise information about the function than - // the resolved token. - mdToken constrainedToken = prefixFlags & PREFIX_CONSTRAINED ? constrainedResolvedToken.token : 0; - methodPointerInfo* heapToken = impAllocateMethodPointerInfo(resolvedToken, constrainedToken); - assert(callInfo.hMethod != nullptr); - heapToken->m_token.hMethod = callInfo.hMethod; - impPushOnStack(op1, typeInfo(heapToken)); + case CORINFO_FIELD_STATIC_ADDRESS: + // Replace static read-only fields with constant if possible + if ((aflags & CORINFO_ACCESS_GET) && (fieldInfo.fieldFlags & CORINFO_FLG_FIELD_FINAL) && + !(fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC_IN_HEAP) && + (varTypeIsIntegral(lclTyp) || varTypeIsFloating(lclTyp))) + { + CorInfoInitClassResult initClassResult = + info.compCompHnd->initClass(resolvedToken.hField, info.compMethodHnd, + impTokenLookupContextHandle); - break; - } + if (initClassResult & CORINFO_INITCLASS_INITIALIZED) + { + void** pFldAddr = nullptr; + void* fldAddr = + info.compCompHnd->getFieldAddress(resolvedToken.hField, (void**)&pFldAddr); - case CEE_LDVIRTFTN: - { - /* Get the method token */ + // We should always be able to access this static's address directly + // + assert(pFldAddr == nullptr); - _impResolveToken(CORINFO_TOKENKIND_Method); + op1 = impImportStaticReadOnlyField(fldAddr, lclTyp); - JITDUMP(" %08X", resolvedToken.token); + // Widen small types since we're propagating the value + // instead of producing an indir. + // + op1->gtType = genActualType(lclTyp); - eeGetCallInfo(&resolvedToken, nullptr /* constraint typeRef */, - combine(combine(CORINFO_CALLINFO_SECURITYCHECKS, CORINFO_CALLINFO_LDFTN), - CORINFO_CALLINFO_CALLVIRT), - &callInfo); + goto FIELD_DONE; + } + } - // This check really only applies to intrinsic Array.Address methods - if (callInfo.sig.callConv & CORINFO_CALLCONV_PARAMTYPE) - { - NO_WAY("Currently do not support LDFTN of Parameterized functions"); - } + FALLTHROUGH; - mflags = callInfo.methodFlags; + case CORINFO_FIELD_STATIC_RVA_ADDRESS: + case CORINFO_FIELD_STATIC_SHARED_STATIC_HELPER: + case CORINFO_FIELD_STATIC_GENERICS_STATIC_HELPER: + case CORINFO_FIELD_STATIC_READYTORUN_HELPER: + op1 = impImportStaticFieldAccess(&resolvedToken, (CORINFO_ACCESS_FLAGS)aflags, &fieldInfo, + lclTyp); + break; - impHandleAccessAllowed(callInfo.accessAllowed, &callInfo.callsiteCalloutHelper); + case CORINFO_FIELD_INTRINSIC_ZERO: + { + assert(aflags & CORINFO_ACCESS_GET); + // Widen to stack type + lclTyp = genActualType(lclTyp); + op1 = gtNewIconNode(0, lclTyp); + goto FIELD_DONE; + } + break; - if (compIsForInlining()) - { - if (mflags & (CORINFO_FLG_FINAL | CORINFO_FLG_STATIC) || !(mflags & CORINFO_FLG_VIRTUAL)) + case CORINFO_FIELD_INTRINSIC_EMPTY_STRING: { - compInlineResult->NoteFatal(InlineObservation::CALLSITE_LDVIRTFN_ON_NON_VIRTUAL); - return; + assert(aflags & CORINFO_ACCESS_GET); + + // Import String.Empty as "" (GT_CNS_STR with a fake SconCPX = 0) + op1 = gtNewSconNode(EMPTY_STRING_SCON, nullptr); + goto FIELD_DONE; } - } + break; - CORINFO_SIG_INFO& ftnSig = callInfo.sig; + case CORINFO_FIELD_INTRINSIC_ISLITTLEENDIAN: + { + assert(aflags & CORINFO_ACCESS_GET); + // Widen to stack type + lclTyp = genActualType(lclTyp); +#if BIGENDIAN + op1 = gtNewIconNode(0, lclTyp); +#else + op1 = gtNewIconNode(1, lclTyp); +#endif + goto FIELD_DONE; + } + break; - /* Get the object-ref */ - op1 = impPopStack().val; - assertImp(op1->gtType == TYP_REF); + default: + assert(!"Unexpected fieldAccessor"); + } - if (opts.IsReadyToRun()) + if (!isLoadAddress) { - if (callInfo.kind != CORINFO_VIRTUALCALL_LDVIRTFTN) + + if (prefixFlags & PREFIX_VOLATILE) { - if (op1->gtFlags & GTF_SIDE_EFFECT) + op1->gtFlags |= GTF_ORDER_SIDEEFF; // Prevent this from being reordered + + if (!usesHelper) { - op1 = gtUnusedValNode(op1); - impAppendTree(op1, CHECK_SPILL_ALL, impCurStmtDI); + assert((op1->OperGet() == GT_FIELD) || (op1->OperGet() == GT_IND) || + (op1->OperGet() == GT_OBJ)); + op1->gtFlags |= GTF_IND_VOLATILE; } - goto DO_LDFTN; } - } - else if (mflags & (CORINFO_FLG_FINAL | CORINFO_FLG_STATIC) || !(mflags & CORINFO_FLG_VIRTUAL)) - { - if (op1->gtFlags & GTF_SIDE_EFFECT) + + if ((prefixFlags & PREFIX_UNALIGNED) && !varTypeIsByte(lclTyp)) { - op1 = gtUnusedValNode(op1); - impAppendTree(op1, CHECK_SPILL_ALL, impCurStmtDI); + if (!usesHelper) + { + assert((op1->OperGet() == GT_FIELD) || (op1->OperGet() == GT_IND) || + (op1->OperGet() == GT_OBJ)); + op1->gtFlags |= GTF_IND_UNALIGNED; + } } - goto DO_LDFTN; } - GenTree* fptr = impImportLdvirtftn(op1, &resolvedToken, &callInfo); - if (compDonotInline()) + /* Check if the class needs explicit initialization */ + + if (fieldInfo.fieldFlags & CORINFO_FLG_FIELD_INITCLASS) { - return; + GenTree* helperNode = impInitClass(&resolvedToken); + if (compDonotInline()) + { + return; + } + if (helperNode != nullptr) + { + op1 = gtNewOperNode(GT_COMMA, op1->TypeGet(), helperNode, op1); + } } - methodPointerInfo* heapToken = impAllocateMethodPointerInfo(resolvedToken, 0); + FIELD_DONE: + impPushOnStack(op1, tiRetVal); + } + break; - assert(heapToken->m_token.tokenType == CORINFO_TOKENKIND_Method); - assert(callInfo.hMethod != nullptr); + case CEE_STFLD: + case CEE_STSFLD: + { - heapToken->m_token.tokenType = CORINFO_TOKENKIND_Ldvirtftn; - heapToken->m_token.hMethod = callInfo.hMethod; - impPushOnStack(fptr, typeInfo(heapToken)); + bool isStoreStatic = (opcode == CEE_STSFLD); - break; - } + CORINFO_CLASS_HANDLE fieldClsHnd; // class of the field (if it's a ref type) - case CEE_CONSTRAINED: + /* Get the CP_Fieldref index */ assertImp(sz == sizeof(unsigned)); - impResolveToken(codeAddr, &constrainedResolvedToken, CORINFO_TOKENKIND_Constrained); - codeAddr += sizeof(unsigned); // prefix instructions must increment codeAddr manually - JITDUMP(" (%08X) ", constrainedResolvedToken.token); - Verify(!(prefixFlags & PREFIX_CONSTRAINED), "Multiple constrained. prefixes"); - prefixFlags |= PREFIX_CONSTRAINED; + _impResolveToken(CORINFO_TOKENKIND_Field); + + JITDUMP(" %08X", resolvedToken.token); + + int aflags = CORINFO_ACCESS_SET; + GenTree* obj = nullptr; + typeInfo* tiObj = nullptr; + typeInfo tiVal; + + /* Pull the value from the stack */ + StackEntry se = impPopStack(); + op2 = se.val; + tiVal = se.seTypeInfo; + clsHnd = tiVal.GetClassHandle(); + if (opcode == CEE_STFLD) { - OPCODE actualOpcode = impGetNonPrefixOpcode(codeAddr, codeEndp); - if (actualOpcode != CEE_CALLVIRT && actualOpcode != CEE_CALL && actualOpcode != CEE_LDFTN) + tiObj = &impStackTop().seTypeInfo; + obj = impPopStack().val; + + if (impIsThis(obj)) { - BADCODE("constrained. has to be followed by callvirt, call or ldftn"); + aflags |= CORINFO_ACCESS_THIS; } } - goto PREFIX; + eeGetFieldInfo(&resolvedToken, (CORINFO_ACCESS_FLAGS)aflags, &fieldInfo); - case CEE_READONLY: - JITDUMP(" readonly."); + // Figure out the type of the member. We always call canAccessField, so you always need this + // handle + CorInfoType ciType = fieldInfo.fieldType; + fieldClsHnd = fieldInfo.structType; - Verify(!(prefixFlags & PREFIX_READONLY), "Multiple readonly. prefixes"); - prefixFlags |= PREFIX_READONLY; + lclTyp = JITtype2varType(ciType); + if (compIsForInlining()) { - OPCODE actualOpcode = impGetNonPrefixOpcode(codeAddr, codeEndp); - if (actualOpcode != CEE_LDELEMA && !impOpcodeIsCallOpcode(actualOpcode)) + /* Is this a 'special' (COM) field? or a TLS ref static field?, field stored int GC heap? or + * per-inst static? */ + + switch (fieldInfo.fieldAccessor) { - BADCODE("readonly. has to be followed by ldelema or call"); + case CORINFO_FIELD_INSTANCE_HELPER: + case CORINFO_FIELD_INSTANCE_ADDR_HELPER: + case CORINFO_FIELD_STATIC_ADDR_HELPER: + case CORINFO_FIELD_STATIC_TLS: + + compInlineResult->NoteFatal(InlineObservation::CALLEE_STFLD_NEEDS_HELPER); + return; + + case CORINFO_FIELD_STATIC_GENERICS_STATIC_HELPER: + case CORINFO_FIELD_STATIC_READYTORUN_HELPER: + /* We may be able to inline the field accessors in specific instantiations of generic + * methods */ + compInlineResult->NoteFatal(InlineObservation::CALLSITE_STFLD_NEEDS_HELPER); + return; + + default: + break; } } - assert(sz == 0); - goto PREFIX; - - case CEE_TAILCALL: - JITDUMP(" tail."); + impHandleAccessAllowed(fieldInfo.accessAllowed, &fieldInfo.accessCalloutHelper); - Verify(!(prefixFlags & PREFIX_TAILCALL_EXPLICIT), "Multiple tailcall. prefixes"); - prefixFlags |= PREFIX_TAILCALL_EXPLICIT; + // Raise InvalidProgramException if static store accesses non-static field + if (isStoreStatic && ((fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC) == 0)) + { + BADCODE("static access on an instance field"); + } + // We are using stfld on a static field. + // We allow it, but need to eval any side-effects for obj + if ((fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC) && obj != nullptr) { - OPCODE actualOpcode = impGetNonPrefixOpcode(codeAddr, codeEndp); - if (!impOpcodeIsCallOpcode(actualOpcode)) + if (obj->gtFlags & GTF_SIDE_EFFECT) { - BADCODE("tailcall. has to be followed by call, callvirt or calli"); + obj = gtUnusedValNode(obj); + impAppendTree(obj, (unsigned)CHECK_SPILL_ALL, impCurStmtDI); } + obj = nullptr; } - assert(sz == 0); - goto PREFIX; - case CEE_NEWOBJ: + /* Preserve 'small' int types */ + if (!varTypeIsSmall(lclTyp)) + { + lclTyp = genActualType(lclTyp); + } + + switch (fieldInfo.fieldAccessor) + { + case CORINFO_FIELD_INSTANCE: +#ifdef FEATURE_READYTORUN + case CORINFO_FIELD_INSTANCE_WITH_BASE: +#endif + { + /* Create the data member node */ + op1 = gtNewFieldRef(lclTyp, resolvedToken.hField, obj, fieldInfo.offset); + DWORD typeFlags = info.compCompHnd->getClassAttribs(resolvedToken.hClass); + if (StructHasOverlappingFields(typeFlags)) + { + op1->AsField()->gtFldMayOverlap = true; + } + +#ifdef FEATURE_READYTORUN + if (fieldInfo.fieldAccessor == CORINFO_FIELD_INSTANCE_WITH_BASE) + { + op1->AsField()->gtFieldLookup = fieldInfo.fieldLookup; + } +#endif + + if (fgAddrCouldBeNull(obj)) + { + op1->gtFlags |= GTF_EXCEPT; + } + + if (compIsForInlining() && + impInlineIsGuaranteedThisDerefBeforeAnySideEffects(op2, nullptr, obj, + impInlineInfo->inlArgInfo)) + { + impInlineInfo->thisDereferencedFirst = true; + } + } + break; - /* Since we will implicitly insert newObjThisPtr at the start of the - argument list, spill any GTF_ORDER_SIDEEFF */ - impSpillSpecialSideEff(); + case CORINFO_FIELD_STATIC_TLS: +#ifdef TARGET_X86 + // Legacy TLS access is implemented as intrinsic on x86 only - /* NEWOBJ does not respond to TAIL */ - prefixFlags &= ~PREFIX_TAILCALL_EXPLICIT; + /* Create the data member node */ + op1 = gtNewFieldRef(lclTyp, resolvedToken.hField, NULL, fieldInfo.offset); + op1->gtFlags |= GTF_IND_TLS_REF; // fgMorphField will handle the transformation - /* NEWOBJ does not respond to CONSTRAINED */ - prefixFlags &= ~PREFIX_CONSTRAINED; + break; +#else + fieldInfo.fieldAccessor = CORINFO_FIELD_STATIC_ADDR_HELPER; - _impResolveToken(CORINFO_TOKENKIND_NewObj); + FALLTHROUGH; +#endif - eeGetCallInfo(&resolvedToken, nullptr /* constraint typeRef*/, - combine(CORINFO_CALLINFO_SECURITYCHECKS, CORINFO_CALLINFO_ALLOWINSTPARAM), &callInfo); + case CORINFO_FIELD_STATIC_ADDR_HELPER: + case CORINFO_FIELD_INSTANCE_HELPER: + case CORINFO_FIELD_INSTANCE_ADDR_HELPER: + op1 = gtNewRefCOMfield(obj, &resolvedToken, (CORINFO_ACCESS_FLAGS)aflags, &fieldInfo, lclTyp, + clsHnd, op2); + goto SPILL_APPEND; - mflags = callInfo.methodFlags; + case CORINFO_FIELD_STATIC_ADDRESS: + case CORINFO_FIELD_STATIC_RVA_ADDRESS: + case CORINFO_FIELD_STATIC_SHARED_STATIC_HELPER: + case CORINFO_FIELD_STATIC_GENERICS_STATIC_HELPER: + case CORINFO_FIELD_STATIC_READYTORUN_HELPER: + op1 = impImportStaticFieldAccess(&resolvedToken, (CORINFO_ACCESS_FLAGS)aflags, &fieldInfo, + lclTyp); + break; - if ((mflags & (CORINFO_FLG_STATIC | CORINFO_FLG_ABSTRACT)) != 0) - { - BADCODE("newobj on static or abstract method"); + default: + assert(!"Unexpected fieldAccessor"); } - // Insert the security callout before any actual code is generated - impHandleAccessAllowed(callInfo.accessAllowed, &callInfo.callsiteCalloutHelper); - - // There are three different cases for new. - // Object size is variable (depends on arguments). - // 1) Object is an array (arrays treated specially by the EE) - // 2) Object is some other variable sized object (e.g. String) - // 3) Class Size can be determined beforehand (normal case) - // In the first case, we need to call a NEWOBJ helper (multinewarray). - // In the second case we call the constructor with a '0' this pointer. - // In the third case we alloc the memory, then call the constructor. + // "impAssignStruct" will back-substitute the field address tree into calls that return things via + // return buffers, so we have to delay calling it until after we have spilled everything needed. + bool deferStructAssign = (lclTyp == TYP_STRUCT); - clsFlags = callInfo.classFlags; - if (clsFlags & CORINFO_FLG_ARRAY) + if (!deferStructAssign) { - // Arrays need to call the NEWOBJ helper. - assertImp(clsFlags & CORINFO_FLG_VAROBJSIZE); + assert(op1->OperIs(GT_FIELD, GT_IND)); - impImportNewObjArray(&resolvedToken, &callInfo); - if (compDonotInline()) + if (prefixFlags & PREFIX_VOLATILE) { - return; + op1->gtFlags |= GTF_ORDER_SIDEEFF; // Prevent this from being reordered + op1->gtFlags |= GTF_IND_VOLATILE; + } + if ((prefixFlags & PREFIX_UNALIGNED) && !varTypeIsByte(lclTyp)) + { + op1->gtFlags |= GTF_IND_UNALIGNED; } - callTyp = TYP_REF; - break; - } - // At present this can only be String - else if (clsFlags & CORINFO_FLG_VAROBJSIZE) - { - // Skip this thisPtr argument - newObjThisPtr = nullptr; + // Currently, *all* TYP_REF statics are stored inside an "object[]" array that itself + // resides on the managed heap, and so we can use an unchecked write barrier for this + // store. Likewise if we're storing to a field of an on-heap object. + if ((lclTyp == TYP_REF) && + (((fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC) != 0) || obj->TypeIs(TYP_REF))) + { + static_assert_no_msg(GTF_FLD_TGT_HEAP == GTF_IND_TGT_HEAP); + op1->gtFlags |= GTF_FLD_TGT_HEAP; + } - /* Remember that this basic block contains 'new' of an object */ - block->bbFlags |= BBF_HAS_NEWOBJ; - optMethodFlags |= OMF_HAS_NEWOBJ; - } - else - { - // This is the normal case where the size of the object is - // fixed. Allocate the memory and call the constructor. + /* V4.0 allows assignment of i4 constant values to i8 type vars when IL verifier is bypassed (full + trust apps). The reason this works is that JIT stores an i4 constant in Gentree union during + importation and reads from the union as if it were a long during code generation. Though this + can potentially read garbage, one can get lucky to have this working correctly. - // Note: We cannot add a peep to avoid use of temp here - // because we don't have enough interference info to detect when - // sources and destination interfere, example: s = new S(ref); + This code pattern is generated by Dev10 MC++ compiler while storing to fields when compiled with + /O2 switch (default when compiling retail configs in Dev10) and a customer app has taken a + dependency on it. To be backward compatible, we will explicitly add an upward cast here so that + it works correctly always. - // TODO: We find the correct place to introduce a general - // reverse copy prop for struct return values from newobj or - // any function returning structs. + Note that this is limited to x86 alone as there is no back compat to be addressed for Arm JIT + for V4.0. + */ + CLANG_FORMAT_COMMENT_ANCHOR; - /* get a temporary for the new object */ - lclNum = lvaGrabTemp(true DEBUGARG("NewObj constructor temp")); - if (compDonotInline()) +#ifndef TARGET_64BIT + // In UWP6.0 and beyond (post-.NET Core 2.0), we decided to let this cast from int to long be + // generated for ARM as well as x86, so the following IR will be accepted: + // STMTx (IL 0x... ???) + // * ASG long + // +--* LCL_VAR long + // \--* CNS_INT int 2 + + if ((op1->TypeGet() != op2->TypeGet()) && op2->OperIsConst() && varTypeIsIntOrI(op2->TypeGet()) && + varTypeIsLong(op1->TypeGet())) { - // Fail fast if lvaGrabTemp fails with CALLSITE_TOO_MANY_LOCALS. - assert(compInlineResult->GetObservation() == InlineObservation::CALLSITE_TOO_MANY_LOCALS); - return; + op2 = gtNewCastNode(op1->TypeGet(), op2, false, op1->TypeGet()); } +#endif - // In the value class case we only need clsHnd for size calcs. - // - // The lookup of the code pointer will be handled by CALL in this case - if (clsFlags & CORINFO_FLG_VALUECLASS) +#ifdef TARGET_64BIT + // Automatic upcast for a GT_CNS_INT into TYP_I_IMPL + if ((op2->OperGet() == GT_CNS_INT) && varTypeIsI(lclTyp) && !varTypeIsI(op2->gtType)) { - if (compIsForInlining()) - { - // If value class has GC fields, inform the inliner. It may choose to - // bail out on the inline. - DWORD typeFlags = info.compCompHnd->getClassAttribs(resolvedToken.hClass); - if ((typeFlags & CORINFO_FLG_CONTAINS_GC_PTR) != 0) - { - compInlineResult->Note(InlineObservation::CALLEE_HAS_GC_STRUCT); - if (compInlineResult->IsFailure()) - { - return; - } - - // Do further notification in the case where the call site is rare; - // some policies do not track the relative hotness of call sites for - // "always" inline cases. - if (impInlineInfo->iciBlock->isRunRarely()) - { - compInlineResult->Note(InlineObservation::CALLSITE_RARE_GC_STRUCT); - if (compInlineResult->IsFailure()) - { - return; - } - } - } - } - - CorInfoType jitTyp = info.compCompHnd->asCorInfoType(resolvedToken.hClass); - - if (impIsPrimitive(jitTyp)) + op2->gtType = TYP_I_IMPL; + } + else + { + // Allow a downcast of op2 from TYP_I_IMPL into a 32-bit Int for x86 JIT compatiblity + // + if (varTypeIsI(op2->gtType) && (genActualType(lclTyp) == TYP_INT)) { - lvaTable[lclNum].lvType = JITtype2varType(jitTyp); + op2 = gtNewCastNode(TYP_INT, op2, false, TYP_INT); } - else + // Allow an upcast of op2 from a 32-bit Int into TYP_I_IMPL for x86 JIT compatiblity + // + if (varTypeIsI(lclTyp) && (genActualType(op2->gtType) == TYP_INT)) { - // The local variable itself is the allocated space. - // Here we need unsafe value cls check, since the address of struct is taken for further use - // and potentially exploitable. - lvaSetStruct(lclNum, resolvedToken.hClass, true /* unsafe value cls check */); + op2 = gtNewCastNode(TYP_I_IMPL, op2, false, TYP_I_IMPL); } + } +#endif - bool bbInALoop = impBlockIsInALoop(block); - bool bbIsReturn = (block->bbJumpKind == BBJ_RETURN) && - (!compIsForInlining() || (impInlineInfo->iciBlock->bbJumpKind == BBJ_RETURN)); - LclVarDsc* const lclDsc = lvaGetDesc(lclNum); - if (fgVarNeedsExplicitZeroInit(lclNum, bbInALoop, bbIsReturn)) - { - // Append a tree to zero-out the temp - GenTree* newObjDst = gtNewLclvNode(lclNum, lclDsc->TypeGet()); - GenTree* newObjInit; - if (lclDsc->TypeGet() == TYP_STRUCT) - { - newObjInit = gtNewBlkOpNode(newObjDst, gtNewIconNode(0)); - } - else - { - newObjInit = gtNewAssignNode(newObjDst, gtNewZeroConNode(lclDsc->TypeGet())); - } - impAppendTree(newObjInit, CHECK_SPILL_NONE, impCurStmtDI); - } - else - { - JITDUMP("\nSuppressing zero-init for V%02u -- expect to zero in prolog\n", lclNum); - lclDsc->lvSuppressedZeroInit = 1; - compSuppressedZeroInit = true; - } + // We can generate an assignment to a TYP_FLOAT from a TYP_DOUBLE + // We insert a cast to the dest 'op1' type + // + if ((op1->TypeGet() != op2->TypeGet()) && varTypeIsFloating(op1->gtType) && + varTypeIsFloating(op2->gtType)) + { + op2 = gtNewCastNode(op1->TypeGet(), op2, false, op1->TypeGet()); + } - // The constructor may store "this", with subsequent code mutating the underlying local - // through the captured reference. To correctly spill the node we'll push onto the stack - // in such a case, we must mark the temp as potentially aliased. - lclDsc->lvHasLdAddrOp = true; + op1 = gtNewAssignNode(op1, op2); + } - // Obtain the address of the temp - newObjThisPtr = gtNewLclVarAddrNode(lclNum, TYP_BYREF); + /* Check if the class needs explicit initialization */ + + if (fieldInfo.fieldFlags & CORINFO_FLG_FIELD_INITCLASS) + { + GenTree* helperNode = impInitClass(&resolvedToken); + if (compDonotInline()) + { + return; } - else + if (helperNode != nullptr) { - // If we're newing up a finalizable object, spill anything that can cause exceptions. - // - bool hasSideEffects = false; - CorInfoHelpFunc newHelper = - info.compCompHnd->getNewHelper(&resolvedToken, info.compMethodHnd, &hasSideEffects); - - if (hasSideEffects) - { - JITDUMP("\nSpilling stack for finalizable newobj\n"); - impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("finalizable newobj spill")); - } + op1 = gtNewOperNode(GT_COMMA, op1->TypeGet(), helperNode, op1); + } + } - const bool useParent = true; - op1 = gtNewAllocObjNode(&resolvedToken, useParent); - if (op1 == nullptr) - { - return; - } + // An indirect store such as "st[s]fld" interferes with indirect accesses, so we must spill + // global refs and potentially aliased locals. + impSpillSideEffects(true, (unsigned)CHECK_SPILL_ALL DEBUGARG("spill side effects before STFLD")); - // Remember that this basic block contains 'new' of an object - block->bbFlags |= BBF_HAS_NEWOBJ; - optMethodFlags |= OMF_HAS_NEWOBJ; + if (deferStructAssign) + { + op1 = impAssignStruct(op1, op2, clsHnd, (unsigned)CHECK_SPILL_ALL); + } + } + goto APPEND; - // Append the assignment to the temp/local. Dont need to spill - // at all as we are just calling an EE-Jit helper which can only - // cause an (async) OutOfMemoryException. + case CEE_NEWARR: + { - // We assign the newly allocated object (by a GT_ALLOCOBJ node) - // to a temp. Note that the pattern "temp = allocObj" is required - // by ObjectAllocator phase to be able to determine GT_ALLOCOBJ nodes - // without exhaustive walk over all expressions. + /* Get the class type index operand */ - impAssignTempGen(lclNum, op1, CHECK_SPILL_NONE); + _impResolveToken(CORINFO_TOKENKIND_Newarr); - assert(lvaTable[lclNum].lvSingleDef == 0); - lvaTable[lclNum].lvSingleDef = 1; - JITDUMP("Marked V%02u as a single def local\n", lclNum); - lvaSetClass(lclNum, resolvedToken.hClass, true /* is Exact */); + JITDUMP(" %08X", resolvedToken.token); - newObjThisPtr = gtNewLclvNode(lclNum, TYP_REF); + if (!opts.IsReadyToRun()) + { + // Need to restore array classes before creating array objects on the heap + op1 = impTokenToHandle(&resolvedToken, nullptr, true /*mustRestoreHandle*/); + if (op1 == nullptr) + { // compDonotInline() + return; } } - goto CALL; - case CEE_CALLI: + tiRetVal = verMakeTypeInfo(resolvedToken.hClass); - /* CALLI does not respond to CONSTRAINED */ - prefixFlags &= ~PREFIX_CONSTRAINED; + accessAllowedResult = + info.compCompHnd->canAccessClass(&resolvedToken, info.compMethodHnd, &calloutHelper); + impHandleAccessAllowed(accessAllowedResult, &calloutHelper); - FALLTHROUGH; + /* Form the arglist: array class handle, size */ + op2 = impPopStack().val; + assertImp(genActualTypeIsIntOrI(op2->gtType)); - case CEE_CALLVIRT: - case CEE_CALL: +#ifdef TARGET_64BIT + // The array helper takes a native int for array length. + // So if we have an int, explicitly extend it to be a native int. + if (genActualType(op2->TypeGet()) != TYP_I_IMPL) + { + if (op2->IsIntegralConst()) + { + op2->gtType = TYP_I_IMPL; + } + else + { + bool isUnsigned = false; + op2 = gtNewCastNode(TYP_I_IMPL, op2, isUnsigned, TYP_I_IMPL); + } + } +#endif // TARGET_64BIT - // We can't call getCallInfo on the token from a CALLI, but we need it in - // many other places. We unfortunately embed that knowledge here. - if (opcode != CEE_CALLI) +#ifdef FEATURE_READYTORUN + if (opts.IsReadyToRun()) { - _impResolveToken(CORINFO_TOKENKIND_Method); + op1 = impReadyToRunHelperToTree(&resolvedToken, CORINFO_HELP_READYTORUN_NEWARR_1, TYP_REF, nullptr, + op2); + usingReadyToRunHelper = (op1 != nullptr); - eeGetCallInfo(&resolvedToken, - (prefixFlags & PREFIX_CONSTRAINED) ? &constrainedResolvedToken : nullptr, - // this is how impImportCall invokes getCallInfo - combine(combine(CORINFO_CALLINFO_ALLOWINSTPARAM, CORINFO_CALLINFO_SECURITYCHECKS), - (opcode == CEE_CALLVIRT) ? CORINFO_CALLINFO_CALLVIRT : CORINFO_CALLINFO_NONE), - &callInfo); + if (!usingReadyToRunHelper) + { + // TODO: ReadyToRun: When generic dictionary lookups are necessary, replace the lookup call + // and the newarr call with a single call to a dynamic R2R cell that will: + // 1) Load the context + // 2) Perform the generic dictionary lookup and caching, and generate the appropriate stub + // 3) Allocate the new array + // Reason: performance (today, we'll always use the slow helper for the R2R generics case) + + // Need to restore array classes before creating array objects on the heap + op1 = impTokenToHandle(&resolvedToken, nullptr, true /*mustRestoreHandle*/); + if (op1 == nullptr) + { // compDonotInline() + return; + } + } } - else + + if (!usingReadyToRunHelper) +#endif { - // Suppress uninitialized use warning. - memset(&resolvedToken, 0, sizeof(resolvedToken)); - memset(&callInfo, 0, sizeof(callInfo)); + /* Create a call to 'new' */ - resolvedToken.token = getU4LittleEndian(codeAddr); - resolvedToken.tokenContext = impTokenLookupContextHandle; - resolvedToken.tokenScope = info.compScopeHnd; + // Note that this only works for shared generic code because the same helper is used for all + // reference array types + op1 = + gtNewHelperCallNode(info.compCompHnd->getNewArrHelper(resolvedToken.hClass), TYP_REF, op1, op2); } - CALL: // memberRef should be set. - // newObjThisPtr should be set for CEE_NEWOBJ + op1->AsCall()->compileTimeHelperArgumentHandle = (CORINFO_GENERIC_HANDLE)resolvedToken.hClass; - JITDUMP(" %08X", resolvedToken.token); - constraintCall = (prefixFlags & PREFIX_CONSTRAINED) != 0; + // Remember that this function contains 'new' of an SD array. + optMethodFlags |= OMF_HAS_NEWARRAY; - bool newBBcreatedForTailcallStress; - bool passedStressModeValidation; + /* Push the result of the call on the stack */ - newBBcreatedForTailcallStress = false; - passedStressModeValidation = true; + impPushOnStack(op1, tiRetVal); - if (compIsForInlining()) + callTyp = TYP_REF; + } + break; + + case CEE_LOCALLOC: + // We don't allow locallocs inside handlers + if (block->hasHndIndex()) { - if (compDonotInline()) - { - return; - } - // We rule out inlinees with explicit tail calls in fgMakeBasicBlocks. - assert((prefixFlags & PREFIX_TAILCALL_EXPLICIT) == 0); + BADCODE("Localloc can't be inside handler"); } - else + + // Get the size to allocate + + op2 = impPopStack().val; + assertImp(genActualTypeIsIntOrI(op2->gtType)); + + if (verCurrentState.esStackDepth != 0) { - if (compTailCallStress()) + BADCODE("Localloc can only be used when the stack is empty"); + } + + // If the localloc is not in a loop and its size is a small constant, + // create a new local var of TYP_BLK and return its address. + { + bool convertedToLocal = false; + + // Need to aggressively fold here, as even fixed-size locallocs + // will have casts in the way. + op2 = gtFoldExpr(op2); + + if (op2->IsIntegralConst()) { - // Have we created a new BB after the "call" instruction in fgMakeBasicBlocks()? - // Tail call stress only recognizes call+ret patterns and forces them to be - // explicit tail prefixed calls. Also fgMakeBasicBlocks() under tail call stress - // doesn't import 'ret' opcode following the call into the basic block containing - // the call instead imports it to a new basic block. Note that fgMakeBasicBlocks() - // is already checking that there is an opcode following call and hence it is - // safe here to read next opcode without bounds check. - newBBcreatedForTailcallStress = - impOpcodeIsCallOpcode(opcode) && // Current opcode is a CALL, (not a CEE_NEWOBJ). So, don't - // make it jump to RET. - (OPCODE)getU1LittleEndian(codeAddr + sz) == CEE_RET; // Next opcode is a CEE_RET + const ssize_t allocSize = op2->AsIntCon()->IconValue(); - bool hasTailPrefix = (prefixFlags & PREFIX_TAILCALL_EXPLICIT); - if (newBBcreatedForTailcallStress && !hasTailPrefix) + bool bbInALoop = impBlockIsInALoop(block); + + if (allocSize == 0) { - // Do a more detailed evaluation of legality - const bool passedConstraintCheck = - verCheckTailCallConstraint(opcode, &resolvedToken, - constraintCall ? &constrainedResolvedToken : nullptr); + // Result is nullptr + JITDUMP("Converting stackalloc of 0 bytes to push null unmanaged pointer\n"); + op1 = gtNewIconNode(0, TYP_I_IMPL); + convertedToLocal = true; + } + else if ((allocSize > 0) && !bbInALoop) + { + // Get the size threshold for local conversion + ssize_t maxSize = DEFAULT_MAX_LOCALLOC_TO_LOCAL_SIZE; - // Avoid setting compHasBackwardsJump = true via tail call stress if the method cannot have - // patchpoints. - // - const bool mayHavePatchpoints = opts.jitFlags->IsSet(JitFlags::JIT_FLAG_TIER0) && - (JitConfig.TC_OnStackReplacement() > 0) && - compCanHavePatchpoints(); - if (passedConstraintCheck && (mayHavePatchpoints || compHasBackwardJump)) +#ifdef DEBUG + // Optionally allow this to be modified + maxSize = JitConfig.JitStackAllocToLocalSize(); +#endif // DEBUG + + if (allocSize <= maxSize) { - // Now check with the runtime - CORINFO_METHOD_HANDLE declaredCalleeHnd = callInfo.hMethod; - bool isVirtual = (callInfo.kind == CORINFO_VIRTUALCALL_STUB) || - (callInfo.kind == CORINFO_VIRTUALCALL_VTABLE); - CORINFO_METHOD_HANDLE exactCalleeHnd = isVirtual ? nullptr : declaredCalleeHnd; - if (info.compCompHnd->canTailCall(info.compMethodHnd, declaredCalleeHnd, exactCalleeHnd, - hasTailPrefix)) // Is it legal to do tailcall? - { - // Stress the tailcall. - JITDUMP(" (Tailcall stress: prefixFlags |= PREFIX_TAILCALL_EXPLICIT)"); - prefixFlags |= PREFIX_TAILCALL_EXPLICIT; - prefixFlags |= PREFIX_TAILCALL_STRESS; - } - else + const unsigned stackallocAsLocal = lvaGrabTemp(false DEBUGARG("stackallocLocal")); + JITDUMP("Converting stackalloc of %zd bytes to new local V%02u\n", allocSize, + stackallocAsLocal); + lvaTable[stackallocAsLocal].lvType = TYP_BLK; + lvaTable[stackallocAsLocal].lvExactSize = (unsigned)allocSize; + lvaTable[stackallocAsLocal].lvIsUnsafeBuffer = true; + op1 = gtNewLclvNode(stackallocAsLocal, TYP_BLK); + op1 = gtNewOperNode(GT_ADDR, TYP_I_IMPL, op1); + convertedToLocal = true; + + if (!this->opts.compDbgEnC) { - // Runtime disallows this tail call - JITDUMP(" (Tailcall stress: runtime preventing tailcall)"); - passedStressModeValidation = false; + // Ensure we have stack security for this method. + // Reorder layout since the converted localloc is treated as an unsafe buffer. + setNeedsGSSecurityCookie(); + compGSReorderStackLayout = true; } } - else - { - // Constraints disallow this tail call - JITDUMP(" (Tailcall stress: constraint check failed)"); - passedStressModeValidation = false; - } } } - } - - // This is split up to avoid goto flow warnings. - bool isRecursive; - isRecursive = !compIsForInlining() && (callInfo.hMethod == info.compMethodHnd); - // If we've already disqualified this call as a tail call under tail call stress, - // don't consider it for implicit tail calling either. - // - // When not running under tail call stress, we may mark this call as an implicit - // tail call candidate. We'll do an "equivalent" validation during impImportCall. - // - // Note that when running under tail call stress, a call marked as explicit - // tail prefixed will not be considered for implicit tail calling. - if (passedStressModeValidation && - impIsImplicitTailCallCandidate(opcode, codeAddr + sz, codeEndp, prefixFlags, isRecursive)) - { - if (compIsForInlining()) + if (!convertedToLocal) { -#if FEATURE_TAILCALL_OPT_SHARED_RETURN - // Are we inlining at an implicit tail call site? If so the we can flag - // implicit tail call sites in the inline body. These call sites - // often end up in non BBJ_RETURN blocks, so only flag them when - // we're able to handle shared returns. - if (impInlineInfo->iciCall->IsImplicitTailCall()) + // Bail out if inlining and the localloc was not converted. + // + // Note we might consider allowing the inline, if the call + // site is not in a loop. + if (compIsForInlining()) { - JITDUMP("\n (Inline Implicit Tail call: prefixFlags |= PREFIX_TAILCALL_IMPLICIT)"); - prefixFlags |= PREFIX_TAILCALL_IMPLICIT; + InlineObservation obs = op2->IsIntegralConst() + ? InlineObservation::CALLEE_LOCALLOC_TOO_LARGE + : InlineObservation::CALLSITE_LOCALLOC_SIZE_UNKNOWN; + compInlineResult->NoteFatal(obs); + return; } -#endif // FEATURE_TAILCALL_OPT_SHARED_RETURN + + op1 = gtNewOperNode(GT_LCLHEAP, TYP_I_IMPL, op2); + // May throw a stack overflow exception. Obviously, we don't want locallocs to be CSE'd. + op1->gtFlags |= (GTF_EXCEPT | GTF_DONT_CSE); + + // Ensure we have stack security for this method. + setNeedsGSSecurityCookie(); + + /* The FP register may not be back to the original value at the end + of the method, even if the frame size is 0, as localloc may + have modified it. So we will HAVE to reset it */ + compLocallocUsed = true; } else { - JITDUMP("\n (Implicit Tail call: prefixFlags |= PREFIX_TAILCALL_IMPLICIT)"); - prefixFlags |= PREFIX_TAILCALL_IMPLICIT; + compLocallocOptimized = true; } } - // Treat this call as tail call for verification only if "tail" prefixed (i.e. explicit tail call). - explicitTailCall = (prefixFlags & PREFIX_TAILCALL_EXPLICIT) != 0; - readonlyCall = (prefixFlags & PREFIX_READONLY) != 0; - - if (opcode != CEE_CALLI && opcode != CEE_NEWOBJ) - { - // All calls and delegates need a security callout. - // For delegates, this is the call to the delegate constructor, not the access check on the - // LD(virt)FTN. - impHandleAccessAllowed(callInfo.accessAllowed, &callInfo.callsiteCalloutHelper); - } - - callTyp = impImportCall(opcode, &resolvedToken, constraintCall ? &constrainedResolvedToken : nullptr, - newObjThisPtr, prefixFlags, &callInfo, opcodeOffs); - if (compDonotInline()) - { - // We do not check fails after lvaGrabTemp. It is covered with CoreCLR_13272 issue. - assert((callTyp == TYP_UNDEF) || - (compInlineResult->GetObservation() == InlineObservation::CALLSITE_TOO_MANY_LOCALS)); - return; - } - - if (explicitTailCall || newBBcreatedForTailcallStress) // If newBBcreatedForTailcallStress is true, we - // have created a new BB after the "call" - // instruction in fgMakeBasicBlocks(). So we need to jump to RET regardless. - { - assert(!compIsForInlining()); - goto RET; - } - + impPushOnStack(op1, tiRetVal); break; - case CEE_LDFLD: - case CEE_LDSFLD: - case CEE_LDFLDA: - case CEE_LDSFLDA: + case CEE_ISINST: { - - bool isLoadAddress = (opcode == CEE_LDFLDA || opcode == CEE_LDSFLDA); - bool isLoadStatic = (opcode == CEE_LDSFLD || opcode == CEE_LDSFLDA); - - /* Get the CP_Fieldref index */ + /* Get the type token */ assertImp(sz == sizeof(unsigned)); - _impResolveToken(CORINFO_TOKENKIND_Field); + _impResolveToken(CORINFO_TOKENKIND_Casting); JITDUMP(" %08X", resolvedToken.token); - int aflags = isLoadAddress ? CORINFO_ACCESS_ADDRESS : CORINFO_ACCESS_GET; - - GenTree* obj = nullptr; - CORINFO_CLASS_HANDLE objType = nullptr; // used for fields - - if ((opcode == CEE_LDFLD) || (opcode == CEE_LDFLDA)) + if (!opts.IsReadyToRun()) { - StackEntry se = impPopStack(); - objType = se.seTypeInfo.GetClassHandle(); - obj = se.val; - - if (impIsThis(obj)) - { - aflags |= CORINFO_ACCESS_THIS; + op2 = impTokenToHandle(&resolvedToken, nullptr, false); + if (op2 == nullptr) + { // compDonotInline() + return; } } - eeGetFieldInfo(&resolvedToken, (CORINFO_ACCESS_FLAGS)aflags, &fieldInfo); + accessAllowedResult = + info.compCompHnd->canAccessClass(&resolvedToken, info.compMethodHnd, &calloutHelper); + impHandleAccessAllowed(accessAllowedResult, &calloutHelper); - // Figure out the type of the member. We always call canAccessField, so you always need this - // handle - CorInfoType ciType = fieldInfo.fieldType; - clsHnd = fieldInfo.structType; + op1 = impPopStack().val; - lclTyp = JITtype2varType(ciType); + GenTree* optTree = impOptimizeCastClassOrIsInst(op1, &resolvedToken, false); - if (compIsForInlining()) + if (optTree != nullptr) + { + impPushOnStack(optTree, tiRetVal); + } + else { - switch (fieldInfo.fieldAccessor) - { - case CORINFO_FIELD_INSTANCE_HELPER: - case CORINFO_FIELD_INSTANCE_ADDR_HELPER: - case CORINFO_FIELD_STATIC_ADDR_HELPER: - case CORINFO_FIELD_STATIC_TLS: - - compInlineResult->NoteFatal(InlineObservation::CALLEE_LDFLD_NEEDS_HELPER); - return; - - case CORINFO_FIELD_STATIC_GENERICS_STATIC_HELPER: - case CORINFO_FIELD_STATIC_READYTORUN_HELPER: - /* We may be able to inline the field accessors in specific instantiations of generic - * methods */ - compInlineResult->NoteFatal(InlineObservation::CALLSITE_LDFLD_NEEDS_HELPER); - return; - - default: - break; - } - if (!isLoadAddress && (fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC) && lclTyp == TYP_STRUCT && - clsHnd) +#ifdef FEATURE_READYTORUN + if (opts.IsReadyToRun()) { - if ((info.compCompHnd->getTypeForPrimitiveValueClass(clsHnd) == CORINFO_TYPE_UNDEF) && - !(info.compFlags & CORINFO_FLG_FORCEINLINE)) + GenTreeCall* opLookup = + impReadyToRunHelperToTree(&resolvedToken, CORINFO_HELP_READYTORUN_ISINSTANCEOF, TYP_REF, + nullptr, op1); + usingReadyToRunHelper = (opLookup != nullptr); + op1 = (usingReadyToRunHelper ? opLookup : op1); + + if (!usingReadyToRunHelper) { - // Loading a static valuetype field usually will cause a JitHelper to be called - // for the static base. This will bloat the code. - compInlineResult->Note(InlineObservation::CALLEE_LDFLD_STATIC_VALUECLASS); + // TODO: ReadyToRun: When generic dictionary lookups are necessary, replace the lookup call + // and the isinstanceof_any call with a single call to a dynamic R2R cell that will: + // 1) Load the context + // 2) Perform the generic dictionary lookup and caching, and generate the appropriate + // stub + // 3) Perform the 'is instance' check on the input object + // Reason: performance (today, we'll always use the slow helper for the R2R generics case) - if (compInlineResult->IsFailure()) - { + op2 = impTokenToHandle(&resolvedToken, nullptr, false); + if (op2 == nullptr) + { // compDonotInline() return; } } } - } - - tiRetVal = verMakeTypeInfo(ciType, clsHnd); - if (isLoadAddress) - { - tiRetVal.MakeByRef(); - } - else - { - tiRetVal.NormaliseForStack(); - } - - // Perform this check always to ensure that we get field access exceptions even with - // SkipVerification. - impHandleAccessAllowed(fieldInfo.accessAllowed, &fieldInfo.accessCalloutHelper); - - // Raise InvalidProgramException if static load accesses non-static field - if (isLoadStatic && ((fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC) == 0)) - { - BADCODE("static access on an instance field"); - } - // We are using ldfld/a on a static field. We allow it, but need to get side-effect from obj. - if ((fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC) && obj != nullptr) - { - if (obj->gtFlags & GTF_SIDE_EFFECT) + if (!usingReadyToRunHelper) +#endif { - obj = gtUnusedValNode(obj); - impAppendTree(obj, CHECK_SPILL_ALL, impCurStmtDI); + op1 = impCastClassOrIsInstToTree(op1, op2, &resolvedToken, false, opcodeOffs); } - obj = nullptr; - } - - /* Preserve 'small' int types */ - if (!varTypeIsSmall(lclTyp)) - { - lclTyp = genActualType(lclTyp); - } - - bool usesHelper = false; - - switch (fieldInfo.fieldAccessor) - { - case CORINFO_FIELD_INSTANCE: -#ifdef FEATURE_READYTORUN - case CORINFO_FIELD_INSTANCE_WITH_BASE: -#endif + if (compDonotInline()) { - // If the object is a struct, what we really want is - // for the field to operate on the address of the struct. - if (varTypeIsStruct(obj)) - { - if (opcode != CEE_LDFLD) - { - BADCODE3("Unexpected opcode (has to be LDFLD)", ": %02X", (int)opcode); - } - if (objType == nullptr) - { - BADCODE("top of stack must be a value type"); - } - obj = impGetStructAddr(obj, objType, CHECK_SPILL_ALL, true); - } - - if (isLoadAddress) - { - op1 = gtNewFieldAddrNode(varTypeIsGC(obj) ? TYP_BYREF : TYP_I_IMPL, resolvedToken.hField, - obj, fieldInfo.offset); - } - else - { - op1 = gtNewFieldRef(lclTyp, resolvedToken.hField, obj, fieldInfo.offset); - } + return; + } -#ifdef FEATURE_READYTORUN - if (fieldInfo.fieldAccessor == CORINFO_FIELD_INSTANCE_WITH_BASE) - { - op1->AsField()->gtFieldLookup = fieldInfo.fieldLookup; - } -#endif + impPushOnStack(op1, tiRetVal); + } + break; + } - if (fgAddrCouldBeNull(obj)) - { - op1->gtFlags |= GTF_EXCEPT; - } + case CEE_REFANYVAL: + { - if (StructHasOverlappingFields(info.compCompHnd->getClassAttribs(resolvedToken.hClass))) - { - op1->AsField()->gtFldMayOverlap = true; - } + // get the class handle and make a ICON node out of it - if (!isLoadAddress && compIsForInlining() && - impInlineIsGuaranteedThisDerefBeforeAnySideEffects(nullptr, nullptr, obj, - impInlineInfo->inlArgInfo)) - { - impInlineInfo->thisDereferencedFirst = true; - } - } - break; + _impResolveToken(CORINFO_TOKENKIND_Class); - case CORINFO_FIELD_STATIC_TLS: -#ifdef TARGET_X86 - // Legacy TLS access is implemented as intrinsic on x86 only - op1 = gtNewFieldAddrNode(TYP_I_IMPL, resolvedToken.hField, nullptr, fieldInfo.offset); - op1->gtFlags |= GTF_FLD_TLS; // fgMorphExpandTlsField will handle the transformation. + JITDUMP(" %08X", resolvedToken.token); - if (!isLoadAddress) - { - if (varTypeIsStruct(lclTyp)) - { - op1 = gtNewObjNode(fieldInfo.structType, op1); - op1->gtFlags |= GTF_IND_NONFAULTING; - } - else - { - op1 = gtNewIndir(lclTyp, op1, GTF_IND_NONFAULTING); - op1->gtFlags |= GTF_GLOB_REF; - } - } - break; -#else - fieldInfo.fieldAccessor = CORINFO_FIELD_STATIC_ADDR_HELPER; - FALLTHROUGH; -#endif - case CORINFO_FIELD_STATIC_ADDR_HELPER: - case CORINFO_FIELD_INSTANCE_HELPER: - case CORINFO_FIELD_INSTANCE_ADDR_HELPER: - op1 = gtNewRefCOMfield(obj, &resolvedToken, (CORINFO_ACCESS_FLAGS)aflags, &fieldInfo, lclTyp, - clsHnd, nullptr); - usesHelper = true; - break; + op2 = impTokenToHandle(&resolvedToken); + if (op2 == nullptr) + { // compDonotInline() + return; + } - case CORINFO_FIELD_STATIC_SHARED_STATIC_HELPER: - case CORINFO_FIELD_STATIC_ADDRESS: - // Replace static read-only fields with constant if possible - if ((aflags & CORINFO_ACCESS_GET) && (fieldInfo.fieldFlags & CORINFO_FLG_FIELD_FINAL)) - { - GenTree* newTree = impImportStaticReadOnlyField(resolvedToken.hField, resolvedToken.hClass); + op1 = impPopStack().val; + // make certain it is normalized; + op1 = impNormStructVal(op1, impGetRefAnyClass(), (unsigned)CHECK_SPILL_ALL); - if (newTree != nullptr) - { - op1 = newTree; - goto FIELD_DONE; - } - } - FALLTHROUGH; + // Call helper GETREFANY(classHandle, op1); + GenTreeCall* helperCall = gtNewHelperCallNode(CORINFO_HELP_GETREFANY, TYP_BYREF); + NewCallArg clsHandleArg = NewCallArg::Primitive(op2); + NewCallArg typedRefArg = NewCallArg::Struct(op1, TYP_STRUCT, impGetRefAnyClass()); + helperCall->gtArgs.PushFront(this, clsHandleArg, typedRefArg); + helperCall->gtFlags |= (op1->gtFlags | op2->gtFlags) & GTF_ALL_EFFECT; + op1 = helperCall; - case CORINFO_FIELD_STATIC_RVA_ADDRESS: - case CORINFO_FIELD_STATIC_GENERICS_STATIC_HELPER: - case CORINFO_FIELD_STATIC_READYTORUN_HELPER: - op1 = impImportStaticFieldAccess(&resolvedToken, (CORINFO_ACCESS_FLAGS)aflags, &fieldInfo, - lclTyp); - break; + impPushOnStack(op1, tiRetVal); + break; + } + case CEE_REFANYTYPE: - case CORINFO_FIELD_INTRINSIC_ZERO: - { - assert(aflags & CORINFO_ACCESS_GET); - // Widen to stack type - lclTyp = genActualType(lclTyp); - op1 = gtNewIconNode(0, lclTyp); - goto FIELD_DONE; - } - break; + op1 = impPopStack().val; - case CORINFO_FIELD_INTRINSIC_EMPTY_STRING: - { - assert(aflags & CORINFO_ACCESS_GET); + // make certain it is normalized; + op1 = impNormStructVal(op1, impGetRefAnyClass(), (unsigned)CHECK_SPILL_ALL); - // Import String.Empty as "" (GT_CNS_STR with a fake SconCPX = 0) - op1 = gtNewSconNode(EMPTY_STRING_SCON, nullptr); - goto FIELD_DONE; - } - break; + if (op1->gtOper == GT_OBJ) + { + // Get the address of the refany + op1 = op1->AsOp()->gtOp1; - case CORINFO_FIELD_INTRINSIC_ISLITTLEENDIAN: + // Fetch the type from the correct slot + op1 = gtNewOperNode(GT_ADD, TYP_BYREF, op1, + gtNewIconNode(OFFSETOF__CORINFO_TypedReference__type, TYP_I_IMPL)); + op1 = gtNewOperNode(GT_IND, TYP_BYREF, op1); + } + else + { + assertImp(op1->gtOper == GT_MKREFANY); + + // The pointer may have side-effects + if (op1->AsOp()->gtOp1->gtFlags & GTF_SIDE_EFFECT) { - assert(aflags & CORINFO_ACCESS_GET); - // Widen to stack type - lclTyp = genActualType(lclTyp); -#if BIGENDIAN - op1 = gtNewIconNode(0, lclTyp); -#else - op1 = gtNewIconNode(1, lclTyp); + impAppendTree(op1->AsOp()->gtOp1, (unsigned)CHECK_SPILL_ALL, impCurStmtDI); +#ifdef DEBUG + impNoteLastILoffs(); #endif - goto FIELD_DONE; } - break; - default: - assert(!"Unexpected fieldAccessor"); + // We already have the class handle + op1 = op1->AsOp()->gtOp2; } - if (!isLoadAddress) + // convert native TypeHandle to RuntimeTypeHandle { + op1 = gtNewHelperCallNode(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPEHANDLE_MAYBENULL, TYP_STRUCT, op1); - if (prefixFlags & PREFIX_VOLATILE) - { - op1->gtFlags |= GTF_ORDER_SIDEEFF; // Prevent this from being reordered + CORINFO_CLASS_HANDLE classHandle = impGetTypeHandleClass(); - if (!usesHelper) - { - assert(op1->OperIs(GT_FIELD, GT_IND, GT_OBJ)); - op1->gtFlags |= GTF_IND_VOLATILE; - } - } + // The handle struct is returned in register + op1->AsCall()->gtReturnType = GetRuntimeHandleUnderlyingType(); + op1->AsCall()->gtRetClsHnd = classHandle; +#if FEATURE_MULTIREG_RET + op1->AsCall()->InitializeStructReturnType(this, classHandle, op1->AsCall()->GetUnmanagedCallConv()); +#endif - if ((prefixFlags & PREFIX_UNALIGNED) && !varTypeIsByte(lclTyp)) - { - if (!usesHelper) - { - assert(op1->OperIs(GT_FIELD, GT_IND, GT_OBJ)); - op1->gtFlags |= GTF_IND_UNALIGNED; - } - } + tiRetVal = typeInfo(TI_STRUCT, classHandle); } - // Check if the class needs explicit initialization. - if (fieldInfo.fieldFlags & CORINFO_FLG_FIELD_INITCLASS) + impPushOnStack(op1, tiRetVal); + break; + + case CEE_LDTOKEN: + { + /* Get the Class index */ + assertImp(sz == sizeof(unsigned)); + lastLoadToken = codeAddr; + _impResolveToken(CORINFO_TOKENKIND_Ldtoken); + + tokenType = info.compCompHnd->getTokenTypeAsHandle(&resolvedToken); + + op1 = impTokenToHandle(&resolvedToken, nullptr, true); + if (op1 == nullptr) + { // compDonotInline() + return; + } + + helper = CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPEHANDLE; + assert(resolvedToken.hClass != nullptr); + + if (resolvedToken.hMethod != nullptr) { - GenTree* helperNode = impInitClass(&resolvedToken); - if (compDonotInline()) - { - return; - } - if (helperNode != nullptr) - { - op1 = gtNewOperNode(GT_COMMA, op1->TypeGet(), helperNode, op1); - } + helper = CORINFO_HELP_METHODDESC_TO_STUBRUNTIMEMETHOD; + } + else if (resolvedToken.hField != nullptr) + { + helper = CORINFO_HELP_FIELDDESC_TO_STUBRUNTIMEFIELD; } - FIELD_DONE: + op1 = gtNewHelperCallNode(helper, TYP_STRUCT, op1); + + // The handle struct is returned in register and + // it could be consumed both as `TYP_STRUCT` and `TYP_REF`. + op1->AsCall()->gtReturnType = GetRuntimeHandleUnderlyingType(); +#if FEATURE_MULTIREG_RET + op1->AsCall()->InitializeStructReturnType(this, tokenType, op1->AsCall()->GetUnmanagedCallConv()); +#endif + op1->AsCall()->gtRetClsHnd = tokenType; + + tiRetVal = verMakeTypeInfo(tokenType); impPushOnStack(op1, tiRetVal); } break; - case CEE_STFLD: - case CEE_STSFLD: + case CEE_UNBOX: + case CEE_UNBOX_ANY: { + /* Get the Class index */ + assertImp(sz == sizeof(unsigned)); - bool isStoreStatic = (opcode == CEE_STSFLD); + _impResolveToken(CORINFO_TOKENKIND_Class); - CORINFO_CLASS_HANDLE fieldClsHnd; // class of the field (if it's a ref type) + JITDUMP(" %08X", resolvedToken.token); - /* Get the CP_Fieldref index */ + bool runtimeLookup; + op2 = impTokenToHandle(&resolvedToken, &runtimeLookup); + if (op2 == nullptr) + { + assert(compDonotInline()); + return; + } - assertImp(sz == sizeof(unsigned)); + // Run this always so we can get access exceptions even with SkipVerification. + accessAllowedResult = + info.compCompHnd->canAccessClass(&resolvedToken, info.compMethodHnd, &calloutHelper); + impHandleAccessAllowed(accessAllowedResult, &calloutHelper); - _impResolveToken(CORINFO_TOKENKIND_Field); + if (opcode == CEE_UNBOX_ANY && !eeIsValueClass(resolvedToken.hClass)) + { + JITDUMP("\n Importing UNBOX.ANY(refClass) as CASTCLASS\n"); + op1 = impPopStack().val; + goto CASTCLASS; + } - JITDUMP(" %08X", resolvedToken.token); + /* Pop the object and create the unbox helper call */ + /* You might think that for UNBOX_ANY we need to push a different */ + /* (non-byref) type, but here we're making the tiRetVal that is used */ + /* for the intermediate pointer which we then transfer onto the OBJ */ + /* instruction. OBJ then creates the appropriate tiRetVal. */ - int aflags = CORINFO_ACCESS_SET; - GenTree* obj = nullptr; + op1 = impPopStack().val; + assertImp(op1->gtType == TYP_REF); - // Pull the value from the stack. - StackEntry se = impPopStack(); - op2 = se.val; - clsHnd = se.seTypeInfo.GetClassHandle(); + helper = info.compCompHnd->getUnBoxHelper(resolvedToken.hClass); + assert(helper == CORINFO_HELP_UNBOX || helper == CORINFO_HELP_UNBOX_NULLABLE); - if (opcode == CEE_STFLD) + // Check legality and profitability of inline expansion for unboxing. + const bool canExpandInline = (helper == CORINFO_HELP_UNBOX); + const bool shouldExpandInline = !compCurBB->isRunRarely() && opts.OptimizationEnabled(); + + if (canExpandInline && shouldExpandInline) { - obj = impPopStack().val; + // See if we know anything about the type of op1, the object being unboxed. + bool isExact = false; + bool isNonNull = false; + CORINFO_CLASS_HANDLE clsHnd = gtGetClassHandle(op1, &isExact, &isNonNull); - if (impIsThis(obj)) + // We can skip the "exact" bit here as we are comparing to a value class. + // compareTypesForEquality should bail on comparisons for shared value classes. + if (clsHnd != NO_CLASS_HANDLE) { - aflags |= CORINFO_ACCESS_THIS; + const TypeCompareState compare = + info.compCompHnd->compareTypesForEquality(resolvedToken.hClass, clsHnd); + + if (compare == TypeCompareState::Must) + { + JITDUMP("\nOptimizing %s (%s) -- type test will succeed\n", + opcode == CEE_UNBOX ? "UNBOX" : "UNBOX.ANY", eeGetClassName(clsHnd)); + + // For UNBOX, null check (if necessary), and then leave the box payload byref on the stack. + if (opcode == CEE_UNBOX) + { + GenTree* cloneOperand; + op1 = impCloneExpr(op1, &cloneOperand, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, + nullptr DEBUGARG("optimized unbox clone")); + + GenTree* boxPayloadOffset = gtNewIconNode(TARGET_POINTER_SIZE, TYP_I_IMPL); + GenTree* boxPayloadAddress = + gtNewOperNode(GT_ADD, TYP_BYREF, cloneOperand, boxPayloadOffset); + GenTree* nullcheck = gtNewNullCheck(op1, block); + GenTree* result = gtNewOperNode(GT_COMMA, TYP_BYREF, nullcheck, boxPayloadAddress); + impPushOnStack(result, tiRetVal); + break; + } + + // For UNBOX.ANY load the struct from the box payload byref (the load will nullcheck) + assert(opcode == CEE_UNBOX_ANY); + GenTree* boxPayloadOffset = gtNewIconNode(TARGET_POINTER_SIZE, TYP_I_IMPL); + GenTree* boxPayloadAddress = gtNewOperNode(GT_ADD, TYP_BYREF, op1, boxPayloadOffset); + impPushOnStack(boxPayloadAddress, tiRetVal); + oper = GT_OBJ; + goto OBJ; + } + else + { + JITDUMP("\nUnable to optimize %s -- can't resolve type comparison\n", + opcode == CEE_UNBOX ? "UNBOX" : "UNBOX.ANY"); + } + } + else + { + JITDUMP("\nUnable to optimize %s -- class for [%06u] not known\n", + opcode == CEE_UNBOX ? "UNBOX" : "UNBOX.ANY", dspTreeID(op1)); } - } - eeGetFieldInfo(&resolvedToken, (CORINFO_ACCESS_FLAGS)aflags, &fieldInfo); + JITDUMP("\n Importing %s as inline sequence\n", opcode == CEE_UNBOX ? "UNBOX" : "UNBOX.ANY"); + // we are doing normal unboxing + // inline the common case of the unbox helper + // UNBOX(exp) morphs into + // clone = pop(exp); + // ((*clone == typeToken) ? nop : helper(clone, typeToken)); + // push(clone + TARGET_POINTER_SIZE) + // + GenTree* cloneOperand; + op1 = impCloneExpr(op1, &cloneOperand, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, + nullptr DEBUGARG("inline UNBOX clone1")); + op1 = gtNewMethodTableLookup(op1); - // Figure out the type of the member. We always call canAccessField, so you always need this - // handle - CorInfoType ciType = fieldInfo.fieldType; - fieldClsHnd = fieldInfo.structType; + GenTree* condBox = gtNewOperNode(GT_EQ, TYP_INT, op1, op2); - lclTyp = JITtype2varType(ciType); + op1 = impCloneExpr(cloneOperand, &cloneOperand, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, + nullptr DEBUGARG("inline UNBOX clone2")); + op2 = impTokenToHandle(&resolvedToken); + if (op2 == nullptr) + { // compDonotInline() + return; + } + op1 = gtNewHelperCallNode(helper, TYP_VOID, op2, op1); - if (compIsForInlining()) - { - /* Is this a 'special' (COM) field? or a TLS ref static field?, field stored int GC heap? or - * per-inst static? */ + op1 = new (this, GT_COLON) GenTreeColon(TYP_VOID, gtNewNothingNode(), op1); + op1 = gtNewQmarkNode(TYP_VOID, condBox, op1->AsColon()); - switch (fieldInfo.fieldAccessor) - { - case CORINFO_FIELD_INSTANCE_HELPER: - case CORINFO_FIELD_INSTANCE_ADDR_HELPER: - case CORINFO_FIELD_STATIC_ADDR_HELPER: - case CORINFO_FIELD_STATIC_TLS: + // QMARK nodes cannot reside on the evaluation stack. Because there + // may be other trees on the evaluation stack that side-effect the + // sources of the UNBOX operation we must spill the stack. - compInlineResult->NoteFatal(InlineObservation::CALLEE_STFLD_NEEDS_HELPER); - return; + impAppendTree(op1, (unsigned)CHECK_SPILL_ALL, impCurStmtDI); - case CORINFO_FIELD_STATIC_GENERICS_STATIC_HELPER: - case CORINFO_FIELD_STATIC_READYTORUN_HELPER: - /* We may be able to inline the field accessors in specific instantiations of generic - * methods */ - compInlineResult->NoteFatal(InlineObservation::CALLSITE_STFLD_NEEDS_HELPER); - return; + // Create the address-expression to reference past the object header + // to the beginning of the value-type. Today this means adjusting + // past the base of the objects vtable field which is pointer sized. - default: - break; - } + op2 = gtNewIconNode(TARGET_POINTER_SIZE, TYP_I_IMPL); + op1 = gtNewOperNode(GT_ADD, TYP_BYREF, cloneOperand, op2); } - - impHandleAccessAllowed(fieldInfo.accessAllowed, &fieldInfo.accessCalloutHelper); - - // Raise InvalidProgramException if static store accesses non-static field - if (isStoreStatic && ((fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC) == 0)) + else { - BADCODE("static access on an instance field"); - } + JITDUMP("\n Importing %s as helper call because %s\n", opcode == CEE_UNBOX ? "UNBOX" : "UNBOX.ANY", + canExpandInline ? "want smaller code or faster jitting" : "inline expansion not legal"); - // We are using stfld on a static field. - // We allow it, but need to eval any side-effects for obj - if ((fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC) && obj != nullptr) - { - if (obj->gtFlags & GTF_SIDE_EFFECT) + // Don't optimize, just call the helper and be done with it + op1 = gtNewHelperCallNode(helper, + (var_types)((helper == CORINFO_HELP_UNBOX) ? TYP_BYREF : TYP_STRUCT), op2, + op1); + if (op1->gtType == TYP_STRUCT) { - obj = gtUnusedValNode(obj); - impAppendTree(obj, CHECK_SPILL_ALL, impCurStmtDI); + op1->AsCall()->gtRetClsHnd = resolvedToken.hClass; } - obj = nullptr; } - /* Preserve 'small' int types */ - if (!varTypeIsSmall(lclTyp)) - { - lclTyp = genActualType(lclTyp); - } + assert((helper == CORINFO_HELP_UNBOX && op1->gtType == TYP_BYREF) || // Unbox helper returns a byref. + (helper == CORINFO_HELP_UNBOX_NULLABLE && + varTypeIsStruct(op1)) // UnboxNullable helper returns a struct. + ); - switch (fieldInfo.fieldAccessor) + /* + ---------------------------------------------------------------------- + | \ helper | | | + | \ | | | + | \ | CORINFO_HELP_UNBOX | CORINFO_HELP_UNBOX_NULLABLE | + | \ | (which returns a BYREF) | (which returns a STRUCT) | | + | opcode \ | | | + |--------------------------------------------------------------------- + | UNBOX | push the BYREF | spill the STRUCT to a local, | + | | | push the BYREF to this local | + |--------------------------------------------------------------------- + | UNBOX_ANY | push a GT_OBJ of | push the STRUCT | + | | the BYREF | For Linux when the | + | | | struct is returned in two | + | | | registers create a temp | + | | | which address is passed to | + | | | the unbox_nullable helper. | + |--------------------------------------------------------------------- + */ + + if (opcode == CEE_UNBOX) { - case CORINFO_FIELD_INSTANCE: -#ifdef FEATURE_READYTORUN - case CORINFO_FIELD_INSTANCE_WITH_BASE: -#endif + if (helper == CORINFO_HELP_UNBOX_NULLABLE) { - /* Create the data member node */ - op1 = gtNewFieldRef(lclTyp, resolvedToken.hField, obj, fieldInfo.offset); - DWORD typeFlags = info.compCompHnd->getClassAttribs(resolvedToken.hClass); - if (StructHasOverlappingFields(typeFlags)) - { - op1->AsField()->gtFldMayOverlap = true; - } + // Unbox nullable helper returns a struct type. + // We need to spill it to a temp so than can take the address of it. + // Here we need unsafe value cls check, since the address of struct is taken to be used + // further along and potetially be exploitable. -#ifdef FEATURE_READYTORUN - if (fieldInfo.fieldAccessor == CORINFO_FIELD_INSTANCE_WITH_BASE) - { - op1->AsField()->gtFieldLookup = fieldInfo.fieldLookup; - } -#endif + unsigned tmp = lvaGrabTemp(true DEBUGARG("UNBOXing a nullable")); + lvaSetStruct(tmp, resolvedToken.hClass, true /* unsafe value cls check */); - if (fgAddrCouldBeNull(obj)) - { - op1->gtFlags |= GTF_EXCEPT; - } + op2 = gtNewLclvNode(tmp, TYP_STRUCT); + op1 = impAssignStruct(op2, op1, resolvedToken.hClass, (unsigned)CHECK_SPILL_ALL); + assert(op1->gtType == TYP_VOID); // We must be assigning the return struct to the temp. - if (compIsForInlining() && - impInlineIsGuaranteedThisDerefBeforeAnySideEffects(op2, nullptr, obj, - impInlineInfo->inlArgInfo)) - { - impInlineInfo->thisDereferencedFirst = true; - } + op2 = gtNewLclvNode(tmp, TYP_STRUCT); + op2 = gtNewOperNode(GT_ADDR, TYP_BYREF, op2); + op1 = gtNewOperNode(GT_COMMA, TYP_BYREF, op1, op2); } - break; - - case CORINFO_FIELD_STATIC_TLS: -#ifdef TARGET_X86 - // Legacy TLS access is implemented as intrinsic on x86 only. - op1 = gtNewFieldAddrNode(TYP_I_IMPL, resolvedToken.hField, nullptr, fieldInfo.offset); - op1->gtFlags |= GTF_FLD_TLS; // fgMorphExpandTlsField will handle the transformation. - - if (varTypeIsStruct(lclTyp)) - { - op1 = gtNewObjNode(fieldInfo.structType, op1); - op1->gtFlags |= GTF_IND_NONFAULTING; - } - else - { - op1 = gtNewIndir(lclTyp, op1, GTF_IND_NONFAULTING); - op1->gtFlags |= GTF_GLOB_REF; - } - break; -#else - fieldInfo.fieldAccessor = CORINFO_FIELD_STATIC_ADDR_HELPER; - FALLTHROUGH; -#endif - case CORINFO_FIELD_STATIC_ADDR_HELPER: - case CORINFO_FIELD_INSTANCE_HELPER: - case CORINFO_FIELD_INSTANCE_ADDR_HELPER: - op1 = gtNewRefCOMfield(obj, &resolvedToken, (CORINFO_ACCESS_FLAGS)aflags, &fieldInfo, lclTyp, - clsHnd, op2); - goto SPILL_APPEND; - - case CORINFO_FIELD_STATIC_ADDRESS: - case CORINFO_FIELD_STATIC_RVA_ADDRESS: - case CORINFO_FIELD_STATIC_SHARED_STATIC_HELPER: - case CORINFO_FIELD_STATIC_GENERICS_STATIC_HELPER: - case CORINFO_FIELD_STATIC_READYTORUN_HELPER: - op1 = impImportStaticFieldAccess(&resolvedToken, (CORINFO_ACCESS_FLAGS)aflags, &fieldInfo, - lclTyp); - break; - default: - assert(!"Unexpected fieldAccessor"); + assert(op1->gtType == TYP_BYREF); } - - if (lclTyp != TYP_STRUCT) + else { - assert(op1->OperIs(GT_FIELD, GT_IND)); - - if (prefixFlags & PREFIX_VOLATILE) - { - op1->gtFlags |= GTF_ORDER_SIDEEFF; // Prevent this from being reordered - op1->gtFlags |= GTF_IND_VOLATILE; - } - if ((prefixFlags & PREFIX_UNALIGNED) && !varTypeIsByte(lclTyp)) - { - op1->gtFlags |= GTF_IND_UNALIGNED; - } + assert(opcode == CEE_UNBOX_ANY); - // Currently, *all* TYP_REF statics are stored inside an "object[]" array that itself - // resides on the managed heap, and so we can use an unchecked write barrier for this - // store. Likewise if we're storing to a field of an on-heap object. - if ((lclTyp == TYP_REF) && - (((fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC) != 0) || obj->TypeIs(TYP_REF))) + if (helper == CORINFO_HELP_UNBOX) { - static_assert_no_msg(GTF_FLD_TGT_HEAP == GTF_IND_TGT_HEAP); - op1->gtFlags |= GTF_FLD_TGT_HEAP; + // Normal unbox helper returns a TYP_BYREF. + impPushOnStack(op1, tiRetVal); + oper = GT_OBJ; + goto OBJ; } - /* V4.0 allows assignment of i4 constant values to i8 type vars when IL verifier is bypassed (full - trust apps). The reason this works is that JIT stores an i4 constant in Gentree union during - importation and reads from the union as if it were a long during code generation. Though this - can potentially read garbage, one can get lucky to have this working correctly. - - This code pattern is generated by Dev10 MC++ compiler while storing to fields when compiled with - /O2 switch (default when compiling retail configs in Dev10) and a customer app has taken a - dependency on it. To be backward compatible, we will explicitly add an upward cast here so that - it works correctly always. - - Note that this is limited to x86 alone as there is no back compat to be addressed for Arm JIT - for V4.0. - */ - CLANG_FORMAT_COMMENT_ANCHOR; + assert(helper == CORINFO_HELP_UNBOX_NULLABLE && "Make sure the helper is nullable!"); -#ifndef TARGET_64BIT - // In UWP6.0 and beyond (post-.NET Core 2.0), we decided to let this cast from int to long be - // generated for ARM as well as x86, so the following IR will be accepted: - // STMTx (IL 0x... ???) - // * ASG long - // +--* LCL_VAR long - // \--* CNS_INT int 2 +#if FEATURE_MULTIREG_RET - if ((op1->TypeGet() != op2->TypeGet()) && op2->OperIsConst() && varTypeIsIntOrI(op2->TypeGet()) && - varTypeIsLong(op1->TypeGet())) + if (varTypeIsStruct(op1) && + IsMultiRegReturnedType(resolvedToken.hClass, CorInfoCallConvExtension::Managed)) { - op2 = gtNewCastNode(op1->TypeGet(), op2, false, op1->TypeGet()); - } -#endif + // Unbox nullable helper returns a TYP_STRUCT. + // For the multi-reg case we need to spill it to a temp so that + // we can pass the address to the unbox_nullable jit helper. -#ifdef TARGET_64BIT - // Automatic upcast for a GT_CNS_INT into TYP_I_IMPL - if ((op2->OperGet() == GT_CNS_INT) && varTypeIsI(lclTyp) && !varTypeIsI(op2->gtType)) - { - op2->gtType = TYP_I_IMPL; - } - else - { - // Allow a downcast of op2 from TYP_I_IMPL into a 32-bit Int for x86 JIT compatibility - // - if (varTypeIsI(op2->gtType) && (genActualType(lclTyp) == TYP_INT)) - { - op2 = gtNewCastNode(TYP_INT, op2, false, TYP_INT); - } - // Allow an upcast of op2 from a 32-bit Int into TYP_I_IMPL for x86 JIT compatibility - // - if (varTypeIsI(lclTyp) && (genActualType(op2->gtType) == TYP_INT)) - { - op2 = gtNewCastNode(TYP_I_IMPL, op2, false, TYP_I_IMPL); - } - } -#endif + unsigned tmp = lvaGrabTemp(true DEBUGARG("UNBOXing a register returnable nullable")); + lvaTable[tmp].lvIsMultiRegArg = true; + lvaSetStruct(tmp, resolvedToken.hClass, true /* unsafe value cls check */); + + op2 = gtNewLclvNode(tmp, TYP_STRUCT); + op1 = impAssignStruct(op2, op1, resolvedToken.hClass, (unsigned)CHECK_SPILL_ALL); + assert(op1->gtType == TYP_VOID); // We must be assigning the return struct to the temp. - // Insert an implicit FLOAT<->DOUBLE cast if needed. - op2 = impImplicitR4orR8Cast(op2, op1->TypeGet()); + op2 = gtNewLclvNode(tmp, TYP_STRUCT); + op2 = gtNewOperNode(GT_ADDR, TYP_BYREF, op2); + op1 = gtNewOperNode(GT_COMMA, TYP_BYREF, op1, op2); - op1 = gtNewAssignNode(op1, op2); - } + // In this case the return value of the unbox helper is TYP_BYREF. + // Make sure the right type is placed on the operand type stack. + impPushOnStack(op1, tiRetVal); - // Check if the class needs explicit initialization. - if (fieldInfo.fieldFlags & CORINFO_FLG_FIELD_INITCLASS) - { - GenTree* helperNode = impInitClass(&resolvedToken); - if (compDonotInline()) - { - return; + // Load the struct. + oper = GT_OBJ; + + assert(op1->gtType == TYP_BYREF); + + goto OBJ; } - if (helperNode != nullptr) + else + +#endif // !FEATURE_MULTIREG_RET + { - impAppendTree(helperNode, CHECK_SPILL_ALL, impCurStmtDI); + // If non register passable struct we have it materialized in the RetBuf. + assert(op1->gtType == TYP_STRUCT); + tiRetVal = verMakeTypeInfo(resolvedToken.hClass); + assert(tiRetVal.IsValueClass()); } } - if (lclTyp == TYP_STRUCT) - { - op1 = impAssignStruct(op1, op2, CHECK_SPILL_ALL); - } - goto SPILL_APPEND; + impPushOnStack(op1, tiRetVal); } + break; - case CEE_NEWARR: + case CEE_BOX: { + /* Get the Class index */ + assertImp(sz == sizeof(unsigned)); - /* Get the class type index operand */ - - _impResolveToken(CORINFO_TOKENKIND_Newarr); + _impResolveToken(CORINFO_TOKENKIND_Box); JITDUMP(" %08X", resolvedToken.token); - if (!opts.IsReadyToRun()) - { - // Need to restore array classes before creating array objects on the heap - op1 = impTokenToHandle(&resolvedToken, nullptr, true /*mustRestoreHandle*/); - if (op1 == nullptr) - { // compDonotInline() - return; - } - } - - tiRetVal = verMakeTypeInfo(resolvedToken.hClass); - accessAllowedResult = info.compCompHnd->canAccessClass(&resolvedToken, info.compMethodHnd, &calloutHelper); impHandleAccessAllowed(accessAllowedResult, &calloutHelper); - /* Form the arglist: array class handle, size */ - op2 = impPopStack().val; - assertImp(genActualTypeIsIntOrI(op2->gtType)); - -#ifdef TARGET_64BIT - // The array helper takes a native int for array length. - // So if we have an int, explicitly extend it to be a native int. - if (genActualType(op2->TypeGet()) != TYP_I_IMPL) - { - if (op2->IsIntegralConst()) - { - op2->gtType = TYP_I_IMPL; - } - else - { - bool isUnsigned = false; - op2 = gtNewCastNode(TYP_I_IMPL, op2, isUnsigned, TYP_I_IMPL); - } - } -#endif // TARGET_64BIT - -#ifdef FEATURE_READYTORUN - if (opts.IsReadyToRun()) + // Note BOX can be used on things that are not value classes, in which + // case we get a NOP. However the verifier's view of the type on the + // stack changes (in generic code a 'T' becomes a 'boxed T') + if (!eeIsValueClass(resolvedToken.hClass)) { - op1 = impReadyToRunHelperToTree(&resolvedToken, CORINFO_HELP_READYTORUN_NEWARR_1, TYP_REF, nullptr, - op2); - usingReadyToRunHelper = (op1 != nullptr); - - if (!usingReadyToRunHelper) - { - // TODO: ReadyToRun: When generic dictionary lookups are necessary, replace the lookup call - // and the newarr call with a single call to a dynamic R2R cell that will: - // 1) Load the context - // 2) Perform the generic dictionary lookup and caching, and generate the appropriate stub - // 3) Allocate the new array - // Reason: performance (today, we'll always use the slow helper for the R2R generics case) - - // Need to restore array classes before creating array objects on the heap - op1 = impTokenToHandle(&resolvedToken, nullptr, true /*mustRestoreHandle*/); - if (op1 == nullptr) - { // compDonotInline() - return; - } - } + JITDUMP("\n Importing BOX(refClass) as NOP\n"); + verCurrentState.esStack[verCurrentState.esStackDepth - 1].seTypeInfo = tiRetVal; + break; } - if (!usingReadyToRunHelper) -#endif + bool isByRefLike = + (info.compCompHnd->getClassAttribs(resolvedToken.hClass) & CORINFO_FLG_BYREF_LIKE) != 0; + if (isByRefLike) { - /* Create a call to 'new' */ - - // Note that this only works for shared generic code because the same helper is used for all - // reference array types - op1 = - gtNewHelperCallNode(info.compCompHnd->getNewArrHelper(resolvedToken.hClass), TYP_REF, op1, op2); + // For ByRefLike types we are required to either fold the + // recognized patterns in impBoxPatternMatch or otherwise + // throw InvalidProgramException at runtime. In either case + // we will need to spill side effects of the expression. + impSpillSideEffects(false, CHECK_SPILL_ALL DEBUGARG("Required for box of ByRefLike type")); } - op1->AsCall()->compileTimeHelperArgumentHandle = (CORINFO_GENERIC_HANDLE)resolvedToken.hClass; - - // Remember that this function contains 'new' of an SD array. - optMethodFlags |= OMF_HAS_NEWARRAY; - - /* Push the result of the call on the stack */ - - impPushOnStack(op1, tiRetVal); - - callTyp = TYP_REF; - } - break; - - case CEE_LOCALLOC: - // We don't allow locallocs inside handlers - if (block->hasHndIndex()) + // Look ahead for box idioms + int matched = impBoxPatternMatch(&resolvedToken, codeAddr + sz, codeEndp, + isByRefLike ? BoxPatterns::IsByRefLike : BoxPatterns::None); + if (matched >= 0) { - BADCODE("Localloc can't be inside handler"); + // Skip the matched IL instructions + sz += matched; + break; } - // Get the size to allocate - - op2 = impPopStack().val; - assertImp(genActualTypeIsIntOrI(op2->gtType)); - - if (verCurrentState.esStackDepth != 0) + if (isByRefLike) { - BADCODE("Localloc can only be used when the stack is empty"); + // ByRefLike types are supported in boxing scenarios when the instruction can be elided + // due to a recognized pattern above. If the pattern is not recognized, the code is invalid. + BADCODE("ByRefLike types cannot be boxed"); } - - // If the localloc is not in a loop and its size is a small constant, - // create a new local var of TYP_BLK and return its address. + else { - bool convertedToLocal = false; - - // Need to aggressively fold here, as even fixed-size locallocs - // will have casts in the way. - op2 = gtFoldExpr(op2); - - if (op2->IsIntegralConst()) + impImportAndPushBox(&resolvedToken); + if (compDonotInline()) { - const ssize_t allocSize = op2->AsIntCon()->IconValue(); - - bool bbInALoop = impBlockIsInALoop(block); - - if (allocSize == 0) - { - // Result is nullptr - JITDUMP("Converting stackalloc of 0 bytes to push null unmanaged pointer\n"); - op1 = gtNewIconNode(0, TYP_I_IMPL); - convertedToLocal = true; - } - else if ((allocSize > 0) && !bbInALoop) - { - // Get the size threshold for local conversion - ssize_t maxSize = DEFAULT_MAX_LOCALLOC_TO_LOCAL_SIZE; - -#ifdef DEBUG - // Optionally allow this to be modified - maxSize = JitConfig.JitStackAllocToLocalSize(); -#endif // DEBUG - - if (allocSize <= maxSize) - { - const unsigned stackallocAsLocal = lvaGrabTemp(false DEBUGARG("stackallocLocal")); - JITDUMP("Converting stackalloc of %zd bytes to new local V%02u\n", allocSize, - stackallocAsLocal); - lvaTable[stackallocAsLocal].lvType = TYP_BLK; - lvaTable[stackallocAsLocal].lvExactSize = (unsigned)allocSize; - lvaTable[stackallocAsLocal].lvHasLdAddrOp = true; - lvaTable[stackallocAsLocal].lvIsUnsafeBuffer = true; - op1 = gtNewLclvNode(stackallocAsLocal, TYP_BLK); - op1 = gtNewOperNode(GT_ADDR, TYP_I_IMPL, op1); - convertedToLocal = true; - - if (!this->opts.compDbgEnC) - { - // Ensure we have stack security for this method. - // Reorder layout since the converted localloc is treated as an unsafe buffer. - setNeedsGSSecurityCookie(); - compGSReorderStackLayout = true; - } - } - } + return; } + } + } + break; - if (!convertedToLocal) - { - // Bail out if inlining and the localloc was not converted. - // - // Note we might consider allowing the inline, if the call - // site is not in a loop. - if (compIsForInlining()) - { - InlineObservation obs = op2->IsIntegralConst() - ? InlineObservation::CALLEE_LOCALLOC_TOO_LARGE - : InlineObservation::CALLSITE_LOCALLOC_SIZE_UNKNOWN; - compInlineResult->NoteFatal(obs); - return; - } + case CEE_SIZEOF: - op1 = gtNewOperNode(GT_LCLHEAP, TYP_I_IMPL, op2); - // May throw a stack overflow exception. Obviously, we don't want locallocs to be CSE'd. - op1->gtFlags |= (GTF_EXCEPT | GTF_DONT_CSE); + /* Get the Class index */ + assertImp(sz == sizeof(unsigned)); - // Ensure we have stack security for this method. - setNeedsGSSecurityCookie(); + _impResolveToken(CORINFO_TOKENKIND_Class); - /* The FP register may not be back to the original value at the end - of the method, even if the frame size is 0, as localloc may - have modified it. So we will HAVE to reset it */ - compLocallocUsed = true; - } - else - { - compLocallocOptimized = true; - } - } + JITDUMP(" %08X", resolvedToken.token); + op1 = gtNewIconNode(info.compCompHnd->getClassSize(resolvedToken.hClass)); impPushOnStack(op1, tiRetVal); break; - case CEE_ISINST: - { - /* Get the type token */ + case CEE_CASTCLASS: + + /* Get the Class index */ + assertImp(sz == sizeof(unsigned)); _impResolveToken(CORINFO_TOKENKIND_Casting); @@ -9955,7 +17231,13 @@ void Compiler::impImportBlockCode(BasicBlock* block) op1 = impPopStack().val; - GenTree* optTree = impOptimizeCastClassOrIsInst(op1, &resolvedToken, false); + /* Pop the address and create the 'checked cast' helper call */ + + // At this point we expect typeRef to contain the token, op1 to contain the value being cast, + // and op2 to contain code that creates the type handle corresponding to typeRef + CASTCLASS: + { + GenTree* optTree = impOptimizeCastClassOrIsInst(op1, &resolvedToken, true); if (optTree != nullptr) { @@ -9968,19 +17250,19 @@ void Compiler::impImportBlockCode(BasicBlock* block) if (opts.IsReadyToRun()) { GenTreeCall* opLookup = - impReadyToRunHelperToTree(&resolvedToken, CORINFO_HELP_READYTORUN_ISINSTANCEOF, TYP_REF, - nullptr, op1); + impReadyToRunHelperToTree(&resolvedToken, CORINFO_HELP_READYTORUN_CHKCAST, TYP_REF, nullptr, + op1); usingReadyToRunHelper = (opLookup != nullptr); op1 = (usingReadyToRunHelper ? opLookup : op1); if (!usingReadyToRunHelper) { // TODO: ReadyToRun: When generic dictionary lookups are necessary, replace the lookup call - // and the isinstanceof_any call with a single call to a dynamic R2R cell that will: + // and the chkcastany call with a single call to a dynamic R2R cell that will: // 1) Load the context // 2) Perform the generic dictionary lookup and caching, and generate the appropriate // stub - // 3) Perform the 'is instance' check on the input object + // 3) Check the object on the stack for the type-cast // Reason: performance (today, we'll always use the slow helper for the R2R generics case) op2 = impTokenToHandle(&resolvedToken, nullptr, false); @@ -9994,3931 +17276,5780 @@ void Compiler::impImportBlockCode(BasicBlock* block) if (!usingReadyToRunHelper) #endif { - op1 = impCastClassOrIsInstToTree(op1, op2, &resolvedToken, false, opcodeOffs); + op1 = impCastClassOrIsInstToTree(op1, op2, &resolvedToken, true, opcodeOffs); } if (compDonotInline()) { return; } + /* Push the result back on the stack */ impPushOnStack(op1, tiRetVal); } - break; } + break; - case CEE_REFANYVAL: - { + case CEE_THROW: - // get the class handle and make a ICON node out of it + // Any block with a throw is rarely executed. + block->bbSetRunRarely(); + + // Pop the exception object and create the 'throw' helper call + op1 = gtNewHelperCallNode(CORINFO_HELP_THROW, TYP_VOID, impPopStack().val); + + // Fall through to clear out the eval stack. + + EVAL_APPEND: + if (verCurrentState.esStackDepth > 0) + { + impEvalSideEffects(); + } + + assert(verCurrentState.esStackDepth == 0); + + goto APPEND; + + case CEE_RETHROW: + + assert(!compIsForInlining()); + + if (info.compXcptnsCount == 0) + { + BADCODE("rethrow outside catch"); + } + + /* Create the 'rethrow' helper call */ + + op1 = gtNewHelperCallNode(CORINFO_HELP_RETHROW, TYP_VOID); + + goto EVAL_APPEND; + + case CEE_INITOBJ: + + assertImp(sz == sizeof(unsigned)); _impResolveToken(CORINFO_TOKENKIND_Class); JITDUMP(" %08X", resolvedToken.token); - op2 = impTokenToHandle(&resolvedToken); - if (op2 == nullptr) - { // compDonotInline() - return; + op2 = gtNewIconNode(0); // Value + op1 = impPopStack().val; // Dest + + if (eeIsValueClass(resolvedToken.hClass)) + { + op1 = gtNewStructVal(typGetObjLayout(resolvedToken.hClass), op1); + if (op1->OperIs(GT_OBJ)) + { + gtSetObjGcInfo(op1->AsObj()); + } + } + else + { + size = info.compCompHnd->getClassSize(resolvedToken.hClass); + assert(size == TARGET_POINTER_SIZE); + op1 = gtNewBlockVal(op1, size); } - op1 = impPopStack().val; - // make certain it is normalized; - op1 = impNormStructVal(op1, impGetRefAnyClass(), CHECK_SPILL_ALL); + op1 = gtNewBlkOpNode(op1, op2, (prefixFlags & PREFIX_VOLATILE) != 0, false); + goto SPILL_APPEND; - // Call helper GETREFANY(classHandle, op1); - GenTreeCall* helperCall = gtNewHelperCallNode(CORINFO_HELP_GETREFANY, TYP_BYREF); - NewCallArg clsHandleArg = NewCallArg::Primitive(op2); - NewCallArg typedRefArg = NewCallArg::Struct(op1, TYP_STRUCT, impGetRefAnyClass()); - helperCall->gtArgs.PushFront(this, clsHandleArg, typedRefArg); - helperCall->gtFlags |= (op1->gtFlags | op2->gtFlags) & GTF_ALL_EFFECT; - op1 = helperCall; + case CEE_INITBLK: - impPushOnStack(op1, tiRetVal); - break; - } - case CEE_REFANYTYPE: - { - op1 = impPopStack().val; + op3 = impPopStack().val; // Size + op2 = impPopStack().val; // Value + op1 = impPopStack().val; // Dst addr - if (op1->OperIs(GT_MKREFANY)) + if (op3->IsCnsIntOrI()) { - // The pointer may have side-effects - if (op1->AsOp()->gtOp1->gtFlags & GTF_SIDE_EFFECT) + size = (unsigned)op3->AsIntConCommon()->IconValue(); + op1 = new (this, GT_BLK) GenTreeBlk(GT_BLK, TYP_STRUCT, op1, typGetBlkLayout(size)); + op1 = gtNewBlkOpNode(op1, op2, (prefixFlags & PREFIX_VOLATILE) != 0, false); + } + else + { + if (!op2->IsIntegralConst(0)) { - impAppendTree(op1->AsOp()->gtOp1, CHECK_SPILL_ALL, impCurStmtDI); -#ifdef DEBUG - impNoteLastILoffs(); -#endif + op2 = gtNewOperNode(GT_INIT_VAL, TYP_INT, op2); } - // We already have the class handle - op1 = op1->AsOp()->gtOp2; + op1 = new (this, GT_STORE_DYN_BLK) GenTreeStoreDynBlk(op1, op2, op3); + size = 0; + + if ((prefixFlags & PREFIX_VOLATILE) != 0) + { + op1->gtFlags |= GTF_BLK_VOLATILE; + } + } + + goto SPILL_APPEND; + + case CEE_CPBLK: + + op3 = impPopStack().val; // Size + op2 = impPopStack().val; // Src addr + op1 = impPopStack().val; // Dst addr + + if (op2->OperGet() == GT_ADDR) + { + op2 = op2->AsOp()->gtOp1; } else { - // Get the address of the refany - op1 = impGetStructAddr(op1, impGetRefAnyClass(), CHECK_SPILL_ALL, /* willDeref */ true); + op2 = gtNewOperNode(GT_IND, TYP_STRUCT, op2); + } - // Fetch the type from the correct slot - op1 = gtNewOperNode(GT_ADD, TYP_BYREF, op1, - gtNewIconNode(OFFSETOF__CORINFO_TypedReference__type, TYP_I_IMPL)); - op1 = gtNewOperNode(GT_IND, TYP_BYREF, op1); + if (op3->IsCnsIntOrI()) + { + size = (unsigned)op3->AsIntConCommon()->IconValue(); + op1 = new (this, GT_BLK) GenTreeBlk(GT_BLK, TYP_STRUCT, op1, typGetBlkLayout(size)); + op1 = gtNewBlkOpNode(op1, op2, (prefixFlags & PREFIX_VOLATILE) != 0, true); + } + else + { + op1 = new (this, GT_STORE_DYN_BLK) GenTreeStoreDynBlk(op1, op2, op3); + size = 0; + + if ((prefixFlags & PREFIX_VOLATILE) != 0) + { + op1->gtFlags |= GTF_BLK_VOLATILE; + } } - // Convert native TypeHandle to RuntimeTypeHandle. - op1 = gtNewHelperCallNode(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPEHANDLE_MAYBENULL, TYP_STRUCT, op1); + goto SPILL_APPEND; - CORINFO_CLASS_HANDLE classHandle = impGetTypeHandleClass(); + case CEE_CPOBJ: - // The handle struct is returned in register - op1->AsCall()->gtReturnType = GetRuntimeHandleUnderlyingType(); - op1->AsCall()->gtRetClsHnd = classHandle; -#if FEATURE_MULTIREG_RET - op1->AsCall()->InitializeStructReturnType(this, classHandle, op1->AsCall()->GetUnmanagedCallConv()); -#endif + assertImp(sz == sizeof(unsigned)); - tiRetVal = typeInfo(TI_STRUCT, classHandle); - impPushOnStack(op1, tiRetVal); - } - break; + _impResolveToken(CORINFO_TOKENKIND_Class); - case CEE_LDTOKEN: + JITDUMP(" %08X", resolvedToken.token); + + if (!eeIsValueClass(resolvedToken.hClass)) + { + op1 = impPopStack().val; // address to load from + + impBashVarAddrsToI(op1); + + assertImp(genActualType(op1->gtType) == TYP_I_IMPL || op1->gtType == TYP_BYREF); + + op1 = gtNewOperNode(GT_IND, TYP_REF, op1); + op1->gtFlags |= GTF_EXCEPT | GTF_GLOB_REF; + + impPushOnStack(op1, typeInfo()); + opcode = CEE_STIND_REF; + lclTyp = TYP_REF; + goto STIND; + } + + op2 = impPopStack().val; // Src + op1 = impPopStack().val; // Dest + op1 = gtNewCpObjNode(op1, op2, resolvedToken.hClass, ((prefixFlags & PREFIX_VOLATILE) != 0)); + goto SPILL_APPEND; + + case CEE_STOBJ: { - /* Get the Class index */ assertImp(sz == sizeof(unsigned)); - lastLoadToken = codeAddr; - _impResolveToken(CORINFO_TOKENKIND_Ldtoken); - tokenType = info.compCompHnd->getTokenTypeAsHandle(&resolvedToken); + _impResolveToken(CORINFO_TOKENKIND_Class); - op1 = impTokenToHandle(&resolvedToken, nullptr, true); - if (op1 == nullptr) + JITDUMP(" %08X", resolvedToken.token); + + if (eeIsValueClass(resolvedToken.hClass)) + { + lclTyp = TYP_STRUCT; + } + else + { + lclTyp = TYP_REF; + } + + if (lclTyp == TYP_REF) + { + opcode = CEE_STIND_REF; + goto STIND; + } + + CorInfoType jitTyp = info.compCompHnd->asCorInfoType(resolvedToken.hClass); + if (impIsPrimitive(jitTyp)) + { + lclTyp = JITtype2varType(jitTyp); + goto STIND; + } + + op2 = impPopStack().val; // Value + op1 = impPopStack().val; // Ptr + + assertImp(varTypeIsStruct(op2)); + + op1 = impAssignStructPtr(op1, op2, resolvedToken.hClass, (unsigned)CHECK_SPILL_ALL); + + if (op1->OperIsBlkOp() && (prefixFlags & PREFIX_UNALIGNED)) + { + op1->gtFlags |= GTF_BLK_UNALIGNED; + } + goto SPILL_APPEND; + } + + case CEE_MKREFANY: + + assert(!compIsForInlining()); + + // Being lazy here. Refanys are tricky in terms of gc tracking. + // Since it is uncommon, just don't perform struct promotion in any method that contains mkrefany. + + JITDUMP("disabling struct promotion because of mkrefany\n"); + fgNoStructPromotion = true; + + oper = GT_MKREFANY; + assertImp(sz == sizeof(unsigned)); + + _impResolveToken(CORINFO_TOKENKIND_Class); + + JITDUMP(" %08X", resolvedToken.token); + + op2 = impTokenToHandle(&resolvedToken, nullptr, true); + if (op2 == nullptr) { // compDonotInline() return; } - helper = CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPEHANDLE; - assert(resolvedToken.hClass != nullptr); + accessAllowedResult = + info.compCompHnd->canAccessClass(&resolvedToken, info.compMethodHnd, &calloutHelper); + impHandleAccessAllowed(accessAllowedResult, &calloutHelper); + + op1 = impPopStack().val; + + // @SPECVIOLATION: TYP_INT should not be allowed here by a strict reading of the spec. + // But JIT32 allowed it, so we continue to allow it. + assertImp(op1->TypeGet() == TYP_BYREF || op1->TypeGet() == TYP_I_IMPL || op1->TypeGet() == TYP_INT); + + // MKREFANY returns a struct. op2 is the class token. + op1 = gtNewOperNode(oper, TYP_STRUCT, op1, op2); + + impPushOnStack(op1, verMakeTypeInfo(impGetRefAnyClass())); + break; + + case CEE_LDOBJ: + { + oper = GT_OBJ; + assertImp(sz == sizeof(unsigned)); + + _impResolveToken(CORINFO_TOKENKIND_Class); + + JITDUMP(" %08X", resolvedToken.token); + + OBJ: + + tiRetVal = verMakeTypeInfo(resolvedToken.hClass); - if (resolvedToken.hMethod != nullptr) + if (eeIsValueClass(resolvedToken.hClass)) { - helper = CORINFO_HELP_METHODDESC_TO_STUBRUNTIMEMETHOD; + lclTyp = TYP_STRUCT; } - else if (resolvedToken.hField != nullptr) + else { - helper = CORINFO_HELP_FIELDDESC_TO_STUBRUNTIMEFIELD; + lclTyp = TYP_REF; + opcode = CEE_LDIND_REF; + goto LDIND; } - op1 = gtNewHelperCallNode(helper, TYP_STRUCT, op1); + op1 = impPopStack().val; - // The handle struct is returned in register and - // it could be consumed both as `TYP_STRUCT` and `TYP_REF`. - op1->AsCall()->gtReturnType = GetRuntimeHandleUnderlyingType(); -#if FEATURE_MULTIREG_RET - op1->AsCall()->InitializeStructReturnType(this, tokenType, op1->AsCall()->GetUnmanagedCallConv()); -#endif - op1->AsCall()->gtRetClsHnd = tokenType; + assertImp(op1->TypeGet() == TYP_BYREF || op1->TypeGet() == TYP_I_IMPL); + + CorInfoType jitTyp = info.compCompHnd->asCorInfoType(resolvedToken.hClass); + if (impIsPrimitive(jitTyp)) + { + op1 = gtNewOperNode(GT_IND, JITtype2varType(jitTyp), op1); + + // Could point anywhere, example a boxed class static int + op1->gtFlags |= GTF_GLOB_REF; + assertImp(varTypeIsArithmetic(op1->gtType)); + } + else + { + // OBJ returns a struct + // and an inline argument which is the class token of the loaded obj + op1 = gtNewObjNode(resolvedToken.hClass, op1); + } + op1->gtFlags |= GTF_EXCEPT; + + if (prefixFlags & PREFIX_UNALIGNED) + { + op1->gtFlags |= GTF_IND_UNALIGNED; + } - tiRetVal = verMakeTypeInfo(tokenType); impPushOnStack(op1, tiRetVal); + break; } - break; - case CEE_UNBOX: - case CEE_UNBOX_ANY: - { - /* Get the Class index */ - assertImp(sz == sizeof(unsigned)); + case CEE_LDLEN: + op1 = impPopStack().val; + if (opts.OptimizationEnabled()) + { + /* Use GT_ARR_LENGTH operator so rng check opts see this */ + GenTreeArrLen* arrLen = gtNewArrLen(TYP_INT, op1, OFFSETOF__CORINFO_Array__length, block); - _impResolveToken(CORINFO_TOKENKIND_Class); + op1 = arrLen; + } + else + { + /* Create the expression "*(array_addr + ArrLenOffs)" */ + op1 = gtNewOperNode(GT_ADD, TYP_BYREF, op1, + gtNewIconNode(OFFSETOF__CORINFO_Array__length, TYP_I_IMPL)); + op1 = gtNewIndir(TYP_INT, op1); + } - JITDUMP(" %08X", resolvedToken.token); + /* Push the result back on the stack */ + impPushOnStack(op1, tiRetVal); + break; - bool runtimeLookup; - op2 = impTokenToHandle(&resolvedToken, &runtimeLookup); - if (op2 == nullptr) + case CEE_BREAK: + op1 = gtNewHelperCallNode(CORINFO_HELP_USER_BREAKPOINT, TYP_VOID); + goto SPILL_APPEND; + + case CEE_NOP: + if (opts.compDbgCode) { - assert(compDonotInline()); - return; + op1 = new (this, GT_NO_OP) GenTree(GT_NO_OP, TYP_VOID); + goto SPILL_APPEND; } + break; - // Run this always so we can get access exceptions even with SkipVerification. - accessAllowedResult = - info.compCompHnd->canAccessClass(&resolvedToken, info.compMethodHnd, &calloutHelper); - impHandleAccessAllowed(accessAllowedResult, &calloutHelper); + /******************************** NYI *******************************/ - if (opcode == CEE_UNBOX_ANY && !eeIsValueClass(resolvedToken.hClass)) + case 0xCC: + OutputDebugStringA("CLR: Invalid x86 breakpoint in IL stream\n"); + FALLTHROUGH; + + case CEE_ILLEGAL: + case CEE_MACRO_END: + + default: + if (compIsForInlining()) { - JITDUMP("\n Importing UNBOX.ANY(refClass) as CASTCLASS\n"); - op1 = impPopStack().val; - goto CASTCLASS; + compInlineResult->NoteFatal(InlineObservation::CALLEE_COMPILATION_ERROR); + return; } - /* Pop the object and create the unbox helper call */ - /* You might think that for UNBOX_ANY we need to push a different */ - /* (non-byref) type, but here we're making the tiRetVal that is used */ - /* for the intermediate pointer which we then transfer onto the OBJ */ - /* instruction. OBJ then creates the appropriate tiRetVal. */ - - op1 = impPopStack().val; - assertImp(op1->gtType == TYP_REF); + BADCODE3("unknown opcode", ": %02X", (int)opcode); + } - helper = info.compCompHnd->getUnBoxHelper(resolvedToken.hClass); - assert(helper == CORINFO_HELP_UNBOX || helper == CORINFO_HELP_UNBOX_NULLABLE); + codeAddr += sz; + prevOpcode = opcode; - // Check legality and profitability of inline expansion for unboxing. - const bool canExpandInline = (helper == CORINFO_HELP_UNBOX); - const bool shouldExpandInline = !compCurBB->isRunRarely() && opts.OptimizationEnabled(); + prefixFlags = 0; + } - if (canExpandInline && shouldExpandInline) - { - // See if we know anything about the type of op1, the object being unboxed. - bool isExact = false; - bool isNonNull = false; - CORINFO_CLASS_HANDLE clsHnd = gtGetClassHandle(op1, &isExact, &isNonNull); + return; +#undef _impResolveToken +} +#ifdef _PREFAST_ +#pragma warning(pop) +#endif - // We can skip the "exact" bit here as we are comparing to a value class. - // compareTypesForEquality should bail on comparisons for shared value classes. - if (clsHnd != NO_CLASS_HANDLE) - { - const TypeCompareState compare = - info.compCompHnd->compareTypesForEquality(resolvedToken.hClass, clsHnd); +// Push a local/argument treeon the operand stack +void Compiler::impPushVar(GenTree* op, typeInfo tiRetVal) +{ + tiRetVal.NormaliseForStack(); - if (compare == TypeCompareState::Must) - { - JITDUMP("\nOptimizing %s (%s) -- type test will succeed\n", - opcode == CEE_UNBOX ? "UNBOX" : "UNBOX.ANY", eeGetClassName(clsHnd)); + if (verTrackObjCtorInitState && (verCurrentState.thisInitialized != TIS_Init) && tiRetVal.IsThisPtr()) + { + tiRetVal.SetUninitialisedObjRef(); + } - // For UNBOX, null check (if necessary), and then leave the box payload byref on the stack. - if (opcode == CEE_UNBOX) - { - GenTree* cloneOperand; - op1 = impCloneExpr(op1, &cloneOperand, NO_CLASS_HANDLE, CHECK_SPILL_ALL, - nullptr DEBUGARG("optimized unbox clone")); + impPushOnStack(op, tiRetVal); +} - GenTree* boxPayloadOffset = gtNewIconNode(TARGET_POINTER_SIZE, TYP_I_IMPL); - GenTree* boxPayloadAddress = - gtNewOperNode(GT_ADD, TYP_BYREF, cloneOperand, boxPayloadOffset); - GenTree* nullcheck = gtNewNullCheck(op1, block); - // Add an ordering dependency between the null - // check and forming the byref; the JIT assumes - // in many places that the only legal null - // byref is literally 0, and since the byref - // leaks out here, we need to ensure it is - // nullchecked. - nullcheck->gtFlags |= GTF_ORDER_SIDEEFF; - boxPayloadAddress->gtFlags |= GTF_ORDER_SIDEEFF; - GenTree* result = gtNewOperNode(GT_COMMA, TYP_BYREF, nullcheck, boxPayloadAddress); - impPushOnStack(result, tiRetVal); - break; - } +//------------------------------------------------------------------------ +// impCreateLocal: create a GT_LCL_VAR node to access a local that might need to be normalized on load +// +// Arguments: +// lclNum -- The index into lvaTable +// offset -- The offset to associate with the node +// +// Returns: +// The node +// +GenTreeLclVar* Compiler::impCreateLocalNode(unsigned lclNum DEBUGARG(IL_OFFSET offset)) +{ + var_types lclTyp; - // For UNBOX.ANY load the struct from the box payload byref (the load will nullcheck) - assert(opcode == CEE_UNBOX_ANY); - GenTree* boxPayloadOffset = gtNewIconNode(TARGET_POINTER_SIZE, TYP_I_IMPL); - GenTree* boxPayloadAddress = gtNewOperNode(GT_ADD, TYP_BYREF, op1, boxPayloadOffset); - impPushOnStack(boxPayloadAddress, tiRetVal); - oper = GT_OBJ; - goto OBJ; - } - else - { - JITDUMP("\nUnable to optimize %s -- can't resolve type comparison\n", - opcode == CEE_UNBOX ? "UNBOX" : "UNBOX.ANY"); - } - } - else - { - JITDUMP("\nUnable to optimize %s -- class for [%06u] not known\n", - opcode == CEE_UNBOX ? "UNBOX" : "UNBOX.ANY", dspTreeID(op1)); - } + if (lvaTable[lclNum].lvNormalizeOnLoad()) + { + lclTyp = lvaGetRealType(lclNum); + } + else + { + lclTyp = lvaGetActualType(lclNum); + } - JITDUMP("\n Importing %s as inline sequence\n", opcode == CEE_UNBOX ? "UNBOX" : "UNBOX.ANY"); - // we are doing normal unboxing - // inline the common case of the unbox helper - // UNBOX(exp) morphs into - // clone = pop(exp); - // ((*clone == typeToken) ? nop : helper(clone, typeToken)); - // push(clone + TARGET_POINTER_SIZE) - // - GenTree* cloneOperand; - op1 = impCloneExpr(op1, &cloneOperand, NO_CLASS_HANDLE, CHECK_SPILL_ALL, - nullptr DEBUGARG("inline UNBOX clone1")); - op1 = gtNewMethodTableLookup(op1); + return gtNewLclvNode(lclNum, lclTyp DEBUGARG(offset)); +} - GenTree* condBox = gtNewOperNode(GT_EQ, TYP_INT, op1, op2); +// Load a local/argument on the operand stack +// lclNum is an index into lvaTable *NOT* the arg/lcl index in the IL +void Compiler::impLoadVar(unsigned lclNum, IL_OFFSET offset, const typeInfo& tiRetVal) +{ + impPushVar(impCreateLocalNode(lclNum DEBUGARG(offset)), tiRetVal); +} - op1 = impCloneExpr(cloneOperand, &cloneOperand, NO_CLASS_HANDLE, CHECK_SPILL_ALL, - nullptr DEBUGARG("inline UNBOX clone2")); - op2 = impTokenToHandle(&resolvedToken); - if (op2 == nullptr) - { // compDonotInline() - return; - } - op1 = gtNewHelperCallNode(helper, TYP_VOID, op2, op1); +// Load an argument on the operand stack +// Shared by the various CEE_LDARG opcodes +// ilArgNum is the argument index as specified in IL. +// It will be mapped to the correct lvaTable index +void Compiler::impLoadArg(unsigned ilArgNum, IL_OFFSET offset) +{ + Verify(ilArgNum < info.compILargsCount, "bad arg num"); - op1 = new (this, GT_COLON) GenTreeColon(TYP_VOID, gtNewNothingNode(), op1); - op1 = gtNewQmarkNode(TYP_VOID, condBox, op1->AsColon()); + if (compIsForInlining()) + { + if (ilArgNum >= info.compArgsCount) + { + compInlineResult->NoteFatal(InlineObservation::CALLEE_BAD_ARGUMENT_NUMBER); + return; + } - // QMARK nodes cannot reside on the evaluation stack. Because there - // may be other trees on the evaluation stack that side-effect the - // sources of the UNBOX operation we must spill the stack. + impPushVar(impInlineFetchArg(ilArgNum, impInlineInfo->inlArgInfo, impInlineInfo->lclVarInfo), + impInlineInfo->lclVarInfo[ilArgNum].lclVerTypeInfo); + } + else + { + if (ilArgNum >= info.compArgsCount) + { + BADCODE("Bad IL"); + } - impAppendTree(op1, CHECK_SPILL_ALL, impCurStmtDI); + unsigned lclNum = compMapILargNum(ilArgNum); // account for possible hidden param - // Create the address-expression to reference past the object header - // to the beginning of the value-type. Today this means adjusting - // past the base of the objects vtable field which is pointer sized. + if (lclNum == info.compThisArg) + { + lclNum = lvaArg0Var; + } - op2 = gtNewIconNode(TARGET_POINTER_SIZE, TYP_I_IMPL); - op1 = gtNewOperNode(GT_ADD, TYP_BYREF, cloneOperand, op2); - } - else - { - JITDUMP("\n Importing %s as helper call because %s\n", opcode == CEE_UNBOX ? "UNBOX" : "UNBOX.ANY", - canExpandInline ? "want smaller code or faster jitting" : "inline expansion not legal"); + impLoadVar(lclNum, offset); + } +} - // Don't optimize, just call the helper and be done with it - op1 = gtNewHelperCallNode(helper, - (var_types)((helper == CORINFO_HELP_UNBOX) ? TYP_BYREF : TYP_STRUCT), op2, - op1); - if (op1->gtType == TYP_STRUCT) - { - op1->AsCall()->gtRetClsHnd = resolvedToken.hClass; - } - } +// Load a local on the operand stack +// Shared by the various CEE_LDLOC opcodes +// ilLclNum is the local index as specified in IL. +// It will be mapped to the correct lvaTable index +void Compiler::impLoadLoc(unsigned ilLclNum, IL_OFFSET offset) +{ + if (compIsForInlining()) + { + if (ilLclNum >= info.compMethodInfo->locals.numArgs) + { + compInlineResult->NoteFatal(InlineObservation::CALLEE_BAD_LOCAL_NUMBER); + return; + } - assert((helper == CORINFO_HELP_UNBOX && op1->gtType == TYP_BYREF) || // Unbox helper returns a byref. - (helper == CORINFO_HELP_UNBOX_NULLABLE && - varTypeIsStruct(op1)) // UnboxNullable helper returns a struct. - ); + // Get the local type + var_types lclTyp = impInlineInfo->lclVarInfo[ilLclNum + impInlineInfo->argCnt].lclTypeInfo; - /* - ---------------------------------------------------------------------- - | \ helper | | | - | \ | | | - | \ | CORINFO_HELP_UNBOX | CORINFO_HELP_UNBOX_NULLABLE | - | \ | (which returns a BYREF) | (which returns a STRUCT) | | - | opcode \ | | | - |--------------------------------------------------------------------- - | UNBOX | push the BYREF | spill the STRUCT to a local, | - | | | push the BYREF to this local | - |--------------------------------------------------------------------- - | UNBOX_ANY | push a GT_OBJ of | push the STRUCT | - | | the BYREF | For Linux when the | - | | | struct is returned in two | - | | | registers create a temp | - | | | which address is passed to | - | | | the unbox_nullable helper. | - |--------------------------------------------------------------------- - */ + typeInfo tiRetVal = impInlineInfo->lclVarInfo[ilLclNum + impInlineInfo->argCnt].lclVerTypeInfo; - if (opcode == CEE_UNBOX) - { - if (helper == CORINFO_HELP_UNBOX_NULLABLE) - { - // Unbox nullable helper returns a struct type. - // We need to spill it to a temp so than can take the address of it. - // Here we need unsafe value cls check, since the address of struct is taken to be used - // further along and potetially be exploitable. + /* Have we allocated a temp for this local? */ - unsigned tmp = lvaGrabTemp(true DEBUGARG("UNBOXing a nullable")); - lvaSetStruct(tmp, resolvedToken.hClass, true /* unsafe value cls check */); + unsigned lclNum = impInlineFetchLocal(ilLclNum DEBUGARG("Inline ldloc first use temp")); - op2 = gtNewLclvNode(tmp, TYP_STRUCT); - op1 = impAssignStruct(op2, op1, CHECK_SPILL_ALL); - assert(op1->gtType == TYP_VOID); // We must be assigning the return struct to the temp. + // All vars of inlined methods should be !lvNormalizeOnLoad() - op2 = gtNewLclvNode(tmp, TYP_STRUCT); - op2 = gtNewOperNode(GT_ADDR, TYP_BYREF, op2); - op1 = gtNewOperNode(GT_COMMA, TYP_BYREF, op1, op2); - } + assert(!lvaTable[lclNum].lvNormalizeOnLoad()); + lclTyp = genActualType(lclTyp); - assert(op1->gtType == TYP_BYREF); - } - else - { - assert(opcode == CEE_UNBOX_ANY); + impPushVar(gtNewLclvNode(lclNum, lclTyp), tiRetVal); + } + else + { + if (ilLclNum >= info.compMethodInfo->locals.numArgs) + { + BADCODE("Bad IL"); + } - if (helper == CORINFO_HELP_UNBOX) - { - // Normal unbox helper returns a TYP_BYREF. - impPushOnStack(op1, tiRetVal); - oper = GT_OBJ; - goto OBJ; - } + unsigned lclNum = info.compArgsCount + ilLclNum; - assert(helper == CORINFO_HELP_UNBOX_NULLABLE && "Make sure the helper is nullable!"); + impLoadVar(lclNum, offset); + } +} -#if FEATURE_MULTIREG_RET +#ifdef TARGET_ARM +/************************************************************************************** + * + * When assigning a vararg call src to a HFA lcl dest, mark that we cannot promote the + * dst struct, because struct promotion will turn it into a float/double variable while + * the rhs will be an int/long variable. We don't code generate assignment of int into + * a float, but there is nothing that might prevent us from doing so. The tree however + * would like: (=, (typ_float, typ_int)) or (GT_TRANSFER, (typ_float, typ_int)) + * + * tmpNum - the lcl dst variable num that is a struct. + * src - the src tree assigned to the dest that is a struct/int (when varargs call.) + * hClass - the type handle for the struct variable. + * + * TODO-ARM-CQ: [301608] This is a rare scenario with varargs and struct promotion coming into play, + * however, we could do a codegen of transferring from int to float registers + * (transfer, not a cast.) + * + */ +void Compiler::impMarkLclDstNotPromotable(unsigned tmpNum, GenTree* src, CORINFO_CLASS_HANDLE hClass) +{ + if (src->gtOper == GT_CALL && src->AsCall()->IsVarargs() && IsHfa(hClass)) + { + int hfaSlots = GetHfaCount(hClass); + var_types hfaType = GetHfaType(hClass); + + // If we have varargs we morph the method's return type to be "int" irrespective of its original + // type: struct/float at importer because the ABI calls out return in integer registers. + // We don't want struct promotion to replace an expression like this: + // lclFld_int = callvar_int() into lclFld_float = callvar_int(); + // This means an int is getting assigned to a float without a cast. Prevent the promotion. + if ((hfaType == TYP_DOUBLE && hfaSlots == sizeof(double) / REGSIZE_BYTES) || + (hfaType == TYP_FLOAT && hfaSlots == sizeof(float) / REGSIZE_BYTES)) + { + // Make sure this struct type stays as struct so we can receive the call in a struct. + lvaTable[tmpNum].lvIsMultiRegRet = true; + } + } +} +#endif // TARGET_ARM - if (varTypeIsStruct(op1) && - IsMultiRegReturnedType(resolvedToken.hClass, CorInfoCallConvExtension::Managed)) - { - // Unbox nullable helper returns a TYP_STRUCT. - // For the multi-reg case we need to spill it to a temp so that - // we can pass the address to the unbox_nullable jit helper. +#if FEATURE_MULTIREG_RET +//------------------------------------------------------------------------ +// impAssignMultiRegTypeToVar: ensure calls that return structs in multiple +// registers return values to suitable temps. +// +// Arguments: +// op -- call returning a struct in registers +// hClass -- class handle for struct +// +// Returns: +// Tree with reference to struct local to use as call return value. - unsigned tmp = lvaGrabTemp(true DEBUGARG("UNBOXing a register returnable nullable")); - lvaTable[tmp].lvIsMultiRegArg = true; - lvaSetStruct(tmp, resolvedToken.hClass, true /* unsafe value cls check */); +GenTree* Compiler::impAssignMultiRegTypeToVar(GenTree* op, + CORINFO_CLASS_HANDLE hClass DEBUGARG(CorInfoCallConvExtension callConv)) +{ + unsigned tmpNum = lvaGrabTemp(true DEBUGARG("Return value temp for multireg return")); + impAssignTempGen(tmpNum, op, hClass, (unsigned)CHECK_SPILL_ALL); + GenTree* ret = gtNewLclvNode(tmpNum, lvaTable[tmpNum].lvType); - op2 = gtNewLclvNode(tmp, TYP_STRUCT); - op1 = impAssignStruct(op2, op1, CHECK_SPILL_ALL); - assert(op1->gtType == TYP_VOID); // We must be assigning the return struct to the temp. + // TODO-1stClassStructs: Handle constant propagation and CSE-ing of multireg returns. + ret->gtFlags |= GTF_DONT_CSE; - op2 = gtNewLclvNode(tmp, TYP_STRUCT); - op2 = gtNewOperNode(GT_ADDR, TYP_BYREF, op2); - op1 = gtNewOperNode(GT_COMMA, TYP_BYREF, op1, op2); + assert(IsMultiRegReturnedType(hClass, callConv)); - // In this case the return value of the unbox helper is TYP_BYREF. - // Make sure the right type is placed on the operand type stack. - impPushOnStack(op1, tiRetVal); + // Mark the var so that fields are not promoted and stay together. + lvaTable[tmpNum].lvIsMultiRegRet = true; - // Load the struct. - oper = GT_OBJ; + return ret; +} +#endif // FEATURE_MULTIREG_RET - assert(op1->gtType == TYP_BYREF); +//------------------------------------------------------------------------ +// impReturnInstruction: import a return or an explicit tail call +// +// Arguments: +// prefixFlags -- active IL prefixes +// opcode -- [in, out] IL opcode +// +// Returns: +// True if import was successful (may fail for some inlinees) +// +bool Compiler::impReturnInstruction(int prefixFlags, OPCODE& opcode) +{ + const bool isTailCall = (prefixFlags & PREFIX_TAILCALL) != 0; - goto OBJ; - } - else +#ifdef DEBUG + // If we are importing an inlinee and have GC ref locals we always + // need to have a spill temp for the return value. This temp + // should have been set up in advance, over in fgFindBasicBlocks. + if (compIsForInlining() && impInlineInfo->HasGcRefLocals() && (info.compRetType != TYP_VOID)) + { + assert(lvaInlineeReturnSpillTemp != BAD_VAR_NUM); + } +#endif // DEBUG -#endif // !FEATURE_MULTIREG_RET + GenTree* op2 = nullptr; + GenTree* op1 = nullptr; + CORINFO_CLASS_HANDLE retClsHnd = nullptr; - { - // If non register passable struct we have it materialized in the RetBuf. - assert(op1->gtType == TYP_STRUCT); - tiRetVal = verMakeTypeInfo(resolvedToken.hClass); - assert(tiRetVal.IsValueClass()); - } - } + if (info.compRetType != TYP_VOID) + { + StackEntry se = impPopStack(); + retClsHnd = se.seTypeInfo.GetClassHandle(); + op2 = se.val; - impPushOnStack(op1, tiRetVal); - } - break; + if (!compIsForInlining()) + { + impBashVarAddrsToI(op2); + op2 = impImplicitIorI4Cast(op2, info.compRetType); + op2 = impImplicitR4orR8Cast(op2, info.compRetType); + // Note that we allow TYP_I_IMPL<->TYP_BYREF transformation, but only TYP_I_IMPL<-TYP_REF. + assertImp((genActualType(op2->TypeGet()) == genActualType(info.compRetType)) || + ((op2->TypeGet() == TYP_I_IMPL) && TypeIs(info.compRetType, TYP_BYREF)) || + (op2->TypeIs(TYP_BYREF, TYP_REF) && (info.compRetType == TYP_I_IMPL)) || + (varTypeIsFloating(op2->gtType) && varTypeIsFloating(info.compRetType)) || + (varTypeIsStruct(op2) && varTypeIsStruct(info.compRetType))); - case CEE_BOX: +#ifdef DEBUG + if (!isTailCall && opts.compGcChecks && (info.compRetType == TYP_REF)) { - /* Get the Class index */ - assertImp(sz == sizeof(unsigned)); - - _impResolveToken(CORINFO_TOKENKIND_Box); + // DDB 3483 : JIT Stress: early termination of GC ref's life time in exception code path + // VSW 440513: Incorrect gcinfo on the return value under COMPlus_JitGCChecks=1 for methods with + // one-return BB. - JITDUMP(" %08X", resolvedToken.token); + assert(op2->gtType == TYP_REF); - accessAllowedResult = - info.compCompHnd->canAccessClass(&resolvedToken, info.compMethodHnd, &calloutHelper); - impHandleAccessAllowed(accessAllowedResult, &calloutHelper); + // confirm that the argument is a GC pointer (for debugging (GC stress)) + op2 = gtNewHelperCallNode(CORINFO_HELP_CHECK_OBJ, TYP_REF, op2); - // Note BOX can be used on things that are not value classes, in which - // case we get a NOP. However the verifier's view of the type on the - // stack changes (in generic code a 'T' becomes a 'boxed T') - if (!eeIsValueClass(resolvedToken.hClass)) + if (verbose) { - JITDUMP("\n Importing BOX(refClass) as NOP\n"); - verCurrentState.esStack[verCurrentState.esStackDepth - 1].seTypeInfo = tiRetVal; - break; + printf("\ncompGcChecks tree:\n"); + gtDispTree(op2); } + } +#endif + } + else + { + if (verCurrentState.esStackDepth != 0) + { + assert(compIsForInlining()); + JITDUMP("CALLSITE_COMPILATION_ERROR: inlinee's stack is not empty."); + compInlineResult->NoteFatal(InlineObservation::CALLSITE_COMPILATION_ERROR); + return false; + } + +#ifdef DEBUG + if (verbose) + { + printf("\n\n Inlinee Return expression (before normalization) =>\n"); + gtDispTree(op2); + } +#endif - bool isByRefLike = - (info.compCompHnd->getClassAttribs(resolvedToken.hClass) & CORINFO_FLG_BYREF_LIKE) != 0; - if (isByRefLike) + // Make sure the type matches the original call. + + var_types returnType = genActualType(op2->gtType); + var_types originalCallType = impInlineInfo->inlineCandidateInfo->fncRetType; + if ((returnType != originalCallType) && (originalCallType == TYP_STRUCT)) + { + originalCallType = impNormStructType(impInlineInfo->inlineCandidateInfo->methInfo.args.retTypeClass); + } + + if (returnType != originalCallType) + { + // Allow TYP_BYREF to be returned as TYP_I_IMPL and vice versa. + // Allow TYP_REF to be returned as TYP_I_IMPL and NOT vice verse. + if ((TypeIs(returnType, TYP_BYREF, TYP_REF) && (originalCallType == TYP_I_IMPL)) || + ((returnType == TYP_I_IMPL) && TypeIs(originalCallType, TYP_BYREF))) { - // For ByRefLike types we are required to either fold the - // recognized patterns in impBoxPatternMatch or otherwise - // throw InvalidProgramException at runtime. In either case - // we will need to spill side effects of the expression. - impSpillSideEffects(false, CHECK_SPILL_ALL DEBUGARG("Required for box of ByRefLike type")); + JITDUMP("Allowing return type mismatch: have %s, needed %s\n", varTypeName(returnType), + varTypeName(originalCallType)); } - - // Look ahead for box idioms - int matched = impBoxPatternMatch(&resolvedToken, codeAddr + sz, codeEndp, - isByRefLike ? BoxPatterns::IsByRefLike : BoxPatterns::None); - if (matched >= 0) + else { - // Skip the matched IL instructions - sz += matched; - break; + JITDUMP("Return type mismatch: have %s, needed %s\n", varTypeName(returnType), + varTypeName(originalCallType)); + compInlineResult->NoteFatal(InlineObservation::CALLSITE_RETURN_TYPE_MISMATCH); + return false; } + } - if (isByRefLike) + // Below, we are going to set impInlineInfo->retExpr to the tree with the return + // expression. At this point, retExpr could already be set if there are multiple + // return blocks (meaning fgNeedReturnSpillTemp() == true) and one of + // the other blocks already set it. If there is only a single return block, + // retExpr shouldn't be set. However, this is not true if we reimport a block + // with a return. In that case, retExpr will be set, then the block will be + // reimported, but retExpr won't get cleared as part of setting the block to + // be reimported. The reimported retExpr value should be the same, so even if + // we don't unconditionally overwrite it, it shouldn't matter. + if (info.compRetNativeType != TYP_STRUCT) + { + // compRetNativeType is not TYP_STRUCT. + // This implies it could be either a scalar type or SIMD vector type or + // a struct type that can be normalized to a scalar type. + + if (varTypeIsStruct(info.compRetType)) { - // ByRefLike types are supported in boxing scenarios when the instruction can be elided - // due to a recognized pattern above. If the pattern is not recognized, the code is invalid. - BADCODE("ByRefLike types cannot be boxed"); + noway_assert(info.compRetBuffArg == BAD_VAR_NUM); + // adjust the type away from struct to integral + // and no normalizing + op2 = impFixupStructReturnType(op2, retClsHnd, info.compCallConv); } else { - impImportAndPushBox(&resolvedToken); - if (compDonotInline()) + // Do we have to normalize? + var_types fncRealRetType = JITtype2varType(info.compMethodInfo->args.retType); + // For RET_EXPR get the type info from the call. Regardless + // of whether it ends up inlined or not normalization will + // happen as part of that function's codegen. + GenTree* returnedTree = op2->OperIs(GT_RET_EXPR) ? op2->AsRetExpr()->gtInlineCandidate : op2; + if ((varTypeIsSmall(returnedTree->TypeGet()) || varTypeIsSmall(fncRealRetType)) && + fgCastNeeded(returnedTree, fncRealRetType)) { - return; + // Small-typed return values are normalized by the callee + op2 = gtNewCastNode(TYP_INT, op2, false, fncRealRetType); } } - } - break; - - case CEE_SIZEOF: - /* Get the Class index */ - assertImp(sz == sizeof(unsigned)); + if (fgNeedReturnSpillTemp()) + { + assert(info.compRetNativeType != TYP_VOID && + (fgMoreThanOneReturnBlock() || impInlineInfo->HasGcRefLocals())); - _impResolveToken(CORINFO_TOKENKIND_Class); + // If this method returns a ref type, track the actual types seen + // in the returns. + if (info.compRetType == TYP_REF) + { + bool isExact = false; + bool isNonNull = false; + CORINFO_CLASS_HANDLE returnClsHnd = gtGetClassHandle(op2, &isExact, &isNonNull); - JITDUMP(" %08X", resolvedToken.token); + if (impInlineInfo->retExpr == nullptr) + { + // This is the first return, so best known type is the type + // of this return value. + impInlineInfo->retExprClassHnd = returnClsHnd; + impInlineInfo->retExprClassHndIsExact = isExact; + } + else if (impInlineInfo->retExprClassHnd != returnClsHnd) + { + // This return site type differs from earlier seen sites, + // so reset the info and we'll fall back to using the method's + // declared return type for the return spill temp. + impInlineInfo->retExprClassHnd = nullptr; + impInlineInfo->retExprClassHndIsExact = false; + } + } - op1 = gtNewIconNode(info.compCompHnd->getClassSize(resolvedToken.hClass)); - impPushOnStack(op1, tiRetVal); - break; + impAssignTempGen(lvaInlineeReturnSpillTemp, op2, se.seTypeInfo.GetClassHandle(), + (unsigned)CHECK_SPILL_ALL); - case CEE_CASTCLASS: + var_types lclRetType = lvaGetDesc(lvaInlineeReturnSpillTemp)->lvType; + GenTree* tmpOp2 = gtNewLclvNode(lvaInlineeReturnSpillTemp, lclRetType); - /* Get the Class index */ + op2 = tmpOp2; +#ifdef DEBUG + if (impInlineInfo->retExpr) + { + // Some other block(s) have seen the CEE_RET first. + // Better they spilled to the same temp. + assert(impInlineInfo->retExpr->gtOper == GT_LCL_VAR); + assert(impInlineInfo->retExpr->AsLclVarCommon()->GetLclNum() == + op2->AsLclVarCommon()->GetLclNum()); + } +#endif + } - assertImp(sz == sizeof(unsigned)); +#ifdef DEBUG + if (verbose) + { + printf("\n\n Inlinee Return expression (after normalization) =>\n"); + gtDispTree(op2); + } +#endif - _impResolveToken(CORINFO_TOKENKIND_Casting); + // Report the return expression + impInlineInfo->retExpr = op2; + } + else + { + // compRetNativeType is TYP_STRUCT. + // This implies that struct return via RetBuf arg or multi-reg struct return - JITDUMP(" %08X", resolvedToken.token); + GenTreeCall* iciCall = impInlineInfo->iciCall->AsCall(); - if (!opts.IsReadyToRun()) + // Assign the inlinee return into a spill temp. + // spill temp only exists if there are multiple return points + if (lvaInlineeReturnSpillTemp != BAD_VAR_NUM) { - op2 = impTokenToHandle(&resolvedToken, nullptr, false); - if (op2 == nullptr) - { // compDonotInline() - return; - } - } - - accessAllowedResult = - info.compCompHnd->canAccessClass(&resolvedToken, info.compMethodHnd, &calloutHelper); - impHandleAccessAllowed(accessAllowedResult, &calloutHelper); + // in this case we have to insert multiple struct copies to the temp + // and the retexpr is just the temp. + assert(info.compRetNativeType != TYP_VOID); + assert(fgMoreThanOneReturnBlock() || impInlineInfo->HasGcRefLocals()); - op1 = impPopStack().val; + impAssignTempGen(lvaInlineeReturnSpillTemp, op2, se.seTypeInfo.GetClassHandle(), + (unsigned)CHECK_SPILL_ALL); + } - /* Pop the address and create the 'checked cast' helper call */ +#if defined(TARGET_ARM) || defined(UNIX_AMD64_ABI) +#if defined(TARGET_ARM) + // TODO-ARM64-NYI: HFA + // TODO-AMD64-Unix and TODO-ARM once the ARM64 functionality is implemented the + // next ifdefs could be refactored in a single method with the ifdef inside. + if (IsHfa(retClsHnd)) + { +// Same as !IsHfa but just don't bother with impAssignStructPtr. +#else // defined(UNIX_AMD64_ABI) + ReturnTypeDesc retTypeDesc; + retTypeDesc.InitializeStructReturnType(this, retClsHnd, info.compCallConv); + unsigned retRegCount = retTypeDesc.GetReturnRegCount(); + + if (retRegCount != 0) + { + // If single eightbyte, the return type would have been normalized and there won't be a temp var. + // This code will be called only if the struct return has not been normalized (i.e. 2 eightbytes - + // max allowed.) + assert(retRegCount == MAX_RET_REG_COUNT); + // Same as !structDesc.passedInRegisters but just don't bother with impAssignStructPtr. + CLANG_FORMAT_COMMENT_ANCHOR; +#endif // defined(UNIX_AMD64_ABI) - // At this point we expect typeRef to contain the token, op1 to contain the value being cast, - // and op2 to contain code that creates the type handle corresponding to typeRef - CASTCLASS: - { - GenTree* optTree = impOptimizeCastClassOrIsInst(op1, &resolvedToken, true); + if (fgNeedReturnSpillTemp()) + { + if (!impInlineInfo->retExpr) + { +#if defined(TARGET_ARM) + impInlineInfo->retExpr = gtNewLclvNode(lvaInlineeReturnSpillTemp, info.compRetType); +#else // defined(UNIX_AMD64_ABI) + // The inlinee compiler has figured out the type of the temp already. Use it here. + impInlineInfo->retExpr = + gtNewLclvNode(lvaInlineeReturnSpillTemp, lvaTable[lvaInlineeReturnSpillTemp].lvType); +#endif // defined(UNIX_AMD64_ABI) + } + } + else + { + impInlineInfo->retExpr = op2; + } + } + else +#elif defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) + ReturnTypeDesc retTypeDesc; + retTypeDesc.InitializeStructReturnType(this, retClsHnd, info.compCallConv); + unsigned retRegCount = retTypeDesc.GetReturnRegCount(); - if (optTree != nullptr) + if (retRegCount != 0) { - impPushOnStack(optTree, tiRetVal); + assert(!iciCall->ShouldHaveRetBufArg()); + assert(retRegCount >= 2); + if (fgNeedReturnSpillTemp()) + { + if (!impInlineInfo->retExpr) + { + // The inlinee compiler has figured out the type of the temp already. Use it here. + impInlineInfo->retExpr = + gtNewLclvNode(lvaInlineeReturnSpillTemp, lvaTable[lvaInlineeReturnSpillTemp].lvType); + } + } + else + { + impInlineInfo->retExpr = op2; + } } else - { +#elif defined(TARGET_X86) + ReturnTypeDesc retTypeDesc; + retTypeDesc.InitializeStructReturnType(this, retClsHnd, info.compCallConv); + unsigned retRegCount = retTypeDesc.GetReturnRegCount(); -#ifdef FEATURE_READYTORUN - if (opts.IsReadyToRun()) + if (retRegCount != 0) + { + assert(!iciCall->ShouldHaveRetBufArg()); + assert(retRegCount == MAX_RET_REG_COUNT); + if (fgNeedReturnSpillTemp()) { - GenTreeCall* opLookup = - impReadyToRunHelperToTree(&resolvedToken, CORINFO_HELP_READYTORUN_CHKCAST, TYP_REF, nullptr, - op1); - usingReadyToRunHelper = (opLookup != nullptr); - op1 = (usingReadyToRunHelper ? opLookup : op1); - - if (!usingReadyToRunHelper) + if (!impInlineInfo->retExpr) { - // TODO: ReadyToRun: When generic dictionary lookups are necessary, replace the lookup call - // and the chkcastany call with a single call to a dynamic R2R cell that will: - // 1) Load the context - // 2) Perform the generic dictionary lookup and caching, and generate the appropriate - // stub - // 3) Check the object on the stack for the type-cast - // Reason: performance (today, we'll always use the slow helper for the R2R generics case) - - op2 = impTokenToHandle(&resolvedToken, nullptr, false); - if (op2 == nullptr) - { // compDonotInline() - return; - } + // The inlinee compiler has figured out the type of the temp already. Use it here. + impInlineInfo->retExpr = + gtNewLclvNode(lvaInlineeReturnSpillTemp, lvaTable[lvaInlineeReturnSpillTemp].lvType); } } - - if (!usingReadyToRunHelper) -#endif + else { - op1 = impCastClassOrIsInstToTree(op1, op2, &resolvedToken, true, opcodeOffs); + impInlineInfo->retExpr = op2; } - if (compDonotInline()) + } + else +#endif // defined(TARGET_ARM64) + { + assert(iciCall->gtArgs.HasRetBuffer()); + GenTree* dest = gtCloneExpr(iciCall->gtArgs.GetRetBufferArg()->GetEarlyNode()); + // spill temp only exists if there are multiple return points + if (fgNeedReturnSpillTemp()) { - return; + // if this is the first return we have seen set the retExpr + if (!impInlineInfo->retExpr) + { + impInlineInfo->retExpr = + impAssignStructPtr(dest, gtNewLclvNode(lvaInlineeReturnSpillTemp, info.compRetType), + retClsHnd, (unsigned)CHECK_SPILL_ALL); + } + } + else + { + impInlineInfo->retExpr = impAssignStructPtr(dest, op2, retClsHnd, (unsigned)CHECK_SPILL_ALL); } - - /* Push the result back on the stack */ - impPushOnStack(op1, tiRetVal); } } - break; - case CEE_THROW: + if (impInlineInfo->retExpr != nullptr) + { + impInlineInfo->retBB = compCurBB; + } + } + } - // Any block with a throw is rarely executed. - block->bbSetRunRarely(); + if (compIsForInlining()) + { + return true; + } + + if (info.compRetType == TYP_VOID) + { + // return void + op1 = new (this, GT_RETURN) GenTreeOp(GT_RETURN, TYP_VOID); + } + else if (info.compRetBuffArg != BAD_VAR_NUM) + { + // Assign value to return buff (first param) + GenTree* retBuffAddr = + gtNewLclvNode(info.compRetBuffArg, TYP_BYREF DEBUGARG(impCurStmtDI.GetLocation().GetOffset())); + + op2 = impAssignStructPtr(retBuffAddr, op2, retClsHnd, (unsigned)CHECK_SPILL_ALL); + impAppendTree(op2, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); + + // There are cases where the address of the implicit RetBuf should be returned explicitly (in RAX). + CLANG_FORMAT_COMMENT_ANCHOR; - // Pop the exception object and create the 'throw' helper call - op1 = gtNewHelperCallNode(CORINFO_HELP_THROW, TYP_VOID, impPopStack().val); +#if defined(TARGET_AMD64) - // Fall through to clear out the eval stack. + // x64 (System V and Win64) calling convention requires to + // return the implicit return buffer explicitly (in RAX). + // Change the return type to be BYREF. + op1 = gtNewOperNode(GT_RETURN, TYP_BYREF, gtNewLclvNode(info.compRetBuffArg, TYP_BYREF)); +#else // !defined(TARGET_AMD64) + // In case of non-AMD64 targets the profiler hook requires to return the implicit RetBuf explicitly (in RAX). + // In such case the return value of the function is changed to BYREF. + // If profiler hook is not needed the return type of the function is TYP_VOID. + if (compIsProfilerHookNeeded()) + { + op1 = gtNewOperNode(GT_RETURN, TYP_BYREF, gtNewLclvNode(info.compRetBuffArg, TYP_BYREF)); + } +#if defined(TARGET_ARM64) + // On ARM64, the native instance calling convention variant + // requires the implicit ByRef to be explicitly returned. + else if (TargetOS::IsWindows && callConvIsInstanceMethodCallConv(info.compCallConv)) + { + op1 = gtNewOperNode(GT_RETURN, TYP_BYREF, gtNewLclvNode(info.compRetBuffArg, TYP_BYREF)); + } +#endif +#if defined(TARGET_X86) + else if (info.compCallConv != CorInfoCallConvExtension::Managed) + { + op1 = gtNewOperNode(GT_RETURN, TYP_BYREF, gtNewLclvNode(info.compRetBuffArg, TYP_BYREF)); + } +#endif + else + { + // return void + op1 = new (this, GT_RETURN) GenTreeOp(GT_RETURN, TYP_VOID); + } +#endif // !defined(TARGET_AMD64) + } + else if (varTypeIsStruct(info.compRetType)) + { +#if !FEATURE_MULTIREG_RET + // For both ARM architectures the HFA native types are maintained as structs. + // Also on System V AMD64 the multireg structs returns are also left as structs. + noway_assert(info.compRetNativeType != TYP_STRUCT); +#endif + op2 = impFixupStructReturnType(op2, retClsHnd, info.compCallConv); + // return op2 + var_types returnType = info.compRetType; + op1 = gtNewOperNode(GT_RETURN, genActualType(returnType), op2); + } + else + { + // return op2 + op1 = gtNewOperNode(GT_RETURN, genActualType(info.compRetType), op2); + } - EVAL_APPEND: - if (verCurrentState.esStackDepth > 0) - { - impEvalSideEffects(); - } + // We must have imported a tailcall and jumped to RET + if (isTailCall) + { + assert(verCurrentState.esStackDepth == 0 && impOpcodeIsCallOpcode(opcode)); - assert(verCurrentState.esStackDepth == 0); + opcode = CEE_RET; // To prevent trying to spill if CALL_SITE_BOUNDARIES - goto APPEND; + // impImportCall() would have already appended TYP_VOID calls + if (info.compRetType == TYP_VOID) + { + return true; + } + } - case CEE_RETHROW: + impAppendTree(op1, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); +#ifdef DEBUG + // Remember at which BC offset the tree was finished + impNoteLastILoffs(); +#endif + return true; +} - assert(!compIsForInlining()); +/***************************************************************************** + * Mark the block as unimported. + * Note that the caller is responsible for calling impImportBlockPending(), + * with the appropriate stack-state + */ - if (info.compXcptnsCount == 0) - { - BADCODE("rethrow outside catch"); - } +inline void Compiler::impReimportMarkBlock(BasicBlock* block) +{ +#ifdef DEBUG + if (verbose && (block->bbFlags & BBF_IMPORTED)) + { + printf("\n" FMT_BB " will be reimported\n", block->bbNum); + } +#endif - /* Create the 'rethrow' helper call */ + block->bbFlags &= ~BBF_IMPORTED; +} - op1 = gtNewHelperCallNode(CORINFO_HELP_RETHROW, TYP_VOID); +/***************************************************************************** + * Mark the successors of the given block as unimported. + * Note that the caller is responsible for calling impImportBlockPending() + * for all the successors, with the appropriate stack-state. + */ - goto EVAL_APPEND; +void Compiler::impReimportMarkSuccessors(BasicBlock* block) +{ + for (BasicBlock* const succBlock : block->Succs()) + { + impReimportMarkBlock(succBlock); + } +} - case CEE_INITOBJ: +/***************************************************************************** + * + * Filter wrapper to handle only passed in exception code + * from it). + */ - assertImp(sz == sizeof(unsigned)); +LONG FilterVerificationExceptions(PEXCEPTION_POINTERS pExceptionPointers, LPVOID lpvParam) +{ + if (pExceptionPointers->ExceptionRecord->ExceptionCode == SEH_VERIFICATION_EXCEPTION) + { + return EXCEPTION_EXECUTE_HANDLER; + } - _impResolveToken(CORINFO_TOKENKIND_Class); + return EXCEPTION_CONTINUE_SEARCH; +} - JITDUMP(" %08X", resolvedToken.token); +void Compiler::impVerifyEHBlock(BasicBlock* block, bool isTryStart) +{ + assert(block->hasTryIndex()); + assert(!compIsForInlining()); - lclTyp = JITtype2varType(info.compCompHnd->asCorInfoType(resolvedToken.hClass)); - if (lclTyp != TYP_STRUCT) - { - op2 = gtNewZeroConNode(genActualType(lclTyp)); - goto STIND_VALUE; - } + unsigned tryIndex = block->getTryIndex(); + EHblkDsc* HBtab = ehGetDsc(tryIndex); - op1 = impPopStack().val; - op1 = gtNewStructVal(typGetObjLayout(resolvedToken.hClass), op1); - op2 = gtNewIconNode(0); - op1 = gtNewBlkOpNode(op1, op2, (prefixFlags & PREFIX_VOLATILE) != 0); - goto SPILL_APPEND; + if (isTryStart) + { + assert(block->bbFlags & BBF_TRY_BEG); - case CEE_INITBLK: + // The Stack must be empty + // + if (block->bbStkDepth != 0) + { + BADCODE("Evaluation stack must be empty on entry into a try block"); + } + } - op3 = impPopStack().val; // Size - op2 = impPopStack().val; // Value - op1 = impPopStack().val; // Dst addr + // Save the stack contents, we'll need to restore it later + // + SavedStack blockState; + impSaveStackState(&blockState, false); - if (op3->IsCnsIntOrI()) + while (HBtab != nullptr) + { + if (isTryStart) + { + // Are we verifying that an instance constructor properly initializes it's 'this' pointer once? + // We do not allow the 'this' pointer to be uninitialized when entering most kinds try regions + // + if (verTrackObjCtorInitState && (verCurrentState.thisInitialized != TIS_Init)) + { + // We trigger an invalid program exception here unless we have a try/fault region. + // + if (HBtab->HasCatchHandler() || HBtab->HasFinallyHandler() || HBtab->HasFilter()) { - size = (unsigned)op3->AsIntConCommon()->IconValue(); - op1 = new (this, GT_BLK) GenTreeBlk(GT_BLK, TYP_STRUCT, op1, typGetBlkLayout(size)); - op1 = gtNewBlkOpNode(op1, op2, (prefixFlags & PREFIX_VOLATILE) != 0); + BADCODE( + "The 'this' pointer of an instance constructor is not initialized upon entry to a try region"); } else { - if (!op2->IsIntegralConst(0)) - { - op2 = gtNewOperNode(GT_INIT_VAL, TYP_INT, op2); - } - -#ifdef TARGET_64BIT - // STORE_DYN_BLK takes a native uint size as it turns into call to memset. - op3 = gtNewCastNode(TYP_I_IMPL, op3, /* fromUnsigned */ true, TYP_U_IMPL); -#endif - - op1 = new (this, GT_STORE_DYN_BLK) GenTreeStoreDynBlk(op1, op2, op3); - size = 0; - - if ((prefixFlags & PREFIX_VOLATILE) != 0) - { - op1->gtFlags |= GTF_BLK_VOLATILE; - } + // Allow a try/fault region to proceed. + assert(HBtab->HasFaultHandler()); } - goto SPILL_APPEND; + } + } - case CEE_CPBLK: + // Recursively process the handler block, if we haven't already done so. + BasicBlock* hndBegBB = HBtab->ebdHndBeg; - op3 = impPopStack().val; // Size - op2 = impPopStack().val; // Src addr - op1 = impPopStack().val; // Dst addr + if (((hndBegBB->bbFlags & BBF_IMPORTED) == 0) && (impGetPendingBlockMember(hndBegBB) == 0)) + { + // Construct the proper verification stack state + // either empty or one that contains just + // the Exception Object that we are dealing with + // + verCurrentState.esStackDepth = 0; - if (op3->IsCnsIntOrI()) - { - size = static_cast(op3->AsIntConCommon()->IconValue()); + if (handlerGetsXcptnObj(hndBegBB->bbCatchTyp)) + { + CORINFO_CLASS_HANDLE clsHnd; - op1 = gtNewBlockVal(op1, size); - op2 = gtNewBlockVal(op2, size); - op1 = gtNewBlkOpNode(op1, op2, (prefixFlags & PREFIX_VOLATILE) != 0); + if (HBtab->HasFilter()) + { + clsHnd = impGetObjectClass(); } else { - op2 = gtNewOperNode(GT_IND, TYP_STRUCT, op2); - -#ifdef TARGET_64BIT - // STORE_DYN_BLK takes a native uint size as it turns into call to memcpy. - op3 = gtNewCastNode(TYP_I_IMPL, op3, /* fromUnsigned */ true, TYP_U_IMPL); -#endif + CORINFO_RESOLVED_TOKEN resolvedToken; - op1 = new (this, GT_STORE_DYN_BLK) GenTreeStoreDynBlk(op1, op2, op3); + resolvedToken.tokenContext = impTokenLookupContextHandle; + resolvedToken.tokenScope = info.compScopeHnd; + resolvedToken.token = HBtab->ebdTyp; + resolvedToken.tokenType = CORINFO_TOKENKIND_Class; + info.compCompHnd->resolveToken(&resolvedToken); - if ((prefixFlags & PREFIX_VOLATILE) != 0) - { - op1->gtFlags |= GTF_BLK_VOLATILE; - } + clsHnd = resolvedToken.hClass; } - goto SPILL_APPEND; - case CEE_CPOBJ: - { - assertImp(sz == sizeof(unsigned)); + // push catch arg the stack, spill to a temp if necessary + // Note: can update HBtab->ebdHndBeg! + hndBegBB = impPushCatchArgOnStack(hndBegBB, clsHnd, false); + } - _impResolveToken(CORINFO_TOKENKIND_Class); + // Queue up the handler for importing + // + impImportBlockPending(hndBegBB); + } - JITDUMP(" %08X", resolvedToken.token); + // Process the filter block, if we haven't already done so. + if (HBtab->HasFilter()) + { + /* @VERIFICATION : Ideally the end of filter state should get + propagated to the catch handler, this is an incompleteness, + but is not a security/compliance issue, since the only + interesting state is the 'thisInit' state. + */ - lclTyp = JITtype2varType(info.compCompHnd->asCorInfoType(resolvedToken.hClass)); + BasicBlock* filterBB = HBtab->ebdFilter; - if (lclTyp != TYP_STRUCT) - { - op1 = impPopStack().val; // address to load from + if (((filterBB->bbFlags & BBF_IMPORTED) == 0) && (impGetPendingBlockMember(filterBB) == 0)) + { + verCurrentState.esStackDepth = 0; - op1 = gtNewIndir(lclTyp, op1); - op1->gtFlags |= GTF_GLOB_REF; + // push catch arg the stack, spill to a temp if necessary + // Note: can update HBtab->ebdFilter! + const bool isSingleBlockFilter = (filterBB->bbNext == hndBegBB); + filterBB = impPushCatchArgOnStack(filterBB, impGetObjectClass(), isSingleBlockFilter); - impPushOnStack(op1, typeInfo()); - goto STIND; - } + impImportBlockPending(filterBB); + } + } - op2 = impPopStack().val; // Src addr - op1 = impPopStack().val; // Dest addr + // This seems redundant ....?? + if (verTrackObjCtorInitState && HBtab->HasFaultHandler()) + { + /* Recursively process the handler block */ - ClassLayout* layout = typGetObjLayout(resolvedToken.hClass); - op1 = gtNewStructVal(layout, op1); - op2 = gtNewStructVal(layout, op2); - if (op1->OperIs(GT_OBJ)) - { - gtSetObjGcInfo(op1->AsObj()); - } - op1 = gtNewBlkOpNode(op1, op2, ((prefixFlags & PREFIX_VOLATILE) != 0)); - goto SPILL_APPEND; - } + verCurrentState.esStackDepth = 0; - case CEE_STOBJ: - { - assertImp(sz == sizeof(unsigned)); + // Queue up the fault handler for importing + // + impImportBlockPending(HBtab->ebdHndBeg); + } + + // Now process our enclosing try index (if any) + // + tryIndex = HBtab->ebdEnclosingTryIndex; + if (tryIndex == EHblkDsc::NO_ENCLOSING_INDEX) + { + HBtab = nullptr; + } + else + { + HBtab = ehGetDsc(tryIndex); + } + } - _impResolveToken(CORINFO_TOKENKIND_Class); + // Restore the stack contents + impRestoreStackState(&blockState); +} - JITDUMP(" %08X", resolvedToken.token); +//*************************************************************** +// Import the instructions for the given basic block. Perform +// verification, throwing an exception on failure. Push any successor blocks that are enabled for the first +// time, or whose verification pre-state is changed. - lclTyp = JITtype2varType(info.compCompHnd->asCorInfoType(resolvedToken.hClass)); +#ifdef _PREFAST_ +#pragma warning(push) +#pragma warning(disable : 21000) // Suppress PREFast warning about overly large function +#endif +void Compiler::impImportBlock(BasicBlock* block) +{ + // BBF_INTERNAL blocks only exist during importation due to EH canonicalization. We need to + // handle them specially. In particular, there is no IL to import for them, but we do need + // to mark them as imported and put their successors on the pending import list. + if (block->bbFlags & BBF_INTERNAL) + { + JITDUMP("Marking BBF_INTERNAL block " FMT_BB " as BBF_IMPORTED\n", block->bbNum); + block->bbFlags |= BBF_IMPORTED; - if (lclTyp != TYP_STRUCT) - { - goto STIND; - } + for (BasicBlock* const succBlock : block->Succs()) + { + impImportBlockPending(succBlock); + } - op2 = impPopStack().val; // Value - op1 = impPopStack().val; // Ptr + return; + } - assertImp(varTypeIsStruct(op2)); + bool markImport; - op1 = impAssignStructPtr(op1, op2, resolvedToken.hClass, CHECK_SPILL_ALL); + assert(block); - if (op1->OperIsBlkOp() && (prefixFlags & PREFIX_UNALIGNED)) - { - op1->gtFlags |= GTF_BLK_UNALIGNED; - } - goto SPILL_APPEND; - } + /* Make the block globaly available */ - case CEE_MKREFANY: + compCurBB = block; - assert(!compIsForInlining()); +#ifdef DEBUG + /* Initialize the debug variables */ + impCurOpcName = "unknown"; + impCurOpcOffs = block->bbCodeOffs; +#endif - // Being lazy here. Refanys are tricky in terms of gc tracking. - // Since it is uncommon, just don't perform struct promotion in any method that contains mkrefany. + /* Set the current stack state to the merged result */ + verResetCurrentState(block, &verCurrentState); - JITDUMP("disabling struct promotion because of mkrefany\n"); - fgNoStructPromotion = true; + /* Now walk the code and import the IL into GenTrees */ - oper = GT_MKREFANY; - assertImp(sz == sizeof(unsigned)); + struct FilterVerificationExceptionsParam + { + Compiler* pThis; + BasicBlock* block; + }; + FilterVerificationExceptionsParam param; - _impResolveToken(CORINFO_TOKENKIND_Class); + param.pThis = this; + param.block = block; - JITDUMP(" %08X", resolvedToken.token); + PAL_TRY(FilterVerificationExceptionsParam*, pParam, ¶m) + { + /* @VERIFICATION : For now, the only state propagation from try + to it's handler is "thisInit" state (stack is empty at start of try). + In general, for state that we track in verification, we need to + model the possibility that an exception might happen at any IL + instruction, so we really need to merge all states that obtain + between IL instructions in a try block into the start states of + all handlers. - op2 = impTokenToHandle(&resolvedToken, nullptr, true); - if (op2 == nullptr) - { // compDonotInline() - return; - } + However we do not allow the 'this' pointer to be uninitialized when + entering most kinds try regions (only try/fault are allowed to have + an uninitialized this pointer on entry to the try) - accessAllowedResult = - info.compCompHnd->canAccessClass(&resolvedToken, info.compMethodHnd, &calloutHelper); - impHandleAccessAllowed(accessAllowedResult, &calloutHelper); + Fortunately, the stack is thrown away when an exception + leads to a handler, so we don't have to worry about that. + We DO, however, have to worry about the "thisInit" state. + But only for the try/fault case. - op1 = impPopStack().val; + The only allowed transition is from TIS_Uninit to TIS_Init. - // @SPECVIOLATION: TYP_INT should not be allowed here by a strict reading of the spec. - // But JIT32 allowed it, so we continue to allow it. - assertImp(op1->TypeGet() == TYP_BYREF || op1->TypeGet() == TYP_I_IMPL || op1->TypeGet() == TYP_INT); + So for a try/fault region for the fault handler block + we will merge the start state of the try begin + and the post-state of each block that is part of this try region + */ - // MKREFANY returns a struct. op2 is the class token. - op1 = gtNewOperNode(oper, TYP_STRUCT, op1, op2); + // merge the start state of the try begin + // + if (pParam->block->bbFlags & BBF_TRY_BEG) + { + pParam->pThis->impVerifyEHBlock(pParam->block, true); + } - impPushOnStack(op1, verMakeTypeInfo(impGetRefAnyClass())); - break; + pParam->pThis->impImportBlockCode(pParam->block); - case CEE_LDOBJ: - { - oper = GT_OBJ; - assertImp(sz == sizeof(unsigned)); + // As discussed above: + // merge the post-state of each block that is part of this try region + // + if (pParam->block->hasTryIndex()) + { + pParam->pThis->impVerifyEHBlock(pParam->block, false); + } + } + PAL_EXCEPT_FILTER(FilterVerificationExceptions) + { + verHandleVerificationFailure(block DEBUGARG(false)); + } + PAL_ENDTRY - _impResolveToken(CORINFO_TOKENKIND_Class); + if (compDonotInline()) + { + return; + } - JITDUMP(" %08X", resolvedToken.token); + assert(!compDonotInline()); - OBJ: - lclTyp = JITtype2varType(info.compCompHnd->asCorInfoType(resolvedToken.hClass)); - tiRetVal = verMakeTypeInfo(resolvedToken.hClass); + markImport = false; - if (lclTyp != TYP_STRUCT) - { - goto LDIND; - } +SPILLSTACK: - op1 = impPopStack().val; + unsigned baseTmp = NO_BASE_TMP; // input temps assigned to successor blocks + bool reimportSpillClique = false; + BasicBlock* tgtBlock = nullptr; - assertImp((genActualType(op1) == TYP_I_IMPL) || op1->TypeIs(TYP_BYREF)); + /* If the stack is non-empty, we might have to spill its contents */ - op1 = gtNewObjNode(resolvedToken.hClass, op1); - op1->gtFlags |= GTF_EXCEPT; + if (verCurrentState.esStackDepth != 0) + { + impBoxTemp = BAD_VAR_NUM; // if a box temp is used in a block that leaves something + // on the stack, its lifetime is hard to determine, simply + // don't reuse such temps. - if (prefixFlags & PREFIX_UNALIGNED) - { - op1->gtFlags |= GTF_IND_UNALIGNED; - } + Statement* addStmt = nullptr; - impPushOnStack(op1, tiRetVal); - break; - } + /* Do the successors of 'block' have any other predecessors ? + We do not want to do some of the optimizations related to multiRef + if we can reimport blocks */ - case CEE_LDLEN: - op1 = impPopStack().val; - if (opts.OptimizationEnabled()) - { - /* Use GT_ARR_LENGTH operator so rng check opts see this */ - GenTreeArrLen* arrLen = gtNewArrLen(TYP_INT, op1, OFFSETOF__CORINFO_Array__length, block); + unsigned multRef = impCanReimport ? unsigned(~0) : 0; - op1 = arrLen; - } - else - { - /* Create the expression "*(array_addr + ArrLenOffs)" */ - op1 = gtNewOperNode(GT_ADD, TYP_BYREF, op1, - gtNewIconNode(OFFSETOF__CORINFO_Array__length, TYP_I_IMPL)); - op1 = gtNewIndir(TYP_INT, op1); - } + switch (block->bbJumpKind) + { + case BBJ_COND: - /* Push the result back on the stack */ - impPushOnStack(op1, tiRetVal); - break; + addStmt = impExtractLastStmt(); - case CEE_BREAK: - op1 = gtNewHelperCallNode(CORINFO_HELP_USER_BREAKPOINT, TYP_VOID); - goto SPILL_APPEND; + assert(addStmt->GetRootNode()->gtOper == GT_JTRUE); - case CEE_NOP: - if (opts.compDbgCode) - { - op1 = new (this, GT_NO_OP) GenTree(GT_NO_OP, TYP_VOID); - goto SPILL_APPEND; - } - break; + /* Note if the next block has more than one ancestor */ - /******************************** NYI *******************************/ + multRef |= block->bbNext->bbRefs; - case 0xCC: - OutputDebugStringA("CLR: Invalid x86 breakpoint in IL stream\n"); - FALLTHROUGH; + /* Does the next block have temps assigned? */ - case CEE_ILLEGAL: - case CEE_MACRO_END: + baseTmp = block->bbNext->bbStkTempsIn; + tgtBlock = block->bbNext; - default: - if (compIsForInlining()) + if (baseTmp != NO_BASE_TMP) { - compInlineResult->NoteFatal(InlineObservation::CALLEE_COMPILATION_ERROR); - return; + break; } - BADCODE3("unknown opcode", ": %02X", (int)opcode); - } + /* Try the target of the jump then */ - codeAddr += sz; - prevOpcode = opcode; + multRef |= block->bbJumpDest->bbRefs; + baseTmp = block->bbJumpDest->bbStkTempsIn; + tgtBlock = block->bbJumpDest; + break; - prefixFlags = 0; - } + case BBJ_ALWAYS: + multRef |= block->bbJumpDest->bbRefs; + baseTmp = block->bbJumpDest->bbStkTempsIn; + tgtBlock = block->bbJumpDest; + break; - return; -#undef _impResolveToken -} -#ifdef _PREFAST_ -#pragma warning(pop) -#endif + case BBJ_NONE: + multRef |= block->bbNext->bbRefs; + baseTmp = block->bbNext->bbStkTempsIn; + tgtBlock = block->bbNext; + break; + + case BBJ_SWITCH: + addStmt = impExtractLastStmt(); + assert(addStmt->GetRootNode()->gtOper == GT_SWITCH); -// Push a local/argument treeon the operand stack -void Compiler::impPushVar(GenTree* op, typeInfo tiRetVal) -{ - tiRetVal.NormaliseForStack(); - impPushOnStack(op, tiRetVal); -} + for (BasicBlock* const tgtBlock : block->SwitchTargets()) + { + multRef |= tgtBlock->bbRefs; -//------------------------------------------------------------------------ -// impCreateLocal: create a GT_LCL_VAR node to access a local that might need to be normalized on load -// -// Arguments: -// lclNum -- The index into lvaTable -// offset -- The offset to associate with the node -// -// Returns: -// The node -// -GenTreeLclVar* Compiler::impCreateLocalNode(unsigned lclNum DEBUGARG(IL_OFFSET offset)) -{ - var_types lclTyp; + // Thanks to spill cliques, we should have assigned all or none + assert((baseTmp == NO_BASE_TMP) || (baseTmp == tgtBlock->bbStkTempsIn)); + baseTmp = tgtBlock->bbStkTempsIn; + if (multRef > 1) + { + break; + } + } + break; - if (lvaTable[lclNum].lvNormalizeOnLoad()) - { - lclTyp = lvaGetRealType(lclNum); - } - else - { - lclTyp = lvaGetActualType(lclNum); - } + case BBJ_CALLFINALLY: + case BBJ_EHCATCHRET: + case BBJ_RETURN: + case BBJ_EHFINALLYRET: + case BBJ_EHFILTERRET: + case BBJ_THROW: + BADCODE("can't have 'unreached' end of BB with non-empty stack"); + break; - return gtNewLclvNode(lclNum, lclTyp DEBUGARG(offset)); -} + default: + noway_assert(!"Unexpected bbJumpKind"); + break; + } -// Load a local/argument on the operand stack -// lclNum is an index into lvaTable *NOT* the arg/lcl index in the IL -void Compiler::impLoadVar(unsigned lclNum, IL_OFFSET offset) -{ - impPushVar(impCreateLocalNode(lclNum DEBUGARG(offset)), verMakeTypeInfoForLocal(lclNum)); -} + assert(multRef >= 1); -// Load an argument on the operand stack -// Shared by the various CEE_LDARG opcodes -// ilArgNum is the argument index as specified in IL. -// It will be mapped to the correct lvaTable index -void Compiler::impLoadArg(unsigned ilArgNum, IL_OFFSET offset) -{ - Verify(ilArgNum < info.compILargsCount, "bad arg num"); + /* Do we have a base temp number? */ - if (compIsForInlining()) - { - if (ilArgNum >= info.compArgsCount) - { - compInlineResult->NoteFatal(InlineObservation::CALLEE_BAD_ARGUMENT_NUMBER); - return; - } + bool newTemps = (baseTmp == NO_BASE_TMP); - impPushVar(impInlineFetchArg(ilArgNum, impInlineInfo->inlArgInfo, impInlineInfo->lclVarInfo), - impInlineInfo->lclVarInfo[ilArgNum].lclVerTypeInfo); - } - else - { - if (ilArgNum >= info.compArgsCount) + if (newTemps) { - BADCODE("Bad IL"); + /* Grab enough temps for the whole stack */ + baseTmp = impGetSpillTmpBase(block); } - unsigned lclNum = compMapILargNum(ilArgNum); // account for possible hidden param + /* Spill all stack entries into temps */ + unsigned level, tempNum; - if (lclNum == info.compThisArg) + JITDUMP("\nSpilling stack entries into temps\n"); + for (level = 0, tempNum = baseTmp; level < verCurrentState.esStackDepth; level++, tempNum++) { - lclNum = lvaArg0Var; - } + GenTree* tree = verCurrentState.esStack[level].val; - impLoadVar(lclNum, offset); - } -} + /* VC generates code where it pushes a byref from one branch, and an int (ldc.i4 0) from + the other. This should merge to a byref in unverifiable code. + However, if the branch which leaves the TYP_I_IMPL on the stack is imported first, the + successor would be imported assuming there was a TYP_I_IMPL on + the stack. Thus the value would not get GC-tracked. Hence, + change the temp to TYP_BYREF and reimport the successors. + Note: We should only allow this in unverifiable code. + */ + if (tree->gtType == TYP_BYREF && lvaTable[tempNum].lvType == TYP_I_IMPL) + { + lvaTable[tempNum].lvType = TYP_BYREF; + impReimportMarkSuccessors(block); + markImport = true; + } -// Load a local on the operand stack -// Shared by the various CEE_LDLOC opcodes -// ilLclNum is the local index as specified in IL. -// It will be mapped to the correct lvaTable index -void Compiler::impLoadLoc(unsigned ilLclNum, IL_OFFSET offset) -{ - if (compIsForInlining()) - { - if (ilLclNum >= info.compMethodInfo->locals.numArgs) - { - compInlineResult->NoteFatal(InlineObservation::CALLEE_BAD_LOCAL_NUMBER); - return; - } +#ifdef TARGET_64BIT + if (genActualType(tree->gtType) == TYP_I_IMPL && lvaTable[tempNum].lvType == TYP_INT) + { + // Some other block in the spill clique set this to "int", but now we have "native int". + // Change the type and go back to re-import any blocks that used the wrong type. + lvaTable[tempNum].lvType = TYP_I_IMPL; + reimportSpillClique = true; + } + else if (genActualType(tree->gtType) == TYP_INT && lvaTable[tempNum].lvType == TYP_I_IMPL) + { + // Spill clique has decided this should be "native int", but this block only pushes an "int". + // Insert a sign-extension to "native int" so we match the clique. + verCurrentState.esStack[level].val = gtNewCastNode(TYP_I_IMPL, tree, false, TYP_I_IMPL); + } - // Get the local type - var_types lclTyp = impInlineInfo->lclVarInfo[ilLclNum + impInlineInfo->argCnt].lclTypeInfo; + // Consider the case where one branch left a 'byref' on the stack and the other leaves + // an 'int'. On 32-bit, this is allowed (in non-verifiable code) since they are the same + // size. JIT64 managed to make this work on 64-bit. For compatibility, we support JIT64 + // behavior instead of asserting and then generating bad code (where we save/restore the + // low 32 bits of a byref pointer to an 'int' sized local). If the 'int' side has been + // imported already, we need to change the type of the local and reimport the spill clique. + // If the 'byref' side has imported, we insert a cast from int to 'native int' to match + // the 'byref' size. + if (genActualType(tree->gtType) == TYP_BYREF && lvaTable[tempNum].lvType == TYP_INT) + { + // Some other block in the spill clique set this to "int", but now we have "byref". + // Change the type and go back to re-import any blocks that used the wrong type. + lvaTable[tempNum].lvType = TYP_BYREF; + reimportSpillClique = true; + } + else if (genActualType(tree->gtType) == TYP_INT && lvaTable[tempNum].lvType == TYP_BYREF) + { + // Spill clique has decided this should be "byref", but this block only pushes an "int". + // Insert a sign-extension to "native int" so we match the clique size. + verCurrentState.esStack[level].val = gtNewCastNode(TYP_I_IMPL, tree, false, TYP_I_IMPL); + } - typeInfo tiRetVal = impInlineInfo->lclVarInfo[ilLclNum + impInlineInfo->argCnt].lclVerTypeInfo; +#endif // TARGET_64BIT - /* Have we allocated a temp for this local? */ + if (tree->gtType == TYP_DOUBLE && lvaTable[tempNum].lvType == TYP_FLOAT) + { + // Some other block in the spill clique set this to "float", but now we have "double". + // Change the type and go back to re-import any blocks that used the wrong type. + lvaTable[tempNum].lvType = TYP_DOUBLE; + reimportSpillClique = true; + } + else if (tree->gtType == TYP_FLOAT && lvaTable[tempNum].lvType == TYP_DOUBLE) + { + // Spill clique has decided this should be "double", but this block only pushes a "float". + // Insert a cast to "double" so we match the clique. + verCurrentState.esStack[level].val = gtNewCastNode(TYP_DOUBLE, tree, false, TYP_DOUBLE); + } - unsigned lclNum = impInlineFetchLocal(ilLclNum DEBUGARG("Inline ldloc first use temp")); + /* If addStmt has a reference to tempNum (can only happen if we + are spilling to the temps already used by a previous block), + we need to spill addStmt */ - // All vars of inlined methods should be !lvNormalizeOnLoad() + if (addStmt != nullptr && !newTemps && gtHasRef(addStmt->GetRootNode(), tempNum)) + { + GenTree* addTree = addStmt->GetRootNode(); - assert(!lvaTable[lclNum].lvNormalizeOnLoad()); - lclTyp = genActualType(lclTyp); + if (addTree->gtOper == GT_JTRUE) + { + GenTree* relOp = addTree->AsOp()->gtOp1; + assert(relOp->OperIsCompare()); - impPushVar(gtNewLclvNode(lclNum, lclTyp), tiRetVal); - } - else - { - if (ilLclNum >= info.compMethodInfo->locals.numArgs) - { - BADCODE("Bad IL"); - } + var_types type = genActualType(relOp->AsOp()->gtOp1->TypeGet()); - unsigned lclNum = info.compArgsCount + ilLclNum; + if (gtHasRef(relOp->AsOp()->gtOp1, tempNum)) + { + unsigned temp = lvaGrabTemp(true DEBUGARG("spill addStmt JTRUE ref Op1")); + impAssignTempGen(temp, relOp->AsOp()->gtOp1, level); + type = genActualType(lvaTable[temp].TypeGet()); + relOp->AsOp()->gtOp1 = gtNewLclvNode(temp, type); + } - impLoadVar(lclNum, offset); - } -} + if (gtHasRef(relOp->AsOp()->gtOp2, tempNum)) + { + unsigned temp = lvaGrabTemp(true DEBUGARG("spill addStmt JTRUE ref Op2")); + impAssignTempGen(temp, relOp->AsOp()->gtOp2, level); + type = genActualType(lvaTable[temp].TypeGet()); + relOp->AsOp()->gtOp2 = gtNewLclvNode(temp, type); + } + } + else + { + assert(addTree->gtOper == GT_SWITCH && genActualTypeIsIntOrI(addTree->AsOp()->gtOp1->TypeGet())); -//------------------------------------------------------------------------ -// impAssignMultiRegTypeToVar: ensure calls that return structs in multiple -// registers return values to suitable temps. -// -// Arguments: -// op -- call returning a struct in registers -// hClass -- class handle for struct -// -// Returns: -// Tree with reference to struct local to use as call return value. + unsigned temp = lvaGrabTemp(true DEBUGARG("spill addStmt SWITCH")); + impAssignTempGen(temp, addTree->AsOp()->gtOp1, level); + addTree->AsOp()->gtOp1 = gtNewLclvNode(temp, genActualType(addTree->AsOp()->gtOp1->TypeGet())); + } + } -GenTree* Compiler::impAssignMultiRegTypeToVar(GenTree* op, - CORINFO_CLASS_HANDLE hClass DEBUGARG(CorInfoCallConvExtension callConv)) -{ - unsigned tmpNum = lvaGrabTemp(true DEBUGARG("Return value temp for multireg return")); - impAssignTempGen(tmpNum, op, hClass, CHECK_SPILL_ALL); - GenTree* ret = gtNewLclvNode(tmpNum, lvaTable[tmpNum].lvType); + /* Spill the stack entry, and replace with the temp */ + + if (!impSpillStackEntry(level, tempNum +#ifdef DEBUG + , + true, "Spill Stack Entry" +#endif + )) + { + if (markImport) + { + BADCODE("bad stack state"); + } + + // Oops. Something went wrong when spilling. Bad code. + verHandleVerificationFailure(block DEBUGARG(true)); - // TODO-1stClassStructs: Handle constant propagation and CSE-ing of multireg returns. - ret->gtFlags |= GTF_DONT_CSE; + goto SPILLSTACK; + } + } - assert(IsMultiRegReturnedType(hClass, callConv) || op->IsMultiRegNode()); + /* Put back the 'jtrue'/'switch' if we removed it earlier */ - // Set "lvIsMultiRegRet" to block promotion under "!lvaEnregMultiRegVars". - lvaTable[tmpNum].lvIsMultiRegRet = true; + if (addStmt != nullptr) + { + impAppendStmt(addStmt, (unsigned)CHECK_SPILL_NONE); + } + } - return ret; -} + // Some of the append/spill logic works on compCurBB -//------------------------------------------------------------------------ -// impReturnInstruction: import a return or an explicit tail call -// -// Arguments: -// prefixFlags -- active IL prefixes -// opcode -- [in, out] IL opcode -// -// Returns: -// True if import was successful (may fail for some inlinees) -// -bool Compiler::impReturnInstruction(int prefixFlags, OPCODE& opcode) -{ - const bool isTailCall = (prefixFlags & PREFIX_TAILCALL) != 0; + assert(compCurBB == block); -#ifdef DEBUG - // If we are importing an inlinee and have GC ref locals we always - // need to have a spill temp for the return value. This temp - // should have been set up in advance, over in fgFindBasicBlocks. - if (compIsForInlining() && impInlineInfo->HasGcRefLocals() && (info.compRetType != TYP_VOID)) - { - assert(lvaInlineeReturnSpillTemp != BAD_VAR_NUM); - } -#endif // DEBUG + /* Save the tree list in the block */ + impEndTreeList(block); - GenTree* op2 = nullptr; - GenTree* op1 = nullptr; - CORINFO_CLASS_HANDLE retClsHnd = nullptr; + // impEndTreeList sets BBF_IMPORTED on the block + // We do *NOT* want to set it later than this because + // impReimportSpillClique might clear it if this block is both a + // predecessor and successor in the current spill clique + assert(block->bbFlags & BBF_IMPORTED); - if (info.compRetType != TYP_VOID) + // If we had a int/native int, or float/double collision, we need to re-import + if (reimportSpillClique) { - StackEntry se = impPopStack(); - retClsHnd = se.seTypeInfo.GetClassHandle(); - op2 = se.val; + // This will re-import all the successors of block (as well as each of their predecessors) + impReimportSpillClique(block); - if (!compIsForInlining()) + // For blocks that haven't been imported yet, we still need to mark them as pending import. + for (BasicBlock* const succ : block->Succs()) { - impBashVarAddrsToI(op2); - op2 = impImplicitIorI4Cast(op2, info.compRetType); - op2 = impImplicitR4orR8Cast(op2, info.compRetType); - - assertImp((genActualType(op2->TypeGet()) == genActualType(info.compRetType)) || - ((op2->TypeGet() == TYP_I_IMPL) && (info.compRetType == TYP_BYREF)) || - ((op2->TypeGet() == TYP_BYREF) && (info.compRetType == TYP_I_IMPL)) || - (varTypeIsFloating(op2->gtType) && varTypeIsFloating(info.compRetType)) || - (varTypeIsStruct(op2) && varTypeIsStruct(info.compRetType))); - -#ifdef DEBUG - if (!isTailCall && opts.compGcChecks && (info.compRetType == TYP_REF)) + if ((succ->bbFlags & BBF_IMPORTED) == 0) { - // DDB 3483 : JIT Stress: early termination of GC ref's life time in exception code path - // VSW 440513: Incorrect gcinfo on the return value under COMPlus_JitGCChecks=1 for methods with - // one-return BB. - - assert(op2->gtType == TYP_REF); - - // confirm that the argument is a GC pointer (for debugging (GC stress)) - op2 = gtNewHelperCallNode(CORINFO_HELP_CHECK_OBJ, TYP_REF, op2); - - if (verbose) - { - printf("\ncompGcChecks tree:\n"); - gtDispTree(op2); - } + impImportBlockPending(succ); } -#endif } - else + } + else // the normal case + { + // otherwise just import the successors of block + + /* Does this block jump to any other blocks? */ + for (BasicBlock* const succ : block->Succs()) { - if (verCurrentState.esStackDepth != 0) - { - assert(compIsForInlining()); - JITDUMP("CALLSITE_COMPILATION_ERROR: inlinee's stack is not empty."); - compInlineResult->NoteFatal(InlineObservation::CALLSITE_COMPILATION_ERROR); - return false; - } + impImportBlockPending(succ); + } + } +} +#ifdef _PREFAST_ +#pragma warning(pop) +#endif + +/*****************************************************************************/ +// +// Ensures that "block" is a member of the list of BBs waiting to be imported, pushing it on the list if +// necessary (and ensures that it is a member of the set of BB's on the list, by setting its byte in +// impPendingBlockMembers). Merges the current verification state into the verification state of "block" +// (its "pre-state"). +void Compiler::impImportBlockPending(BasicBlock* block) +{ #ifdef DEBUG - if (verbose) - { - printf("\n\n Inlinee Return expression (before normalization) =>\n"); - gtDispTree(op2); - } + if (verbose) + { + printf("\nimpImportBlockPending for " FMT_BB "\n", block->bbNum); + } #endif - InlineCandidateInfo* inlCandInfo = impInlineInfo->inlineCandidateInfo; - GenTreeRetExpr* inlRetExpr = inlCandInfo->retExpr; - // Make sure the type matches the original call. - - var_types returnType = genActualType(op2->gtType); - var_types originalCallType = inlCandInfo->fncRetType; - if ((returnType != originalCallType) && (originalCallType == TYP_STRUCT)) - { - originalCallType = impNormStructType(inlCandInfo->methInfo.args.retTypeClass); - } + // We will add a block to the pending set if it has not already been imported (or needs to be re-imported), + // or if it has, but merging in a predecessor's post-state changes the block's pre-state. + // (When we're doing verification, we always attempt the merge to detect verification errors.) - if (returnType != originalCallType) - { - // Allow TYP_BYREF to be returned as TYP_I_IMPL and vice versa. - if (((returnType == TYP_BYREF) && (originalCallType == TYP_I_IMPL)) || - ((returnType == TYP_I_IMPL) && (originalCallType == TYP_BYREF))) - { - JITDUMP("Allowing return type mismatch: have %s, needed %s\n", varTypeName(returnType), - varTypeName(originalCallType)); - } - else - { - JITDUMP("Return type mismatch: have %s, needed %s\n", varTypeName(returnType), - varTypeName(originalCallType)); - compInlineResult->NoteFatal(InlineObservation::CALLSITE_RETURN_TYPE_MISMATCH); - return false; - } - } + // If the block has not been imported, add to pending set. + bool addToPending = ((block->bbFlags & BBF_IMPORTED) == 0); - // Below, we are going to set impInlineInfo->retExpr to the tree with the return - // expression. At this point, retExpr could already be set if there are multiple - // return blocks (meaning fgNeedReturnSpillTemp() == true) and one of - // the other blocks already set it. If there is only a single return block, - // retExpr shouldn't be set. However, this is not true if we reimport a block - // with a return. In that case, retExpr will be set, then the block will be - // reimported, but retExpr won't get cleared as part of setting the block to - // be reimported. The reimported retExpr value should be the same, so even if - // we don't unconditionally overwrite it, it shouldn't matter. - if (info.compRetNativeType != TYP_STRUCT) - { - // compRetNativeType is not TYP_STRUCT. - // This implies it could be either a scalar type or SIMD vector type or - // a struct type that can be normalized to a scalar type. + // Initialize bbEntryState just the first time we try to add this block to the pending list + // Just because bbEntryState is NULL, doesn't mean the pre-state wasn't previously set + // We use NULL to indicate the 'common' state to avoid memory allocation + if ((block->bbEntryState == nullptr) && ((block->bbFlags & (BBF_IMPORTED | BBF_FAILED_VERIFICATION)) == 0) && + (impGetPendingBlockMember(block) == 0)) + { + verInitBBEntryState(block, &verCurrentState); + assert(block->bbStkDepth == 0); + block->bbStkDepth = static_cast(verCurrentState.esStackDepth); + assert(addToPending); + assert(impGetPendingBlockMember(block) == 0); + } + else + { + // The stack should have the same height on entry to the block from all its predecessors. + if (block->bbStkDepth != verCurrentState.esStackDepth) + { +#ifdef DEBUG + char buffer[400]; + sprintf_s(buffer, sizeof(buffer), + "Block at offset %4.4x to %4.4x in %0.200s entered with different stack depths.\n" + "Previous depth was %d, current depth is %d", + block->bbCodeOffs, block->bbCodeOffsEnd, info.compFullName, block->bbStkDepth, + verCurrentState.esStackDepth); + buffer[400 - 1] = 0; + NO_WAY(buffer); +#else + NO_WAY("Block entered with different stack depths"); +#endif + } - if (varTypeIsStruct(info.compRetType)) - { - noway_assert(info.compRetBuffArg == BAD_VAR_NUM); - // Handle calls with "fake" return buffers. - op2 = impFixupStructReturnType(op2); - } - else - { - // Do we have to normalize? - var_types fncRealRetType = JITtype2varType(info.compMethodInfo->args.retType); - // For RET_EXPR get the type info from the call. Regardless - // of whether it ends up inlined or not normalization will - // happen as part of that function's codegen. - GenTree* returnedTree = op2->OperIs(GT_RET_EXPR) ? op2->AsRetExpr()->gtInlineCandidate : op2; - if ((varTypeIsSmall(returnedTree->TypeGet()) || varTypeIsSmall(fncRealRetType)) && - fgCastNeeded(returnedTree, fncRealRetType)) - { - // Small-typed return values are normalized by the callee - op2 = gtNewCastNode(TYP_INT, op2, false, fncRealRetType); - } - } + if (!addToPending) + { + return; + } - if (fgNeedReturnSpillTemp()) - { - assert(info.compRetNativeType != TYP_VOID && - (fgMoreThanOneReturnBlock() || impInlineInfo->HasGcRefLocals())); + if (block->bbStkDepth > 0) + { + // We need to fix the types of any spill temps that might have changed: + // int->native int, float->double, int->byref, etc. + impRetypeEntryStateTemps(block); + } - // If this method returns a ref type, track the actual types seen in the returns. - if (info.compRetType == TYP_REF) - { - bool isExact = false; - bool isNonNull = false; - CORINFO_CLASS_HANDLE returnClsHnd = gtGetClassHandle(op2, &isExact, &isNonNull); + // OK, we must add to the pending list, if it's not already in it. + if (impGetPendingBlockMember(block) != 0) + { + return; + } + } - if (inlRetExpr->gtSubstExpr == nullptr) - { - // This is the first return, so best known type is the type - // of this return value. - impInlineInfo->retExprClassHnd = returnClsHnd; - impInlineInfo->retExprClassHndIsExact = isExact; - } - else if (impInlineInfo->retExprClassHnd != returnClsHnd) - { - // This return site type differs from earlier seen sites, - // so reset the info and we'll fall back to using the method's - // declared return type for the return spill temp. - impInlineInfo->retExprClassHnd = nullptr; - impInlineInfo->retExprClassHndIsExact = false; - } - } + // Get an entry to add to the pending list - impAssignTempGen(lvaInlineeReturnSpillTemp, op2, se.seTypeInfo.GetClassHandle(), CHECK_SPILL_ALL); + PendingDsc* dsc; - var_types lclRetType = lvaGetDesc(lvaInlineeReturnSpillTemp)->lvType; - GenTree* tmpOp2 = gtNewLclvNode(lvaInlineeReturnSpillTemp, lclRetType); + if (impPendingFree) + { + // We can reuse one of the freed up dscs. + dsc = impPendingFree; + impPendingFree = dsc->pdNext; + } + else + { + // We have to create a new dsc + dsc = new (this, CMK_Unknown) PendingDsc; + } - op2 = tmpOp2; -#ifdef DEBUG - if (inlRetExpr->gtSubstExpr != nullptr) - { - // Some other block(s) have seen the CEE_RET first. - // Better they spilled to the same temp. - assert(inlRetExpr->gtSubstExpr->gtOper == GT_LCL_VAR); - assert(inlRetExpr->gtSubstExpr->AsLclVarCommon()->GetLclNum() == - op2->AsLclVarCommon()->GetLclNum()); - } -#endif - } + dsc->pdBB = block; + dsc->pdSavedStack.ssDepth = verCurrentState.esStackDepth; + dsc->pdThisPtrInit = verCurrentState.thisInitialized; -#ifdef DEBUG - if (verbose) - { - printf("\n\n Inlinee Return expression (after normalization) =>\n"); - gtDispTree(op2); - } -#endif + // Save the stack trees for later - // Report the return expression - inlRetExpr->gtSubstExpr = op2; - } - else - { - // compRetNativeType is TYP_STRUCT. - // This implies that struct return via RetBuf arg or multi-reg struct return. + if (verCurrentState.esStackDepth) + { + impSaveStackState(&dsc->pdSavedStack, false); + } - GenTreeCall* iciCall = impInlineInfo->iciCall->AsCall(); + // Add the entry to the pending list - // Assign the inlinee return into a spill temp. - if (fgNeedReturnSpillTemp()) - { - // in this case we have to insert multiple struct copies to the temp - // and the retexpr is just the temp. - assert(info.compRetNativeType != TYP_VOID); - assert(fgMoreThanOneReturnBlock() || impInlineInfo->HasGcRefLocals()); + dsc->pdNext = impPendingList; + impPendingList = dsc; + impSetPendingBlockMember(block, 1); // And indicate that it's now a member of the set. - impAssignTempGen(lvaInlineeReturnSpillTemp, op2, se.seTypeInfo.GetClassHandle(), CHECK_SPILL_ALL); - } + // Various assertions require us to now to consider the block as not imported (at least for + // the final time...) + block->bbFlags &= ~BBF_IMPORTED; - if (compMethodReturnsMultiRegRetType()) - { - assert(!iciCall->ShouldHaveRetBufArg()); +#ifdef DEBUG + if (verbose && 0) + { + printf("Added PendingDsc - %08p for " FMT_BB "\n", dspPtr(dsc), block->bbNum); + } +#endif +} - if (fgNeedReturnSpillTemp()) - { - if (inlRetExpr->gtSubstExpr == nullptr) - { - // The inlinee compiler has figured out the type of the temp already. Use it here. - inlRetExpr->gtSubstExpr = - gtNewLclvNode(lvaInlineeReturnSpillTemp, lvaTable[lvaInlineeReturnSpillTemp].lvType); - } - } - else - { - inlRetExpr->gtSubstExpr = op2; - } - } - else // The struct was to be returned via a return buffer. - { - assert(iciCall->gtArgs.HasRetBuffer()); - GenTree* dest = gtCloneExpr(iciCall->gtArgs.GetRetBufferArg()->GetEarlyNode()); +/*****************************************************************************/ +// +// Ensures that "block" is a member of the list of BBs waiting to be imported, pushing it on the list if +// necessary (and ensures that it is a member of the set of BB's on the list, by setting its byte in +// impPendingBlockMembers). Does *NOT* change the existing "pre-state" of the block. - if (fgNeedReturnSpillTemp()) - { - // If this is the first return we have seen set the retExpr. - if (inlRetExpr->gtSubstExpr == nullptr) - { - inlRetExpr->gtSubstExpr = - impAssignStructPtr(dest, gtNewLclvNode(lvaInlineeReturnSpillTemp, info.compRetType), - retClsHnd, CHECK_SPILL_ALL); - } - } - else - { - inlRetExpr->gtSubstExpr = impAssignStructPtr(dest, op2, retClsHnd, CHECK_SPILL_ALL); - } - } - } +void Compiler::impReimportBlockPending(BasicBlock* block) +{ + JITDUMP("\nimpReimportBlockPending for " FMT_BB, block->bbNum); - // If gtSubstExpr is an arbitrary tree then we may need to - // propagate mandatory "IR presence" flags (e.g. BBF_HAS_IDX_LEN) - // to the BB it ends up in. - inlRetExpr->gtSubstBB = fgNeedReturnSpillTemp() ? nullptr : compCurBB; - } - } + assert(block->bbFlags & BBF_IMPORTED); - if (compIsForInlining()) + // OK, we must add to the pending list, if it's not already in it. + if (impGetPendingBlockMember(block) != 0) { - return true; + return; } - if (info.compRetBuffArg != BAD_VAR_NUM) - { - // Assign value to return buff (first param) - GenTree* retBuffAddr = - gtNewLclvNode(info.compRetBuffArg, TYP_BYREF DEBUGARG(impCurStmtDI.GetLocation().GetOffset())); + // Get an entry to add to the pending list - op2 = impAssignStructPtr(retBuffAddr, op2, retClsHnd, CHECK_SPILL_ALL); - impAppendTree(op2, CHECK_SPILL_NONE, impCurStmtDI); + PendingDsc* dsc; - // There are cases where the address of the implicit RetBuf should be returned explicitly. - // - if (compMethodReturnsRetBufAddr()) - { - op1 = gtNewOperNode(GT_RETURN, TYP_BYREF, gtNewLclvNode(info.compRetBuffArg, TYP_BYREF)); - } - else - { - op1 = new (this, GT_RETURN) GenTreeOp(GT_RETURN, TYP_VOID); - } + if (impPendingFree) + { + // We can reuse one of the freed up dscs. + dsc = impPendingFree; + impPendingFree = dsc->pdNext; } - else if (varTypeIsStruct(info.compRetType)) + else { -#if !FEATURE_MULTIREG_RET - // For both ARM architectures the HFA native types are maintained as structs. - // Also on System V AMD64 the multireg structs returns are also left as structs. - noway_assert(info.compRetNativeType != TYP_STRUCT); -#endif - op2 = impFixupStructReturnType(op2); - op1 = gtNewOperNode(GT_RETURN, genActualType(info.compRetType), op2); + // We have to create a new dsc + dsc = new (this, CMK_ImpStack) PendingDsc; } - else if (info.compRetType != TYP_VOID) + + dsc->pdBB = block; + + if (block->bbEntryState) { - op1 = gtNewOperNode(GT_RETURN, genActualType(info.compRetType), op2); + dsc->pdThisPtrInit = block->bbEntryState->thisInitialized; + dsc->pdSavedStack.ssDepth = block->bbEntryState->esStackDepth; + dsc->pdSavedStack.ssTrees = block->bbEntryState->esStack; } else { - op1 = new (this, GT_RETURN) GenTreeOp(GT_RETURN, TYP_VOID); + dsc->pdThisPtrInit = TIS_Bottom; + dsc->pdSavedStack.ssDepth = 0; + dsc->pdSavedStack.ssTrees = nullptr; } - // We must have imported a tailcall and jumped to RET - if (isTailCall) - { - assert(verCurrentState.esStackDepth == 0 && impOpcodeIsCallOpcode(opcode)); + // Add the entry to the pending list - opcode = CEE_RET; // To prevent trying to spill if CALL_SITE_BOUNDARIES + dsc->pdNext = impPendingList; + impPendingList = dsc; + impSetPendingBlockMember(block, 1); // And indicate that it's now a member of the set. - // impImportCall() would have already appended TYP_VOID calls - if (info.compRetType == TYP_VOID) - { - return true; - } - } + // Various assertions require us to now to consider the block as not imported (at least for + // the final time...) + block->bbFlags &= ~BBF_IMPORTED; - impAppendTree(op1, CHECK_SPILL_NONE, impCurStmtDI); #ifdef DEBUG - // Remember at which BC offset the tree was finished - impNoteLastILoffs(); + if (verbose && 0) + { + printf("Added PendingDsc - %08p for " FMT_BB "\n", dspPtr(dsc), block->bbNum); + } #endif - return true; } -/***************************************************************************** - * Mark the block as unimported. - * Note that the caller is responsible for calling impImportBlockPending(), - * with the appropriate stack-state - */ +void* Compiler::BlockListNode::operator new(size_t sz, Compiler* comp) +{ + if (comp->impBlockListNodeFreeList == nullptr) + { + return comp->getAllocator(CMK_BasicBlock).allocate(1); + } + else + { + BlockListNode* res = comp->impBlockListNodeFreeList; + comp->impBlockListNodeFreeList = res->m_next; + return res; + } +} -inline void Compiler::impReimportMarkBlock(BasicBlock* block) +void Compiler::FreeBlockListNode(Compiler::BlockListNode* node) { -#ifdef DEBUG - if (verbose && (block->bbFlags & BBF_IMPORTED)) + node->m_next = impBlockListNodeFreeList; + impBlockListNodeFreeList = node; +} + +void Compiler::impWalkSpillCliqueFromPred(BasicBlock* block, SpillCliqueWalker* callback) +{ + bool toDo = true; + + noway_assert(!fgComputePredsDone); + if (!fgCheapPredsValid) { - printf("\n" FMT_BB " will be reimported\n", block->bbNum); + fgComputeCheapPreds(); + } + + BlockListNode* succCliqueToDo = nullptr; + BlockListNode* predCliqueToDo = new (this) BlockListNode(block); + while (toDo) + { + toDo = false; + // Look at the successors of every member of the predecessor to-do list. + while (predCliqueToDo != nullptr) + { + BlockListNode* node = predCliqueToDo; + predCliqueToDo = node->m_next; + BasicBlock* blk = node->m_blk; + FreeBlockListNode(node); + + for (BasicBlock* const succ : blk->Succs()) + { + // If it's not already in the clique, add it, and also add it + // as a member of the successor "toDo" set. + if (impSpillCliqueGetMember(SpillCliqueSucc, succ) == 0) + { + callback->Visit(SpillCliqueSucc, succ); + impSpillCliqueSetMember(SpillCliqueSucc, succ, 1); + succCliqueToDo = new (this) BlockListNode(succ, succCliqueToDo); + toDo = true; + } + } + } + // Look at the predecessors of every member of the successor to-do list. + while (succCliqueToDo != nullptr) + { + BlockListNode* node = succCliqueToDo; + succCliqueToDo = node->m_next; + BasicBlock* blk = node->m_blk; + FreeBlockListNode(node); + + for (BasicBlockList* pred = blk->bbCheapPreds; pred != nullptr; pred = pred->next) + { + BasicBlock* predBlock = pred->block; + // If it's not already in the clique, add it, and also add it + // as a member of the predecessor "toDo" set. + if (impSpillCliqueGetMember(SpillCliquePred, predBlock) == 0) + { + callback->Visit(SpillCliquePred, predBlock); + impSpillCliqueSetMember(SpillCliquePred, predBlock, 1); + predCliqueToDo = new (this) BlockListNode(predBlock, predCliqueToDo); + toDo = true; + } + } + } } -#endif - block->bbFlags &= ~BBF_IMPORTED; + // If this fails, it means we didn't walk the spill clique properly and somehow managed + // miss walking back to include the predecessor we started from. + // This most likely cause: missing or out of date bbPreds + assert(impSpillCliqueGetMember(SpillCliquePred, block) != 0); } -/***************************************************************************** - * Mark the successors of the given block as unimported. - * Note that the caller is responsible for calling impImportBlockPending() - * for all the successors, with the appropriate stack-state. - */ - -void Compiler::impReimportMarkSuccessors(BasicBlock* block) +void Compiler::SetSpillTempsBase::Visit(SpillCliqueDir predOrSucc, BasicBlock* blk) { - for (BasicBlock* const succBlock : block->Succs()) + if (predOrSucc == SpillCliqueSucc) { - impReimportMarkBlock(succBlock); + assert(blk->bbStkTempsIn == NO_BASE_TMP); // Should not already be a member of a clique as a successor. + blk->bbStkTempsIn = m_baseTmp; } -} - -/***************************************************************************** - * - * Filter wrapper to handle only passed in exception code - * from it). - */ - -LONG FilterVerificationExceptions(PEXCEPTION_POINTERS pExceptionPointers, LPVOID lpvParam) -{ - if (pExceptionPointers->ExceptionRecord->ExceptionCode == SEH_VERIFICATION_EXCEPTION) + else { - return EXCEPTION_EXECUTE_HANDLER; + assert(predOrSucc == SpillCliquePred); + assert(blk->bbStkTempsOut == NO_BASE_TMP); // Should not already be a member of a clique as a predecessor. + blk->bbStkTempsOut = m_baseTmp; } - - return EXCEPTION_CONTINUE_SEARCH; } -void Compiler::impVerifyEHBlock(BasicBlock* block, bool isTryStart) +void Compiler::ReimportSpillClique::Visit(SpillCliqueDir predOrSucc, BasicBlock* blk) { - assert(block->hasTryIndex()); - assert(!compIsForInlining()); - - unsigned tryIndex = block->getTryIndex(); - EHblkDsc* HBtab = ehGetDsc(tryIndex); + // For Preds we could be a little smarter and just find the existing store + // and re-type it/add a cast, but that is complicated and hopefully very rare, so + // just re-import the whole block (just like we do for successors) - if (isTryStart) + if (((blk->bbFlags & BBF_IMPORTED) == 0) && (m_pComp->impGetPendingBlockMember(blk) == 0)) { - assert(block->bbFlags & BBF_TRY_BEG); + // If we haven't imported this block and we're not going to (because it isn't on + // the pending list) then just ignore it for now. - // The Stack must be empty - // - if (block->bbStkDepth != 0) - { - BADCODE("Evaluation stack must be empty on entry into a try block"); - } + // This block has either never been imported (EntryState == NULL) or it failed + // verification. Neither state requires us to force it to be imported now. + assert((blk->bbEntryState == nullptr) || (blk->bbFlags & BBF_FAILED_VERIFICATION)); + return; } - // Save the stack contents, we'll need to restore it later - // - SavedStack blockState; - impSaveStackState(&blockState, false); - - while (HBtab != nullptr) + // For successors we have a valid verCurrentState, so just mark them for reimport + // the 'normal' way + // Unlike predecessors, we *DO* need to reimport the current block because the + // initial import had the wrong entry state types. + // Similarly, blocks that are currently on the pending list, still need to call + // impImportBlockPending to fixup their entry state. + if (predOrSucc == SpillCliqueSucc) { - // Recursively process the handler block, if we haven't already done so. - BasicBlock* hndBegBB = HBtab->ebdHndBeg; - - if (((hndBegBB->bbFlags & BBF_IMPORTED) == 0) && (impGetPendingBlockMember(hndBegBB) == 0)) - { - // Construct the proper verification stack state - // either empty or one that contains just - // the Exception Object that we are dealing with - // - verCurrentState.esStackDepth = 0; - - if (handlerGetsXcptnObj(hndBegBB->bbCatchTyp)) - { - CORINFO_CLASS_HANDLE clsHnd; - - if (HBtab->HasFilter()) - { - clsHnd = impGetObjectClass(); - } - else - { - CORINFO_RESOLVED_TOKEN resolvedToken; - - resolvedToken.tokenContext = impTokenLookupContextHandle; - resolvedToken.tokenScope = info.compScopeHnd; - resolvedToken.token = HBtab->ebdTyp; - resolvedToken.tokenType = CORINFO_TOKENKIND_Class; - info.compCompHnd->resolveToken(&resolvedToken); - - clsHnd = resolvedToken.hClass; - } + m_pComp->impReimportMarkBlock(blk); - // push catch arg the stack, spill to a temp if necessary - // Note: can update HBtab->ebdHndBeg! - hndBegBB = impPushCatchArgOnStack(hndBegBB, clsHnd, false); - } + // Set the current stack state to that of the blk->bbEntryState + m_pComp->verResetCurrentState(blk, &m_pComp->verCurrentState); + assert(m_pComp->verCurrentState.thisInitialized == blk->bbThisOnEntry()); - // Queue up the handler for importing - // - impImportBlockPending(hndBegBB); - } + m_pComp->impImportBlockPending(blk); + } + else if ((blk != m_pComp->compCurBB) && ((blk->bbFlags & BBF_IMPORTED) != 0)) + { + // As described above, we are only visiting predecessors so they can + // add the appropriate casts, since we have already done that for the current + // block, it does not need to be reimported. + // Nor do we need to reimport blocks that are still pending, but not yet + // imported. + // + // For predecessors, we have no state to seed the EntryState, so we just have + // to assume the existing one is correct. + // If the block is also a successor, it will get the EntryState properly + // updated when it is visited as a successor in the above "if" block. + assert(predOrSucc == SpillCliquePred); + m_pComp->impReimportBlockPending(blk); + } +} - // Process the filter block, if we haven't already done so. - if (HBtab->HasFilter()) +// Re-type the incoming lclVar nodes to match the varDsc. +void Compiler::impRetypeEntryStateTemps(BasicBlock* blk) +{ + if (blk->bbEntryState != nullptr) + { + EntryState* es = blk->bbEntryState; + for (unsigned level = 0; level < es->esStackDepth; level++) { - /* @VERIFICATION : Ideally the end of filter state should get - propagated to the catch handler, this is an incompleteness, - but is not a security/compliance issue, since the only - interesting state is the 'thisInit' state. - */ - - BasicBlock* filterBB = HBtab->ebdFilter; - - if (((filterBB->bbFlags & BBF_IMPORTED) == 0) && (impGetPendingBlockMember(filterBB) == 0)) + GenTree* tree = es->esStack[level].val; + if ((tree->gtOper == GT_LCL_VAR) || (tree->gtOper == GT_LCL_FLD)) { - verCurrentState.esStackDepth = 0; - - // push catch arg the stack, spill to a temp if necessary - // Note: can update HBtab->ebdFilter! - const bool isSingleBlockFilter = (filterBB->bbNext == hndBegBB); - filterBB = impPushCatchArgOnStack(filterBB, impGetObjectClass(), isSingleBlockFilter); - - impImportBlockPending(filterBB); + es->esStack[level].val->gtType = lvaGetDesc(tree->AsLclVarCommon())->TypeGet(); } } - - // Now process our enclosing try index (if any) - // - tryIndex = HBtab->ebdEnclosingTryIndex; - if (tryIndex == EHblkDsc::NO_ENCLOSING_INDEX) - { - HBtab = nullptr; - } - else - { - HBtab = ehGetDsc(tryIndex); - } } - - // Restore the stack contents - impRestoreStackState(&blockState); } -//*************************************************************** -// Import the instructions for the given basic block. Perform -// verification, throwing an exception on failure. Push any successor blocks that are enabled for the first -// time, or whose verification pre-state is changed. - -#ifdef _PREFAST_ -#pragma warning(push) -#pragma warning(disable : 21000) // Suppress PREFast warning about overly large function -#endif -void Compiler::impImportBlock(BasicBlock* block) +unsigned Compiler::impGetSpillTmpBase(BasicBlock* block) { - // BBF_INTERNAL blocks only exist during importation due to EH canonicalization. We need to - // handle them specially. In particular, there is no IL to import for them, but we do need - // to mark them as imported and put their successors on the pending import list. - if (block->bbFlags & BBF_INTERNAL) + if (block->bbStkTempsOut != NO_BASE_TMP) { - JITDUMP("Marking BBF_INTERNAL block " FMT_BB " as BBF_IMPORTED\n", block->bbNum); - block->bbFlags |= BBF_IMPORTED; - - for (BasicBlock* const succBlock : block->Succs()) - { - impImportBlockPending(succBlock); - } - - return; + return block->bbStkTempsOut; } - bool markImport; +#ifdef DEBUG + if (verbose) + { + printf("\n*************** In impGetSpillTmpBase(" FMT_BB ")\n", block->bbNum); + } +#endif // DEBUG - assert(block); + // Otherwise, choose one, and propagate to all members of the spill clique. + // Grab enough temps for the whole stack. + unsigned baseTmp = lvaGrabTemps(verCurrentState.esStackDepth DEBUGARG("IL Stack Entries")); + SetSpillTempsBase callback(baseTmp); - /* Make the block globally available */ + // We do *NOT* need to reset the SpillClique*Members because a block can only be the predecessor + // to one spill clique, and similarly can only be the successor to one spill clique + impWalkSpillCliqueFromPred(block, &callback); - compCurBB = block; + return baseTmp; +} +void Compiler::impReimportSpillClique(BasicBlock* block) +{ #ifdef DEBUG - /* Initialize the debug variables */ - impCurOpcName = "unknown"; - impCurOpcOffs = block->bbCodeOffs; -#endif - - /* Set the current stack state to the merged result */ - verResetCurrentState(block, &verCurrentState); + if (verbose) + { + printf("\n*************** In impReimportSpillClique(" FMT_BB ")\n", block->bbNum); + } +#endif // DEBUG - /* Now walk the code and import the IL into GenTrees */ + // If we get here, it is because this block is already part of a spill clique + // and one predecessor had an outgoing live stack slot of type int, and this + // block has an outgoing live stack slot of type native int. + // We need to reset these before traversal because they have already been set + // by the previous walk to determine all the members of the spill clique. + impInlineRoot()->impSpillCliquePredMembers.Reset(); + impInlineRoot()->impSpillCliqueSuccMembers.Reset(); - struct FilterVerificationExceptionsParam - { - Compiler* pThis; - BasicBlock* block; - }; - FilterVerificationExceptionsParam param; + ReimportSpillClique callback(this); - param.pThis = this; - param.block = block; + impWalkSpillCliqueFromPred(block, &callback); +} - PAL_TRY(FilterVerificationExceptionsParam*, pParam, ¶m) +// Set the pre-state of "block" (which should not have a pre-state allocated) to +// a copy of "srcState", cloning tree pointers as required. +void Compiler::verInitBBEntryState(BasicBlock* block, EntryState* srcState) +{ + if (srcState->esStackDepth == 0 && srcState->thisInitialized == TIS_Bottom) { - /* @VERIFICATION : For now, the only state propagation from try - to it's handler is "thisInit" state (stack is empty at start of try). - In general, for state that we track in verification, we need to - model the possibility that an exception might happen at any IL - instruction, so we really need to merge all states that obtain - between IL instructions in a try block into the start states of - all handlers. + block->bbEntryState = nullptr; + return; + } - However we do not allow the 'this' pointer to be uninitialized when - entering most kinds try regions (only try/fault are allowed to have - an uninitialized this pointer on entry to the try) + block->bbEntryState = getAllocator(CMK_Unknown).allocate(1); - Fortunately, the stack is thrown away when an exception - leads to a handler, so we don't have to worry about that. - We DO, however, have to worry about the "thisInit" state. - But only for the try/fault case. + // block->bbEntryState.esRefcount = 1; - The only allowed transition is from TIS_Uninit to TIS_Init. + block->bbEntryState->esStackDepth = srcState->esStackDepth; + block->bbEntryState->thisInitialized = TIS_Bottom; - So for a try/fault region for the fault handler block - we will merge the start state of the try begin - and the post-state of each block that is part of this try region - */ + if (srcState->esStackDepth > 0) + { + block->bbSetStack(new (this, CMK_Unknown) StackEntry[srcState->esStackDepth]); + unsigned stackSize = srcState->esStackDepth * sizeof(StackEntry); - // merge the start state of the try begin - // - if (pParam->block->bbFlags & BBF_TRY_BEG) + memcpy(block->bbEntryState->esStack, srcState->esStack, stackSize); + for (unsigned level = 0; level < srcState->esStackDepth; level++) { - pParam->pThis->impVerifyEHBlock(pParam->block, true); + GenTree* tree = srcState->esStack[level].val; + block->bbEntryState->esStack[level].val = gtCloneExpr(tree); } + } - pParam->pThis->impImportBlockCode(pParam->block); - - // As discussed above: - // merge the post-state of each block that is part of this try region - // - if (pParam->block->hasTryIndex()) - { - pParam->pThis->impVerifyEHBlock(pParam->block, false); - } + if (verTrackObjCtorInitState) + { + verSetThisInit(block, srcState->thisInitialized); } - PAL_EXCEPT_FILTER(FilterVerificationExceptions) + + return; +} + +void Compiler::verSetThisInit(BasicBlock* block, ThisInitState tis) +{ + assert(tis != TIS_Bottom); // Precondition. + if (block->bbEntryState == nullptr) { - verHandleVerificationFailure(block DEBUGARG(false)); + block->bbEntryState = new (this, CMK_Unknown) EntryState(); } - PAL_ENDTRY - if (compDonotInline()) + block->bbEntryState->thisInitialized = tis; +} + +/* + * Resets the current state to the state at the start of the basic block + */ +void Compiler::verResetCurrentState(BasicBlock* block, EntryState* destState) +{ + + if (block->bbEntryState == nullptr) { + destState->esStackDepth = 0; + destState->thisInitialized = TIS_Bottom; return; } - assert(!compDonotInline()); - - markImport = false; + destState->esStackDepth = block->bbEntryState->esStackDepth; -SPILLSTACK: + if (destState->esStackDepth > 0) + { + unsigned stackSize = destState->esStackDepth * sizeof(StackEntry); - unsigned baseTmp = NO_BASE_TMP; // input temps assigned to successor blocks - bool reimportSpillClique = false; - BasicBlock* tgtBlock = nullptr; + memcpy(destState->esStack, block->bbStackOnEntry(), stackSize); + } - /* If the stack is non-empty, we might have to spill its contents */ + destState->thisInitialized = block->bbThisOnEntry(); - if (verCurrentState.esStackDepth != 0) - { - impBoxTemp = BAD_VAR_NUM; // if a box temp is used in a block that leaves something - // on the stack, its lifetime is hard to determine, simply - // don't reuse such temps. + return; +} - Statement* addStmt = nullptr; +ThisInitState BasicBlock::bbThisOnEntry() const +{ + return bbEntryState ? bbEntryState->thisInitialized : TIS_Bottom; +} - /* Do the successors of 'block' have any other predecessors ? - We do not want to do some of the optimizations related to multiRef - if we can reimport blocks */ +unsigned BasicBlock::bbStackDepthOnEntry() const +{ + return (bbEntryState ? bbEntryState->esStackDepth : 0); +} - unsigned multRef = impCanReimport ? unsigned(~0) : 0; +void BasicBlock::bbSetStack(void* stackBuffer) +{ + assert(bbEntryState); + assert(stackBuffer); + bbEntryState->esStack = (StackEntry*)stackBuffer; +} - switch (block->bbJumpKind) - { - case BBJ_COND: +StackEntry* BasicBlock::bbStackOnEntry() const +{ + assert(bbEntryState); + return bbEntryState->esStack; +} - addStmt = impExtractLastStmt(); +void Compiler::verInitCurrentState() +{ + verTrackObjCtorInitState = false; + verCurrentState.thisInitialized = TIS_Bottom; - assert(addStmt->GetRootNode()->gtOper == GT_JTRUE); + // initialize stack info - /* Note if the next block has more than one ancestor */ + verCurrentState.esStackDepth = 0; + assert(verCurrentState.esStack != nullptr); - multRef |= block->bbNext->bbRefs; + // copy current state to entry state of first BB + verInitBBEntryState(fgFirstBB, &verCurrentState); +} - /* Does the next block have temps assigned? */ +Compiler* Compiler::impInlineRoot() +{ + if (impInlineInfo == nullptr) + { + return this; + } + else + { + return impInlineInfo->InlineRoot; + } +} - baseTmp = block->bbNext->bbStkTempsIn; - tgtBlock = block->bbNext; +BYTE Compiler::impSpillCliqueGetMember(SpillCliqueDir predOrSucc, BasicBlock* blk) +{ + if (predOrSucc == SpillCliquePred) + { + return impInlineRoot()->impSpillCliquePredMembers.Get(blk->bbInd()); + } + else + { + assert(predOrSucc == SpillCliqueSucc); + return impInlineRoot()->impSpillCliqueSuccMembers.Get(blk->bbInd()); + } +} - if (baseTmp != NO_BASE_TMP) - { - break; - } +void Compiler::impSpillCliqueSetMember(SpillCliqueDir predOrSucc, BasicBlock* blk, BYTE val) +{ + if (predOrSucc == SpillCliquePred) + { + impInlineRoot()->impSpillCliquePredMembers.Set(blk->bbInd(), val); + } + else + { + assert(predOrSucc == SpillCliqueSucc); + impInlineRoot()->impSpillCliqueSuccMembers.Set(blk->bbInd(), val); + } +} - /* Try the target of the jump then */ +/***************************************************************************** + * + * Convert the instrs ("import") into our internal format (trees). The + * basic flowgraph has already been constructed and is passed in. + */ - multRef |= block->bbJumpDest->bbRefs; - baseTmp = block->bbJumpDest->bbStkTempsIn; - tgtBlock = block->bbJumpDest; - break; +void Compiler::impImport() +{ +#ifdef DEBUG + if (verbose) + { + printf("*************** In impImport() for %s\n", info.compFullName); + } +#endif - case BBJ_ALWAYS: - multRef |= block->bbJumpDest->bbRefs; - baseTmp = block->bbJumpDest->bbStkTempsIn; - tgtBlock = block->bbJumpDest; - break; + Compiler* inlineRoot = impInlineRoot(); - case BBJ_NONE: - multRef |= block->bbNext->bbRefs; - baseTmp = block->bbNext->bbStkTempsIn; - tgtBlock = block->bbNext; - break; + if (info.compMaxStack <= SMALL_STACK_SIZE) + { + impStkSize = SMALL_STACK_SIZE; + } + else + { + impStkSize = info.compMaxStack; + } - case BBJ_SWITCH: - addStmt = impExtractLastStmt(); - assert(addStmt->GetRootNode()->gtOper == GT_SWITCH); + if (this == inlineRoot) + { + // Allocate the stack contents + verCurrentState.esStack = new (this, CMK_ImpStack) StackEntry[impStkSize]; + } + else + { + // This is the inlinee compiler, steal the stack from the inliner compiler + // (after ensuring that it is large enough). + if (inlineRoot->impStkSize < impStkSize) + { + inlineRoot->impStkSize = impStkSize; + inlineRoot->verCurrentState.esStack = new (this, CMK_ImpStack) StackEntry[impStkSize]; + } - for (BasicBlock* const tgtBlock : block->SwitchTargets()) - { - multRef |= tgtBlock->bbRefs; + verCurrentState.esStack = inlineRoot->verCurrentState.esStack; + } - // Thanks to spill cliques, we should have assigned all or none - assert((baseTmp == NO_BASE_TMP) || (baseTmp == tgtBlock->bbStkTempsIn)); - baseTmp = tgtBlock->bbStkTempsIn; - if (multRef > 1) - { - break; - } - } - break; + // initialize the entry state at start of method + verInitCurrentState(); - case BBJ_CALLFINALLY: - case BBJ_EHCATCHRET: - case BBJ_RETURN: - case BBJ_EHFINALLYRET: - case BBJ_EHFILTERRET: - case BBJ_THROW: - BADCODE("can't have 'unreached' end of BB with non-empty stack"); - break; + // Initialize stuff related to figuring "spill cliques" (see spec comment for impGetSpillTmpBase). + if (this == inlineRoot) // These are only used on the root of the inlining tree. + { + // We have initialized these previously, but to size 0. Make them larger. + impPendingBlockMembers.Init(getAllocator(), fgBBNumMax * 2); + impSpillCliquePredMembers.Init(getAllocator(), fgBBNumMax * 2); + impSpillCliqueSuccMembers.Init(getAllocator(), fgBBNumMax * 2); + } + inlineRoot->impPendingBlockMembers.Reset(fgBBNumMax * 2); + inlineRoot->impSpillCliquePredMembers.Reset(fgBBNumMax * 2); + inlineRoot->impSpillCliqueSuccMembers.Reset(fgBBNumMax * 2); + impBlockListNodeFreeList = nullptr; - default: - noway_assert(!"Unexpected bbJumpKind"); - break; - } +#ifdef DEBUG + impLastILoffsStmt = nullptr; + impNestedStackSpill = false; +#endif + impBoxTemp = BAD_VAR_NUM; - assert(multRef >= 1); + impPendingList = impPendingFree = nullptr; - /* Do we have a base temp number? */ + // Skip leading internal blocks. + // These can arise from needing a leading scratch BB, from EH normalization, and from OSR entry redirects. + // + BasicBlock* entryBlock = fgFirstBB; - bool newTemps = (baseTmp == NO_BASE_TMP); + while (entryBlock->bbFlags & BBF_INTERNAL) + { + JITDUMP("Marking leading BBF_INTERNAL block " FMT_BB " as BBF_IMPORTED\n", entryBlock->bbNum); + entryBlock->bbFlags |= BBF_IMPORTED; - if (newTemps) + if (entryBlock->bbJumpKind == BBJ_NONE) { - /* Grab enough temps for the whole stack */ - baseTmp = impGetSpillTmpBase(block); + entryBlock = entryBlock->bbNext; } - - /* Spill all stack entries into temps */ - unsigned level, tempNum; - - JITDUMP("\nSpilling stack entries into temps\n"); - for (level = 0, tempNum = baseTmp; level < verCurrentState.esStackDepth; level++, tempNum++) + else if (opts.IsOSR() && (entryBlock->bbJumpKind == BBJ_ALWAYS)) { - GenTree* tree = verCurrentState.esStack[level].val; - - /* VC generates code where it pushes a byref from one branch, and an int (ldc.i4 0) from - the other. This should merge to a byref in unverifiable code. - However, if the branch which leaves the TYP_I_IMPL on the stack is imported first, the - successor would be imported assuming there was a TYP_I_IMPL on - the stack. Thus the value would not get GC-tracked. Hence, - change the temp to TYP_BYREF and reimport the successors. - Note: We should only allow this in unverifiable code. - */ - if (tree->gtType == TYP_BYREF && lvaTable[tempNum].lvType == TYP_I_IMPL) - { - lvaTable[tempNum].lvType = TYP_BYREF; - impReimportMarkSuccessors(block); - markImport = true; - } + entryBlock = entryBlock->bbJumpDest; + } + else + { + assert(!"unexpected bbJumpKind in entry sequence"); + } + } -#ifdef TARGET_64BIT - if (genActualType(tree->gtType) == TYP_I_IMPL && lvaTable[tempNum].lvType == TYP_INT) - { - // Some other block in the spill clique set this to "int", but now we have "native int". - // Change the type and go back to re-import any blocks that used the wrong type. - lvaTable[tempNum].lvType = TYP_I_IMPL; - reimportSpillClique = true; - } - else if (genActualType(tree->gtType) == TYP_INT && lvaTable[tempNum].lvType == TYP_I_IMPL) - { - // Spill clique has decided this should be "native int", but this block only pushes an "int". - // Insert a sign-extension to "native int" so we match the clique. - verCurrentState.esStack[level].val = gtNewCastNode(TYP_I_IMPL, tree, false, TYP_I_IMPL); - } + // Note for OSR we'd like to be able to verify this block must be + // stack empty, but won't know that until we've imported...so instead + // we'll BADCODE out if we mess up. + // + // (the concern here is that the runtime asks us to OSR a + // different IL version than the one that matched the method that + // triggered OSR). This should not happen but I might have the + // IL versioning stuff wrong. + // + // TODO: we also currently expect this block to be a join point, + // which we should verify over when we find jump targets. + impImportBlockPending(entryBlock); - // Consider the case where one branch left a 'byref' on the stack and the other leaves - // an 'int'. On 32-bit, this is allowed (in non-verifiable code) since they are the same - // size. JIT64 managed to make this work on 64-bit. For compatibility, we support JIT64 - // behavior instead of asserting and then generating bad code (where we save/restore the - // low 32 bits of a byref pointer to an 'int' sized local). If the 'int' side has been - // imported already, we need to change the type of the local and reimport the spill clique. - // If the 'byref' side has imported, we insert a cast from int to 'native int' to match - // the 'byref' size. - if (genActualType(tree->gtType) == TYP_BYREF && lvaTable[tempNum].lvType == TYP_INT) - { - // Some other block in the spill clique set this to "int", but now we have "byref". - // Change the type and go back to re-import any blocks that used the wrong type. - lvaTable[tempNum].lvType = TYP_BYREF; - reimportSpillClique = true; - } - else if (genActualType(tree->gtType) == TYP_INT && lvaTable[tempNum].lvType == TYP_BYREF) - { - // Spill clique has decided this should be "byref", but this block only pushes an "int". - // Insert a sign-extension to "native int" so we match the clique size. - verCurrentState.esStack[level].val = gtNewCastNode(TYP_I_IMPL, tree, false, TYP_I_IMPL); - } + /* Import blocks in the worker-list until there are no more */ -#endif // TARGET_64BIT + while (impPendingList) + { + /* Remove the entry at the front of the list */ - if (tree->gtType == TYP_DOUBLE && lvaTable[tempNum].lvType == TYP_FLOAT) - { - // Some other block in the spill clique set this to "float", but now we have "double". - // Change the type and go back to re-import any blocks that used the wrong type. - lvaTable[tempNum].lvType = TYP_DOUBLE; - reimportSpillClique = true; - } - else if (tree->gtType == TYP_FLOAT && lvaTable[tempNum].lvType == TYP_DOUBLE) - { - // Spill clique has decided this should be "double", but this block only pushes a "float". - // Insert a cast to "double" so we match the clique. - verCurrentState.esStack[level].val = gtNewCastNode(TYP_DOUBLE, tree, false, TYP_DOUBLE); - } + PendingDsc* dsc = impPendingList; + impPendingList = impPendingList->pdNext; + impSetPendingBlockMember(dsc->pdBB, 0); - /* If addStmt has a reference to tempNum (can only happen if we - are spilling to the temps already used by a previous block), - we need to spill addStmt */ + /* Restore the stack state */ - if (addStmt != nullptr && !newTemps && gtHasRef(addStmt->GetRootNode(), tempNum)) - { - GenTree* addTree = addStmt->GetRootNode(); + verCurrentState.thisInitialized = dsc->pdThisPtrInit; + verCurrentState.esStackDepth = dsc->pdSavedStack.ssDepth; + if (verCurrentState.esStackDepth) + { + impRestoreStackState(&dsc->pdSavedStack); + } - if (addTree->gtOper == GT_JTRUE) - { - GenTree* relOp = addTree->AsOp()->gtOp1; - assert(relOp->OperIsCompare()); + /* Add the entry to the free list for reuse */ - var_types type = genActualType(relOp->AsOp()->gtOp1->TypeGet()); + dsc->pdNext = impPendingFree; + impPendingFree = dsc; - if (gtHasRef(relOp->AsOp()->gtOp1, tempNum)) - { - unsigned temp = lvaGrabTemp(true DEBUGARG("spill addStmt JTRUE ref Op1")); - impAssignTempGen(temp, relOp->AsOp()->gtOp1, level); - type = genActualType(lvaTable[temp].TypeGet()); - relOp->AsOp()->gtOp1 = gtNewLclvNode(temp, type); - } + /* Now import the block */ - if (gtHasRef(relOp->AsOp()->gtOp2, tempNum)) - { - unsigned temp = lvaGrabTemp(true DEBUGARG("spill addStmt JTRUE ref Op2")); - impAssignTempGen(temp, relOp->AsOp()->gtOp2, level); - type = genActualType(lvaTable[temp].TypeGet()); - relOp->AsOp()->gtOp2 = gtNewLclvNode(temp, type); - } - } - else - { - assert(addTree->gtOper == GT_SWITCH && genActualTypeIsIntOrI(addTree->AsOp()->gtOp1->TypeGet())); + if (dsc->pdBB->bbFlags & BBF_FAILED_VERIFICATION) + { + verConvertBBToThrowVerificationException(dsc->pdBB DEBUGARG(true)); + impEndTreeList(dsc->pdBB); + } + else + { + impImportBlock(dsc->pdBB); - unsigned temp = lvaGrabTemp(true DEBUGARG("spill addStmt SWITCH")); - impAssignTempGen(temp, addTree->AsOp()->gtOp1, level); - addTree->AsOp()->gtOp1 = gtNewLclvNode(temp, genActualType(addTree->AsOp()->gtOp1->TypeGet())); - } + if (compDonotInline()) + { + return; } + if (compIsForImportOnly()) + { + return; + } + } + } - /* Spill the stack entry, and replace with the temp */ - - if (!impSpillStackEntry(level, tempNum #ifdef DEBUG - , - true, "Spill Stack Entry" + if (verbose && info.compXcptnsCount) + { + printf("\nAfter impImport() added block for try,catch,finally"); + fgDispBasicBlocks(); + printf("\n"); + } + + // Used in impImportBlockPending() for STRESS_CHK_REIMPORT + for (BasicBlock* const block : Blocks()) + { + block->bbFlags &= ~BBF_VISITED; + } #endif - )) - { - if (markImport) - { - BADCODE("bad stack state"); - } +} - // Oops. Something went wrong when spilling. Bad code. - verHandleVerificationFailure(block DEBUGARG(true)); +// Checks if a typeinfo (usually stored in the type stack) is a struct. +// The invariant here is that if it's not a ref or a method and has a class handle +// it's a valuetype +bool Compiler::impIsValueType(typeInfo* pTypeInfo) +{ + if (pTypeInfo && pTypeInfo->IsValueClassWithClsHnd()) + { + return true; + } + else + { + return false; + } +} + +//------------------------------------------------------------------------ +// impIsInvariant: check if a tree (created during import) is invariant. +// +// Arguments: +// tree -- The tree +// +// Returns: +// true if it is invariant +// +// Remarks: +// This is a variant of GenTree::IsInvariant that is more suitable for use +// during import. Unlike that function, this one handles GT_FIELD nodes. +// +bool Compiler::impIsInvariant(const GenTree* tree) +{ + return tree->OperIsConst() || impIsAddressInLocal(tree); +} + +//------------------------------------------------------------------------ +// impIsAddressInLocal: +// Check to see if the tree is the address of a local or +// the address of a field in a local. +// Arguments: +// tree -- The tree +// lclVarTreeOut -- [out] the local that this points into +// +// Returns: +// true if it points into a local +// +// Remarks: +// This is a variant of GenTree::IsLocalAddrExpr that is more suitable for +// use during import. Unlike that function, this one handles GT_FIELD nodes. +// +bool Compiler::impIsAddressInLocal(const GenTree* tree, GenTree** lclVarTreeOut) +{ + if (tree->gtOper != GT_ADDR) + { + return false; + } - goto SPILLSTACK; - } + GenTree* op = tree->AsOp()->gtOp1; + while (op->gtOper == GT_FIELD) + { + op = op->AsField()->GetFldObj(); + if (op && op->gtOper == GT_ADDR) // Skip static fields where op will be NULL. + { + op = op->AsOp()->gtOp1; } + else + { + return false; + } + } - /* Put back the 'jtrue'/'switch' if we removed it earlier */ - - if (addStmt != nullptr) + if (op->gtOper == GT_LCL_VAR) + { + if (lclVarTreeOut != nullptr) { - impAppendStmt(addStmt, CHECK_SPILL_NONE); + *lclVarTreeOut = op; } + return true; + } + else + { + return false; } +} - // Some of the append/spill logic works on compCurBB +//------------------------------------------------------------------------ +// impMakeDiscretionaryInlineObservations: make observations that help +// determine the profitability of a discretionary inline +// +// Arguments: +// pInlineInfo -- InlineInfo for the inline, or null for the prejit root +// inlineResult -- InlineResult accumulating information about this inline +// +// Notes: +// If inlining or prejitting the root, this method also makes +// various observations about the method that factor into inline +// decisions. It sets `compNativeSizeEstimate` as a side effect. - assert(compCurBB == block); +void Compiler::impMakeDiscretionaryInlineObservations(InlineInfo* pInlineInfo, InlineResult* inlineResult) +{ + assert((pInlineInfo != nullptr && compIsForInlining()) || // Perform the actual inlining. + (pInlineInfo == nullptr && !compIsForInlining()) // Calculate the static inlining hint for ngen. + ); - /* Save the tree list in the block */ - impEndTreeList(block); + // If we're really inlining, we should just have one result in play. + assert((pInlineInfo == nullptr) || (inlineResult == pInlineInfo->inlineResult)); - // impEndTreeList sets BBF_IMPORTED on the block - // We do *NOT* want to set it later than this because - // impReimportSpillClique might clear it if this block is both a - // predecessor and successor in the current spill clique - assert(block->bbFlags & BBF_IMPORTED); + // If this is a "forceinline" method, the JIT probably shouldn't have gone + // to the trouble of estimating the native code size. Even if it did, it + // shouldn't be relying on the result of this method. + assert(inlineResult->GetObservation() == InlineObservation::CALLEE_IS_DISCRETIONARY_INLINE); - // If we had a int/native int, or float/double collision, we need to re-import - if (reimportSpillClique) + // Note if the caller contains NEWOBJ or NEWARR. + Compiler* rootCompiler = impInlineRoot(); + + if ((rootCompiler->optMethodFlags & OMF_HAS_NEWARRAY) != 0) { - // This will re-import all the successors of block (as well as each of their predecessors) - impReimportSpillClique(block); + inlineResult->Note(InlineObservation::CALLER_HAS_NEWARRAY); + } - // For blocks that haven't been imported yet, we still need to mark them as pending import. - for (BasicBlock* const succ : block->Succs()) + if ((rootCompiler->optMethodFlags & OMF_HAS_NEWOBJ) != 0) + { + inlineResult->Note(InlineObservation::CALLER_HAS_NEWOBJ); + } + + bool calleeIsStatic = (info.compFlags & CORINFO_FLG_STATIC) != 0; + bool isSpecialMethod = (info.compFlags & CORINFO_FLG_CONSTRUCTOR) != 0; + + if (isSpecialMethod) + { + if (calleeIsStatic) { - if ((succ->bbFlags & BBF_IMPORTED) == 0) - { - impImportBlockPending(succ); - } + inlineResult->Note(InlineObservation::CALLEE_IS_CLASS_CTOR); + } + else + { + inlineResult->Note(InlineObservation::CALLEE_IS_INSTANCE_CTOR); } } - else // the normal case + else if (!calleeIsStatic) { - // otherwise just import the successors of block - - /* Does this block jump to any other blocks? */ - for (BasicBlock* const succ : block->Succs()) + // Callee is an instance method. + // + // Check if the callee has the same 'this' as the root. + if (pInlineInfo != nullptr) { - impImportBlockPending(succ); + GenTree* thisArg = pInlineInfo->iciCall->AsCall()->gtArgs.GetThisArg()->GetNode(); + assert(thisArg); + bool isSameThis = impIsThis(thisArg); + inlineResult->NoteBool(InlineObservation::CALLSITE_IS_SAME_THIS, isSameThis); } } -} -#ifdef _PREFAST_ -#pragma warning(pop) -#endif -/*****************************************************************************/ -// -// Ensures that "block" is a member of the list of BBs waiting to be imported, pushing it on the list if -// necessary (and ensures that it is a member of the set of BB's on the list, by setting its byte in -// impPendingBlockMembers). Merges the current verification state into the verification state of "block" -// (its "pre-state"). + bool callsiteIsGeneric = (rootCompiler->info.compMethodInfo->args.sigInst.methInstCount != 0) || + (rootCompiler->info.compMethodInfo->args.sigInst.classInstCount != 0); -void Compiler::impImportBlockPending(BasicBlock* block) -{ -#ifdef DEBUG - if (verbose) + bool calleeIsGeneric = (info.compMethodInfo->args.sigInst.methInstCount != 0) || + (info.compMethodInfo->args.sigInst.classInstCount != 0); + + if (!callsiteIsGeneric && calleeIsGeneric) { - printf("\nimpImportBlockPending for " FMT_BB "\n", block->bbNum); + inlineResult->Note(InlineObservation::CALLSITE_NONGENERIC_CALLS_GENERIC); } -#endif - // We will add a block to the pending set if it has not already been imported (or needs to be re-imported), - // or if it has, but merging in a predecessor's post-state changes the block's pre-state. - // (When we're doing verification, we always attempt the merge to detect verification errors.) + // Inspect callee's arguments (and the actual values at the callsite for them) + CORINFO_SIG_INFO sig = info.compMethodInfo->args; + CORINFO_ARG_LIST_HANDLE sigArg = sig.args; - // If the block has not been imported, add to pending set. - bool addToPending = ((block->bbFlags & BBF_IMPORTED) == 0); + CallArg* argUse = pInlineInfo == nullptr ? nullptr : &*pInlineInfo->iciCall->AsCall()->gtArgs.Args().begin(); - // Initialize bbEntryState just the first time we try to add this block to the pending list - // Just because bbEntryState is NULL, doesn't mean the pre-state wasn't previously set - // We use NULL to indicate the 'common' state to avoid memory allocation - if ((block->bbEntryState == nullptr) && ((block->bbFlags & (BBF_IMPORTED | BBF_FAILED_VERIFICATION)) == 0) && - (impGetPendingBlockMember(block) == 0)) - { - verInitBBEntryState(block, &verCurrentState); - assert(block->bbStkDepth == 0); - block->bbStkDepth = static_cast(verCurrentState.esStackDepth); - assert(addToPending); - assert(impGetPendingBlockMember(block) == 0); - } - else + for (unsigned i = 0; i < info.compMethodInfo->args.numArgs; i++) { - // The stack should have the same height on entry to the block from all its predecessors. - if (block->bbStkDepth != verCurrentState.esStackDepth) + if ((argUse != nullptr) && (argUse->GetWellKnownArg() == WellKnownArg::ThisPointer)) { -#ifdef DEBUG - char buffer[400]; - sprintf_s(buffer, sizeof(buffer), - "Block at offset %4.4x to %4.4x in %0.200s entered with different stack depths.\n" - "Previous depth was %d, current depth is %d", - block->bbCodeOffs, block->bbCodeOffsEnd, info.compFullName, block->bbStkDepth, - verCurrentState.esStackDepth); - buffer[400 - 1] = 0; - NO_WAY(buffer); -#else - NO_WAY("Block entered with different stack depths"); -#endif + argUse = argUse->GetNext(); } - if (!addToPending) + CORINFO_CLASS_HANDLE sigClass; + CorInfoType corType = strip(info.compCompHnd->getArgType(&sig, sigArg, &sigClass)); + GenTree* argNode = argUse == nullptr ? nullptr : argUse->GetEarlyNode(); + + if (corType == CORINFO_TYPE_CLASS) { - return; + sigClass = info.compCompHnd->getArgClass(&sig, sigArg); + } + else if (corType == CORINFO_TYPE_VALUECLASS) + { + inlineResult->Note(InlineObservation::CALLEE_ARG_STRUCT); + } + else if (corType == CORINFO_TYPE_BYREF) + { + sigClass = info.compCompHnd->getArgClass(&sig, sigArg); + corType = info.compCompHnd->getChildType(sigClass, &sigClass); } - if (block->bbStkDepth > 0) + if (argNode != nullptr) { - // We need to fix the types of any spill temps that might have changed: - // int->native int, float->double, int->byref, etc. - impRetypeEntryStateTemps(block); + bool isExact = false; + bool isNonNull = false; + CORINFO_CLASS_HANDLE argCls = gtGetClassHandle(argNode, &isExact, &isNonNull); + if (argCls != nullptr) + { + const bool isArgValueType = eeIsValueClass(argCls); + // Exact class of the arg is known + if (isExact && !isArgValueType) + { + inlineResult->Note(InlineObservation::CALLSITE_ARG_EXACT_CLS); + if ((argCls != sigClass) && (sigClass != nullptr)) + { + // .. but the signature accepts a less concrete type. + inlineResult->Note(InlineObservation::CALLSITE_ARG_EXACT_CLS_SIG_IS_NOT); + } + } + // Arg is a reference type in the signature and a boxed value type was passed. + else if (isArgValueType && (corType == CORINFO_TYPE_CLASS)) + { + inlineResult->Note(InlineObservation::CALLSITE_ARG_BOXED); + } + } + + if (argNode->OperIsConst()) + { + inlineResult->Note(InlineObservation::CALLSITE_ARG_CONST); + } + argUse = argUse->GetNext(); } + sigArg = info.compCompHnd->getArgNext(sigArg); + } - // OK, we must add to the pending list, if it's not already in it. - if (impGetPendingBlockMember(block) != 0) + // Note if the callee's return type is a value type + if (info.compMethodInfo->args.retType == CORINFO_TYPE_VALUECLASS) + { + inlineResult->Note(InlineObservation::CALLEE_RETURNS_STRUCT); + } + + // Note if the callee's class is a promotable struct + if ((info.compClassAttr & CORINFO_FLG_VALUECLASS) != 0) + { + assert(structPromotionHelper != nullptr); + if (structPromotionHelper->CanPromoteStructType(info.compClassHnd)) { - return; + inlineResult->Note(InlineObservation::CALLEE_CLASS_PROMOTABLE); } + inlineResult->Note(InlineObservation::CALLEE_CLASS_VALUETYPE); + } + +#ifdef FEATURE_SIMD + + // Note if this method is has SIMD args or return value + if (pInlineInfo != nullptr && pInlineInfo->hasSIMDTypeArgLocalOrReturn) + { + inlineResult->Note(InlineObservation::CALLEE_HAS_SIMD); + } + +#endif // FEATURE_SIMD + + // Roughly classify callsite frequency. + InlineCallsiteFrequency frequency = InlineCallsiteFrequency::UNUSED; + + // If this is a prejit root, or a maximally hot block... + if ((pInlineInfo == nullptr) || (pInlineInfo->iciBlock->isMaxBBWeight())) + { + frequency = InlineCallsiteFrequency::HOT; + } + // No training data. Look for loop-like things. + // We consider a recursive call loop-like. Do not give the inlining boost to the method itself. + // However, give it to things nearby. + else if ((pInlineInfo->iciBlock->bbFlags & BBF_BACKWARD_JUMP) && + (pInlineInfo->fncHandle != pInlineInfo->inlineCandidateInfo->ilCallerHandle)) + { + frequency = InlineCallsiteFrequency::LOOP; } - - // Get an entry to add to the pending list - - PendingDsc* dsc; - - if (impPendingFree) + else if (pInlineInfo->iciBlock->hasProfileWeight() && (pInlineInfo->iciBlock->bbWeight > BB_ZERO_WEIGHT)) { - // We can reuse one of the freed up dscs. - dsc = impPendingFree; - impPendingFree = dsc->pdNext; + frequency = InlineCallsiteFrequency::WARM; + } + // Now modify the multiplier based on where we're called from. + else if (pInlineInfo->iciBlock->isRunRarely() || ((info.compFlags & FLG_CCTOR) == FLG_CCTOR)) + { + frequency = InlineCallsiteFrequency::RARE; } else { - // We have to create a new dsc - dsc = new (this, CMK_Unknown) PendingDsc; + frequency = InlineCallsiteFrequency::BORING; } - dsc->pdBB = block; - dsc->pdSavedStack.ssDepth = verCurrentState.esStackDepth; - - // Save the stack trees for later + // Also capture the block weight of the call site. + // + // In the prejit root case, assume at runtime there might be a hot call site + // for this method, so we won't prematurely conclude this method should never + // be inlined. + // + weight_t weight = 0; - if (verCurrentState.esStackDepth) + if (pInlineInfo != nullptr) { - impSaveStackState(&dsc->pdSavedStack, false); + weight = pInlineInfo->iciBlock->bbWeight; + } + else + { + const weight_t prejitHotCallerWeight = 1000000.0; + weight = prejitHotCallerWeight; } - // Add the entry to the pending list + inlineResult->NoteInt(InlineObservation::CALLSITE_FREQUENCY, static_cast(frequency)); + inlineResult->NoteInt(InlineObservation::CALLSITE_WEIGHT, (int)(weight)); - dsc->pdNext = impPendingList; - impPendingList = dsc; - impSetPendingBlockMember(block, 1); // And indicate that it's now a member of the set. + bool hasProfile = false; + double profileFreq = 0.0; - // Various assertions require us to now to consider the block as not imported (at least for - // the final time...) - block->bbFlags &= ~BBF_IMPORTED; + // If the call site has profile data, report the relative frequency of the site. + // + if ((pInlineInfo != nullptr) && rootCompiler->fgHaveSufficientProfileData()) + { + const weight_t callSiteWeight = pInlineInfo->iciBlock->bbWeight; + const weight_t entryWeight = rootCompiler->fgFirstBB->bbWeight; + profileFreq = fgProfileWeightsEqual(entryWeight, 0.0) ? 0.0 : callSiteWeight / entryWeight; + hasProfile = true; -#ifdef DEBUG - if (verbose && 0) + assert(callSiteWeight >= 0); + assert(entryWeight >= 0); + } + else if (pInlineInfo == nullptr) { - printf("Added PendingDsc - %08p for " FMT_BB "\n", dspPtr(dsc), block->bbNum); + // Simulate a hot callsite for PrejitRoot mode. + hasProfile = true; + profileFreq = 1.0; } -#endif + + inlineResult->NoteBool(InlineObservation::CALLSITE_HAS_PROFILE, hasProfile); + inlineResult->NoteDouble(InlineObservation::CALLSITE_PROFILE_FREQUENCY, profileFreq); } -/*****************************************************************************/ -// -// Ensures that "block" is a member of the list of BBs waiting to be imported, pushing it on the list if -// necessary (and ensures that it is a member of the set of BB's on the list, by setting its byte in -// impPendingBlockMembers). Does *NOT* change the existing "pre-state" of the block. +/***************************************************************************** + This method makes STATIC inlining decision based on the IL code. + It should not make any inlining decision based on the context. + If forceInline is true, then the inlining decision should not depend on + performance heuristics (code size, etc.). + */ -void Compiler::impReimportBlockPending(BasicBlock* block) +void Compiler::impCanInlineIL(CORINFO_METHOD_HANDLE fncHandle, + CORINFO_METHOD_INFO* methInfo, + bool forceInline, + InlineResult* inlineResult) { - JITDUMP("\nimpReimportBlockPending for " FMT_BB, block->bbNum); + unsigned codeSize = methInfo->ILCodeSize; - assert(block->bbFlags & BBF_IMPORTED); + // We shouldn't have made up our minds yet... + assert(!inlineResult->IsDecided()); - // OK, we must add to the pending list, if it's not already in it. - if (impGetPendingBlockMember(block) != 0) + if (methInfo->EHcount) { + inlineResult->NoteFatal(InlineObservation::CALLEE_HAS_EH); return; } - // Get an entry to add to the pending list + if ((methInfo->ILCode == nullptr) || (codeSize == 0)) + { + inlineResult->NoteFatal(InlineObservation::CALLEE_HAS_NO_BODY); + return; + } - PendingDsc* dsc; + // For now we don't inline varargs (import code can't handle it) - if (impPendingFree) + if (methInfo->args.isVarArg()) { - // We can reuse one of the freed up dscs. - dsc = impPendingFree; - impPendingFree = dsc->pdNext; + inlineResult->NoteFatal(InlineObservation::CALLEE_HAS_MANAGED_VARARGS); + return; } - else + + // Reject if it has too many locals. + // This is currently an implementation limit due to fixed-size arrays in the + // inline info, rather than a performance heuristic. + + inlineResult->NoteInt(InlineObservation::CALLEE_NUMBER_OF_LOCALS, methInfo->locals.numArgs); + + if (methInfo->locals.numArgs > MAX_INL_LCLS) { - // We have to create a new dsc - dsc = new (this, CMK_ImpStack) PendingDsc; + inlineResult->NoteFatal(InlineObservation::CALLEE_TOO_MANY_LOCALS); + return; } - dsc->pdBB = block; + // Make sure there aren't too many arguments. + // This is currently an implementation limit due to fixed-size arrays in the + // inline info, rather than a performance heuristic. - if (block->bbEntryState) + inlineResult->NoteInt(InlineObservation::CALLEE_NUMBER_OF_ARGUMENTS, methInfo->args.numArgs); + + if (methInfo->args.numArgs > MAX_INL_ARGS) { - dsc->pdSavedStack.ssDepth = block->bbEntryState->esStackDepth; - dsc->pdSavedStack.ssTrees = block->bbEntryState->esStack; + inlineResult->NoteFatal(InlineObservation::CALLEE_TOO_MANY_ARGUMENTS); + return; } - else + + // Note force inline state + + inlineResult->NoteBool(InlineObservation::CALLEE_IS_FORCE_INLINE, forceInline); + + // Note IL code size + + inlineResult->NoteInt(InlineObservation::CALLEE_IL_CODE_SIZE, codeSize); + + if (inlineResult->IsFailure()) { - dsc->pdSavedStack.ssDepth = 0; - dsc->pdSavedStack.ssTrees = nullptr; + return; } - // Add the entry to the pending list - - dsc->pdNext = impPendingList; - impPendingList = dsc; - impSetPendingBlockMember(block, 1); // And indicate that it's now a member of the set. + // Make sure maxstack is not too big - // Various assertions require us to now to consider the block as not imported (at least for - // the final time...) - block->bbFlags &= ~BBF_IMPORTED; + inlineResult->NoteInt(InlineObservation::CALLEE_MAXSTACK, methInfo->maxStack); -#ifdef DEBUG - if (verbose && 0) + if (inlineResult->IsFailure()) { - printf("Added PendingDsc - %08p for " FMT_BB "\n", dspPtr(dsc), block->bbNum); + return; } -#endif } -void* Compiler::BlockListNode::operator new(size_t sz, Compiler* comp) +/***************************************************************************** + */ + +void Compiler::impCheckCanInline(GenTreeCall* call, + CORINFO_METHOD_HANDLE fncHandle, + unsigned methAttr, + CORINFO_CONTEXT_HANDLE exactContextHnd, + InlineCandidateInfo** ppInlineCandidateInfo, + InlineResult* inlineResult) { - if (comp->impBlockListNodeFreeList == nullptr) - { - return comp->getAllocator(CMK_BasicBlock).allocate(1); - } - else + // Either EE or JIT might throw exceptions below. + // If that happens, just don't inline the method. + + struct Param + { + Compiler* pThis; + GenTreeCall* call; + CORINFO_METHOD_HANDLE fncHandle; + unsigned methAttr; + CORINFO_CONTEXT_HANDLE exactContextHnd; + InlineResult* result; + InlineCandidateInfo** ppInlineCandidateInfo; + } param; + memset(¶m, 0, sizeof(param)); + + param.pThis = this; + param.call = call; + param.fncHandle = fncHandle; + param.methAttr = methAttr; + param.exactContextHnd = (exactContextHnd != nullptr) ? exactContextHnd : MAKE_METHODCONTEXT(fncHandle); + param.result = inlineResult; + param.ppInlineCandidateInfo = ppInlineCandidateInfo; + + bool success = eeRunWithErrorTrap( + [](Param* pParam) { + CorInfoInitClassResult initClassResult; + +#ifdef DEBUG + const char* methodName; + const char* className; + methodName = pParam->pThis->eeGetMethodName(pParam->fncHandle, &className); + + if (JitConfig.JitNoInline()) + { + pParam->result->NoteFatal(InlineObservation::CALLEE_IS_JIT_NOINLINE); + goto _exit; + } +#endif + + /* Try to get the code address/size for the method */ + + CORINFO_METHOD_INFO methInfo; + if (!pParam->pThis->info.compCompHnd->getMethodInfo(pParam->fncHandle, &methInfo)) + { + pParam->result->NoteFatal(InlineObservation::CALLEE_NO_METHOD_INFO); + goto _exit; + } + + // Profile data allows us to avoid early "too many IL bytes" outs. + pParam->result->NoteBool(InlineObservation::CALLSITE_HAS_PROFILE, + pParam->pThis->fgHaveSufficientProfileData()); + + bool forceInline; + forceInline = !!(pParam->methAttr & CORINFO_FLG_FORCEINLINE); + + pParam->pThis->impCanInlineIL(pParam->fncHandle, &methInfo, forceInline, pParam->result); + + if (pParam->result->IsFailure()) + { + assert(pParam->result->IsNever()); + goto _exit; + } + + // Speculatively check if initClass() can be done. + // If it can be done, we will try to inline the method. + initClassResult = + pParam->pThis->info.compCompHnd->initClass(nullptr /* field */, pParam->fncHandle /* method */, + pParam->exactContextHnd /* context */); + + if (initClassResult & CORINFO_INITCLASS_DONT_INLINE) + { + pParam->result->NoteFatal(InlineObservation::CALLSITE_CANT_CLASS_INIT); + goto _exit; + } + + // Given the EE the final say in whether to inline or not. + // This should be last since for verifiable code, this can be expensive + + /* VM Inline check also ensures that the method is verifiable if needed */ + CorInfoInline vmResult; + vmResult = pParam->pThis->info.compCompHnd->canInline(pParam->pThis->info.compMethodHnd, pParam->fncHandle); + + if (vmResult == INLINE_FAIL) + { + pParam->result->NoteFatal(InlineObservation::CALLSITE_IS_VM_NOINLINE); + } + else if (vmResult == INLINE_NEVER) + { + pParam->result->NoteFatal(InlineObservation::CALLEE_IS_VM_NOINLINE); + } + + if (pParam->result->IsFailure()) + { + // Do not report this as a failure. Instead report as a "VM failure" + pParam->result->SetVMFailure(); + goto _exit; + } + + /* Get the method properties */ + + CORINFO_CLASS_HANDLE clsHandle; + clsHandle = pParam->pThis->info.compCompHnd->getMethodClass(pParam->fncHandle); + unsigned clsAttr; + clsAttr = pParam->pThis->info.compCompHnd->getClassAttribs(clsHandle); + + /* Get the return type */ + + var_types fncRetType; + fncRetType = pParam->call->TypeGet(); + +#ifdef DEBUG + var_types fncRealRetType; + fncRealRetType = JITtype2varType(methInfo.args.retType); + + assert((genActualType(fncRealRetType) == genActualType(fncRetType)) || + // VSW 288602 + // In case of IJW, we allow to assign a native pointer to a BYREF. + (fncRetType == TYP_BYREF && methInfo.args.retType == CORINFO_TYPE_PTR) || + (varTypeIsStruct(fncRetType) && (fncRealRetType == TYP_STRUCT))); +#endif + + // Allocate an InlineCandidateInfo structure, + // + // Or, reuse the existing GuardedDevirtualizationCandidateInfo, + // which was pre-allocated to have extra room. + // + InlineCandidateInfo* pInfo; + + if (pParam->call->IsGuardedDevirtualizationCandidate()) + { + pInfo = pParam->call->gtInlineCandidateInfo; + } + else + { + pInfo = new (pParam->pThis, CMK_Inlining) InlineCandidateInfo; + + // Null out bits we don't use when we're just inlining + pInfo->guardedClassHandle = nullptr; + pInfo->guardedMethodHandle = nullptr; + pInfo->guardedMethodUnboxedEntryHandle = nullptr; + pInfo->likelihood = 0; + pInfo->requiresInstMethodTableArg = false; + } + + pInfo->methInfo = methInfo; + pInfo->ilCallerHandle = pParam->pThis->info.compMethodHnd; + pInfo->clsHandle = clsHandle; + pInfo->exactContextHnd = pParam->exactContextHnd; + pInfo->retExpr = nullptr; + pInfo->preexistingSpillTemp = BAD_VAR_NUM; + pInfo->clsAttr = clsAttr; + pInfo->methAttr = pParam->methAttr; + pInfo->initClassResult = initClassResult; + pInfo->fncRetType = fncRetType; + pInfo->exactContextNeedsRuntimeLookup = false; + pInfo->inlinersContext = pParam->pThis->compInlineContext; + + // Note exactContextNeedsRuntimeLookup is reset later on, + // over in impMarkInlineCandidate. + + *(pParam->ppInlineCandidateInfo) = pInfo; + + _exit:; + }, + ¶m); + if (!success) { - BlockListNode* res = comp->impBlockListNodeFreeList; - comp->impBlockListNodeFreeList = res->m_next; - return res; + param.result->NoteFatal(InlineObservation::CALLSITE_COMPILATION_ERROR); } } -void Compiler::FreeBlockListNode(Compiler::BlockListNode* node) +//------------------------------------------------------------------------ +// impInlineRecordArgInfo: record information about an inline candidate argument +// +// Arguments: +// pInlineInfo - inline info for the inline candidate +// arg - the caller argument +// argNum - logical index of this argument +// inlineResult - result of ongoing inline evaluation +// +// Notes: +// +// Checks for various inline blocking conditions and makes notes in +// the inline info arg table about the properties of the actual. These +// properties are used later by impInlineFetchArg to determine how best to +// pass the argument into the inlinee. + +void Compiler::impInlineRecordArgInfo(InlineInfo* pInlineInfo, + CallArg* arg, + unsigned argNum, + InlineResult* inlineResult) { - node->m_next = impBlockListNodeFreeList; - impBlockListNodeFreeList = node; -} + InlArgInfo* inlCurArgInfo = &pInlineInfo->inlArgInfo[argNum]; + + inlCurArgInfo->arg = arg; + GenTree* curArgVal = arg->GetNode(); -void Compiler::impWalkSpillCliqueFromPred(BasicBlock* block, SpillCliqueWalker* callback) -{ - bool toDo = true; + assert(!curArgVal->OperIs(GT_RET_EXPR)); - noway_assert(!fgComputePredsDone); - if (!fgCheapPredsValid) + if (curArgVal->gtOper == GT_MKREFANY) { - fgComputeCheapPreds(); + inlineResult->NoteFatal(InlineObservation::CALLSITE_ARG_IS_MKREFANY); + return; } - BlockListNode* succCliqueToDo = nullptr; - BlockListNode* predCliqueToDo = new (this) BlockListNode(block); - while (toDo) + GenTree* lclVarTree; + + const bool isAddressInLocal = impIsAddressInLocal(curArgVal, &lclVarTree); + if (isAddressInLocal && varTypeIsStruct(lclVarTree)) { - toDo = false; - // Look at the successors of every member of the predecessor to-do list. - while (predCliqueToDo != nullptr) + inlCurArgInfo->argIsByRefToStructLocal = true; +#ifdef FEATURE_SIMD + if (lvaTable[lclVarTree->AsLclVarCommon()->GetLclNum()].lvSIMDType) { - BlockListNode* node = predCliqueToDo; - predCliqueToDo = node->m_next; - BasicBlock* blk = node->m_blk; - FreeBlockListNode(node); - - for (BasicBlock* const succ : blk->Succs()) - { - // If it's not already in the clique, add it, and also add it - // as a member of the successor "toDo" set. - if (impSpillCliqueGetMember(SpillCliqueSucc, succ) == 0) - { - callback->Visit(SpillCliqueSucc, succ); - impSpillCliqueSetMember(SpillCliqueSucc, succ, 1); - succCliqueToDo = new (this) BlockListNode(succ, succCliqueToDo); - toDo = true; - } - } + pInlineInfo->hasSIMDTypeArgLocalOrReturn = true; } - // Look at the predecessors of every member of the successor to-do list. - while (succCliqueToDo != nullptr) - { - BlockListNode* node = succCliqueToDo; - succCliqueToDo = node->m_next; - BasicBlock* blk = node->m_blk; - FreeBlockListNode(node); +#endif // FEATURE_SIMD + } - for (BasicBlockList* pred = blk->bbCheapPreds; pred != nullptr; pred = pred->next) - { - BasicBlock* predBlock = pred->block; - // If it's not already in the clique, add it, and also add it - // as a member of the predecessor "toDo" set. - if (impSpillCliqueGetMember(SpillCliquePred, predBlock) == 0) - { - callback->Visit(SpillCliquePred, predBlock); - impSpillCliqueSetMember(SpillCliquePred, predBlock, 1); - predCliqueToDo = new (this) BlockListNode(predBlock, predCliqueToDo); - toDo = true; - } - } - } + if (curArgVal->gtFlags & GTF_ALL_EFFECT) + { + inlCurArgInfo->argHasGlobRef = (curArgVal->gtFlags & GTF_GLOB_REF) != 0; + inlCurArgInfo->argHasSideEff = (curArgVal->gtFlags & (GTF_ALL_EFFECT & ~GTF_GLOB_REF)) != 0; } - // If this fails, it means we didn't walk the spill clique properly and somehow managed - // miss walking back to include the predecessor we started from. - // This most likely cause: missing or out of date bbPreds - assert(impSpillCliqueGetMember(SpillCliquePred, block) != 0); -} + if (curArgVal->gtOper == GT_LCL_VAR) + { + inlCurArgInfo->argIsLclVar = true; -void Compiler::SetSpillTempsBase::Visit(SpillCliqueDir predOrSucc, BasicBlock* blk) -{ - if (predOrSucc == SpillCliqueSucc) + /* Remember the "original" argument number */ + INDEBUG(curArgVal->AsLclVar()->gtLclILoffs = argNum;) + } + + if (impIsInvariant(curArgVal)) { - assert(blk->bbStkTempsIn == NO_BASE_TMP); // Should not already be a member of a clique as a successor. - blk->bbStkTempsIn = m_baseTmp; + inlCurArgInfo->argIsInvariant = true; + if (inlCurArgInfo->argIsThis && (curArgVal->gtOper == GT_CNS_INT) && (curArgVal->AsIntCon()->gtIconVal == 0)) + { + // Abort inlining at this call site + inlineResult->NoteFatal(InlineObservation::CALLSITE_ARG_HAS_NULL_THIS); + return; + } } else { - assert(predOrSucc == SpillCliquePred); - assert(blk->bbStkTempsOut == NO_BASE_TMP); // Should not already be a member of a clique as a predecessor. - blk->bbStkTempsOut = m_baseTmp; + if (curArgVal->IsHelperCall() && gtIsTypeHandleToRuntimeTypeHelper(curArgVal->AsCall()) && + (gtGetHelperArgClassHandle(curArgVal->AsCall()->gtArgs.GetArgByIndex(0)->GetEarlyNode()) != + NO_CLASS_HANDLE)) + { + inlCurArgInfo->argIsInvariant = true; + inlCurArgInfo->argHasSideEff = false; + } } -} -void Compiler::ReimportSpillClique::Visit(SpillCliqueDir predOrSucc, BasicBlock* blk) -{ - // For Preds we could be a little smarter and just find the existing store - // and re-type it/add a cast, but that is complicated and hopefully very rare, so - // just re-import the whole block (just like we do for successors) + bool isExact = false; + bool isNonNull = false; + inlCurArgInfo->argIsExact = (gtGetClassHandle(curArgVal, &isExact, &isNonNull) != NO_CLASS_HANDLE) && isExact; - if (((blk->bbFlags & BBF_IMPORTED) == 0) && (m_pComp->impGetPendingBlockMember(blk) == 0)) + // If the arg is a local that is address-taken, we can't safely + // directly substitute it into the inlinee. + // + // Previously we'd accomplish this by setting "argHasLdargaOp" but + // that has a stronger meaning: that the arg value can change in + // the method body. Using that flag prevents type propagation, + // which is safe in this case. + // + // Instead mark the arg as having a caller local ref. + if (!inlCurArgInfo->argIsInvariant && gtHasLocalsWithAddrOp(curArgVal)) { - // If we haven't imported this block and we're not going to (because it isn't on - // the pending list) then just ignore it for now. - - // This block has either never been imported (EntryState == NULL) or it failed - // verification. Neither state requires us to force it to be imported now. - assert((blk->bbEntryState == nullptr) || (blk->bbFlags & BBF_FAILED_VERIFICATION)); - return; + inlCurArgInfo->argHasCallerLocalRef = true; } - // For successors we have a valid verCurrentState, so just mark them for reimport - // the 'normal' way - // Unlike predecessors, we *DO* need to reimport the current block because the - // initial import had the wrong entry state types. - // Similarly, blocks that are currently on the pending list, still need to call - // impImportBlockPending to fixup their entry state. - if (predOrSucc == SpillCliqueSucc) +#ifdef DEBUG + if (verbose) { - m_pComp->impReimportMarkBlock(blk); - - // Set the current stack state to that of the blk->bbEntryState - m_pComp->verResetCurrentState(blk, &m_pComp->verCurrentState); + if (inlCurArgInfo->argIsThis) + { + printf("thisArg:"); + } + else + { + printf("\nArgument #%u:", argNum); + } + if (inlCurArgInfo->argIsLclVar) + { + printf(" is a local var"); + } + if (inlCurArgInfo->argIsInvariant) + { + printf(" is a constant or invariant"); + } + if (inlCurArgInfo->argHasGlobRef) + { + printf(" has global refs"); + } + if (inlCurArgInfo->argHasCallerLocalRef) + { + printf(" has caller local ref"); + } + if (inlCurArgInfo->argHasSideEff) + { + printf(" has side effects"); + } + if (inlCurArgInfo->argHasLdargaOp) + { + printf(" has ldarga effect"); + } + if (inlCurArgInfo->argHasStargOp) + { + printf(" has starg effect"); + } + if (inlCurArgInfo->argIsByRefToStructLocal) + { + printf(" is byref to a struct local"); + } - m_pComp->impImportBlockPending(blk); - } - else if ((blk != m_pComp->compCurBB) && ((blk->bbFlags & BBF_IMPORTED) != 0)) - { - // As described above, we are only visiting predecessors so they can - // add the appropriate casts, since we have already done that for the current - // block, it does not need to be reimported. - // Nor do we need to reimport blocks that are still pending, but not yet - // imported. - // - // For predecessors, we have no state to seed the EntryState, so we just have - // to assume the existing one is correct. - // If the block is also a successor, it will get the EntryState properly - // updated when it is visited as a successor in the above "if" block. - assert(predOrSucc == SpillCliquePred); - m_pComp->impReimportBlockPending(blk); + printf("\n"); + gtDispTree(curArgVal); + printf("\n"); } +#endif } -// Re-type the incoming lclVar nodes to match the varDsc. -void Compiler::impRetypeEntryStateTemps(BasicBlock* blk) +//------------------------------------------------------------------------ +// impInlineInitVars: setup inline information for inlinee args and locals +// +// Arguments: +// pInlineInfo - inline info for the inline candidate +// +// Notes: +// This method primarily adds caller-supplied info to the inlArgInfo +// and sets up the lclVarInfo table. +// +// For args, the inlArgInfo records properties of the actual argument +// including the tree node that produces the arg value. This node is +// usually the tree node present at the call, but may also differ in +// various ways: +// - when the call arg is a GT_RET_EXPR, we search back through the ret +// expr chain for the actual node. Note this will either be the original +// call (which will be a failed inline by this point), or the return +// expression from some set of inlines. +// - when argument type casting is needed the necessary casts are added +// around the argument node. +// - if an argument can be simplified by folding then the node here is the +// folded value. +// +// The method may make observations that lead to marking this candidate as +// a failed inline. If this happens the initialization is abandoned immediately +// to try and reduce the jit time cost for a failed inline. + +void Compiler::impInlineInitVars(InlineInfo* pInlineInfo) { - if (blk->bbEntryState != nullptr) + assert(!compIsForInlining()); + + GenTreeCall* call = pInlineInfo->iciCall; + CORINFO_METHOD_INFO* methInfo = &pInlineInfo->inlineCandidateInfo->methInfo; + unsigned clsAttr = pInlineInfo->inlineCandidateInfo->clsAttr; + InlArgInfo* inlArgInfo = pInlineInfo->inlArgInfo; + InlLclVarInfo* lclVarInfo = pInlineInfo->lclVarInfo; + InlineResult* inlineResult = pInlineInfo->inlineResult; + + // Inlined methods always use the managed calling convention + const bool hasRetBuffArg = impMethodInfo_hasRetBuffArg(methInfo, CorInfoCallConvExtension::Managed); + + /* init the argument stuct */ + + memset(inlArgInfo, 0, (MAX_INL_ARGS + 1) * sizeof(inlArgInfo[0])); + + unsigned ilArgCnt = 0; + for (CallArg& arg : call->gtArgs.Args()) { - EntryState* es = blk->bbEntryState; - for (unsigned level = 0; level < es->esStackDepth; level++) + switch (arg.GetWellKnownArg()) { - GenTree* tree = es->esStack[level].val; - if ((tree->gtOper == GT_LCL_VAR) || (tree->gtOper == GT_LCL_FLD)) - { - es->esStack[level].val->gtType = lvaGetDesc(tree->AsLclVarCommon())->TypeGet(); - } + case WellKnownArg::ThisPointer: + inlArgInfo[ilArgCnt].argIsThis = true; + break; + case WellKnownArg::RetBuffer: + case WellKnownArg::InstParam: + // These do not appear in the table of inline arg info; do not include them + continue; + default: + break; } - } -} -unsigned Compiler::impGetSpillTmpBase(BasicBlock* block) -{ - if (block->bbStkTempsOut != NO_BASE_TMP) - { - return block->bbStkTempsOut; - } + arg.SetEarlyNode(gtFoldExpr(arg.GetEarlyNode())); + impInlineRecordArgInfo(pInlineInfo, &arg, ilArgCnt, inlineResult); -#ifdef DEBUG - if (verbose) - { - printf("\n*************** In impGetSpillTmpBase(" FMT_BB ")\n", block->bbNum); + if (inlineResult->IsFailure()) + { + return; + } + + ilArgCnt++; } -#endif // DEBUG - // Otherwise, choose one, and propagate to all members of the spill clique. - // Grab enough temps for the whole stack. - unsigned baseTmp = lvaGrabTemps(verCurrentState.esStackDepth DEBUGARG("IL Stack Entries")); - SetSpillTempsBase callback(baseTmp); + /* Make sure we got the arg number right */ + assert(ilArgCnt == methInfo->args.totalILArgs()); - // We do *NOT* need to reset the SpillClique*Members because a block can only be the predecessor - // to one spill clique, and similarly can only be the successor to one spill clique - impWalkSpillCliqueFromPred(block, &callback); +#ifdef FEATURE_SIMD + bool foundSIMDType = pInlineInfo->hasSIMDTypeArgLocalOrReturn; +#endif // FEATURE_SIMD - return baseTmp; -} + /* We have typeless opcodes, get type information from the signature */ -void Compiler::impReimportSpillClique(BasicBlock* block) -{ -#ifdef DEBUG - if (verbose) + CallArg* thisArg = call->gtArgs.GetThisArg(); + if (thisArg != nullptr) { - printf("\n*************** In impReimportSpillClique(" FMT_BB ")\n", block->bbNum); - } -#endif // DEBUG - - // If we get here, it is because this block is already part of a spill clique - // and one predecessor had an outgoing live stack slot of type int, and this - // block has an outgoing live stack slot of type native int. - // We need to reset these before traversal because they have already been set - // by the previous walk to determine all the members of the spill clique. - impInlineRoot()->impSpillCliquePredMembers.Reset(); - impInlineRoot()->impSpillCliqueSuccMembers.Reset(); + lclVarInfo[0].lclVerTypeInfo = verMakeTypeInfo(pInlineInfo->inlineCandidateInfo->clsHandle); + lclVarInfo[0].lclHasLdlocaOp = false; - ReimportSpillClique callback(this); +#ifdef FEATURE_SIMD + // We always want to check isSIMDClass, since we want to set foundSIMDType (to increase + // the inlining multiplier) for anything in that assembly. + // But we only need to normalize it if it is a TYP_STRUCT + // (which we need to do even if we have already set foundSIMDType). + if (!foundSIMDType && isSIMDorHWSIMDClass(&(lclVarInfo[0].lclVerTypeInfo))) + { + foundSIMDType = true; + } +#endif // FEATURE_SIMD - impWalkSpillCliqueFromPred(block, &callback); -} + var_types sigType = ((clsAttr & CORINFO_FLG_VALUECLASS) != 0) ? TYP_BYREF : TYP_REF; + lclVarInfo[0].lclTypeInfo = sigType; -// Set the pre-state of "block" (which should not have a pre-state allocated) to -// a copy of "srcState", cloning tree pointers as required. -void Compiler::verInitBBEntryState(BasicBlock* block, EntryState* srcState) -{ - if (srcState->esStackDepth == 0) - { - block->bbEntryState = nullptr; - return; - } + GenTree* thisArgNode = thisArg->GetEarlyNode(); - block->bbEntryState = getAllocator(CMK_Unknown).allocate(1); + assert(varTypeIsGC(thisArgNode->TypeGet()) || // "this" is managed + ((thisArgNode->TypeGet() == TYP_I_IMPL) && // "this" is unmgd but the method's class doesnt care + (clsAttr & CORINFO_FLG_VALUECLASS))); - // block->bbEntryState.esRefcount = 1; + if (genActualType(thisArgNode->TypeGet()) != genActualType(sigType)) + { + if (sigType == TYP_REF) + { + /* The argument cannot be bashed into a ref (see bug 750871) */ + inlineResult->NoteFatal(InlineObservation::CALLSITE_ARG_NO_BASH_TO_REF); + return; + } - block->bbEntryState->esStackDepth = srcState->esStackDepth; + /* This can only happen with byrefs <-> ints/shorts */ - if (srcState->esStackDepth > 0) - { - block->bbSetStack(new (this, CMK_Unknown) StackEntry[srcState->esStackDepth]); - unsigned stackSize = srcState->esStackDepth * sizeof(StackEntry); + assert(sigType == TYP_BYREF); + assert((genActualType(thisArgNode->TypeGet()) == TYP_I_IMPL) || (thisArgNode->TypeGet() == TYP_BYREF)); - memcpy(block->bbEntryState->esStack, srcState->esStack, stackSize); - for (unsigned level = 0; level < srcState->esStackDepth; level++) - { - GenTree* tree = srcState->esStack[level].val; - block->bbEntryState->esStack[level].val = gtCloneExpr(tree); + lclVarInfo[0].lclVerTypeInfo = typeInfo(varType2tiType(TYP_I_IMPL)); } } -} -/* - * Resets the current state to the state at the start of the basic block - */ -void Compiler::verResetCurrentState(BasicBlock* block, EntryState* destState) -{ - if (block->bbEntryState == nullptr) - { - destState->esStackDepth = 0; - return; - } + /* Init the types of the arguments and make sure the types + * from the trees match the types in the signature */ - destState->esStackDepth = block->bbEntryState->esStackDepth; + CORINFO_ARG_LIST_HANDLE argLst; + argLst = methInfo->args.args; - if (destState->esStackDepth > 0) + // TODO-ARGS: We can presumably just use type info stored in CallArgs + // instead of reiterating the signature. + unsigned i; + for (i = (thisArg ? 1 : 0); i < ilArgCnt; i++, argLst = info.compCompHnd->getArgNext(argLst)) { - unsigned stackSize = destState->esStackDepth * sizeof(StackEntry); + var_types sigType = (var_types)eeGetArgType(argLst, &methInfo->args); - memcpy(destState->esStack, block->bbStackOnEntry(), stackSize); - } -} + lclVarInfo[i].lclVerTypeInfo = verParseArgSigToTypeInfo(&methInfo->args, argLst); -unsigned BasicBlock::bbStackDepthOnEntry() const -{ - return (bbEntryState ? bbEntryState->esStackDepth : 0); -} +#ifdef FEATURE_SIMD + if ((!foundSIMDType || (sigType == TYP_STRUCT)) && isSIMDorHWSIMDClass(&(lclVarInfo[i].lclVerTypeInfo))) + { + // If this is a SIMD class (i.e. in the SIMD assembly), then we will consider that we've + // found a SIMD type, even if this may not be a type we recognize (the assumption is that + // it is likely to use a SIMD type, and therefore we want to increase the inlining multiplier). + foundSIMDType = true; + if (sigType == TYP_STRUCT) + { + var_types structType = impNormStructType(lclVarInfo[i].lclVerTypeInfo.GetClassHandle()); + sigType = structType; + } + } +#endif // FEATURE_SIMD -void BasicBlock::bbSetStack(void* stackBuffer) -{ - assert(bbEntryState); - assert(stackBuffer); - bbEntryState->esStack = (StackEntry*)stackBuffer; -} + lclVarInfo[i].lclTypeInfo = sigType; + lclVarInfo[i].lclHasLdlocaOp = false; -StackEntry* BasicBlock::bbStackOnEntry() const -{ - assert(bbEntryState); - return bbEntryState->esStack; -} + // Does the tree type match the signature type? -void Compiler::verInitCurrentState() -{ - // initialize stack info - verCurrentState.esStackDepth = 0; - assert(verCurrentState.esStack != nullptr); + GenTree* inlArgNode = inlArgInfo[i].arg->GetNode(); - // copy current state to entry state of first BB - verInitBBEntryState(fgFirstBB, &verCurrentState); -} + if (sigType == inlArgNode->gtType) + { + continue; + } -Compiler* Compiler::impInlineRoot() -{ - if (impInlineInfo == nullptr) - { - return this; - } - else - { - return impInlineInfo->InlineRoot; - } -} + assert(impCheckImplicitArgumentCoercion(sigType, inlArgNode->gtType)); + assert(!varTypeIsStruct(inlArgNode->gtType) && !varTypeIsStruct(sigType)); -BYTE Compiler::impSpillCliqueGetMember(SpillCliqueDir predOrSucc, BasicBlock* blk) -{ - if (predOrSucc == SpillCliquePred) - { - return impInlineRoot()->impSpillCliquePredMembers.Get(blk->bbInd()); - } - else - { - assert(predOrSucc == SpillCliqueSucc); - return impInlineRoot()->impSpillCliqueSuccMembers.Get(blk->bbInd()); - } -} + // In valid IL, this can only happen for short integer types or byrefs <-> [native] ints, + // but in bad IL cases with caller-callee signature mismatches we can see other types. + // Intentionally reject cases with mismatches so the jit is more flexible when + // encountering bad IL. -void Compiler::impSpillCliqueSetMember(SpillCliqueDir predOrSucc, BasicBlock* blk, BYTE val) -{ - if (predOrSucc == SpillCliquePred) - { - impInlineRoot()->impSpillCliquePredMembers.Set(blk->bbInd(), val); - } - else - { - assert(predOrSucc == SpillCliqueSucc); - impInlineRoot()->impSpillCliqueSuccMembers.Set(blk->bbInd(), val); - } -} + bool isPlausibleTypeMatch = (genActualType(sigType) == genActualType(inlArgNode->gtType)) || + (genActualTypeIsIntOrI(sigType) && inlArgNode->gtType == TYP_BYREF) || + (sigType == TYP_BYREF && genActualTypeIsIntOrI(inlArgNode->gtType)); -/***************************************************************************** - * - * Convert the instrs ("import") into our internal format (trees). The - * basic flowgraph has already been constructed and is passed in. - */ + if (!isPlausibleTypeMatch) + { + inlineResult->NoteFatal(InlineObservation::CALLSITE_ARG_TYPES_INCOMPATIBLE); + return; + } + + // The same size but different type of the arguments. + GenTree** pInlArgNode = &inlArgInfo[i].arg->EarlyNodeRef(); + + // Is it a narrowing or widening cast? + // Widening casts are ok since the value computed is already + // normalized to an int (on the IL stack) + if (genTypeSize(inlArgNode->gtType) >= genTypeSize(sigType)) + { + if (sigType == TYP_BYREF) + { + lclVarInfo[i].lclVerTypeInfo = typeInfo(varType2tiType(TYP_I_IMPL)); + } + else if (inlArgNode->gtType == TYP_BYREF) + { + assert(varTypeIsIntOrI(sigType)); + + /* If possible bash the BYREF to an int */ + if (inlArgNode->IsLocalAddrExpr() != nullptr) + { + inlArgNode->gtType = TYP_I_IMPL; + lclVarInfo[i].lclVerTypeInfo = typeInfo(varType2tiType(TYP_I_IMPL)); + } + else + { + // Arguments 'int <- byref' cannot be changed + inlineResult->NoteFatal(InlineObservation::CALLSITE_ARG_NO_BASH_TO_INT); + return; + } + } + else if (genTypeSize(sigType) < TARGET_POINTER_SIZE) + { + // Narrowing cast. + if (inlArgNode->OperIs(GT_LCL_VAR)) + { + const unsigned lclNum = inlArgNode->AsLclVarCommon()->GetLclNum(); + if (!lvaTable[lclNum].lvNormalizeOnLoad() && sigType == lvaGetRealType(lclNum)) + { + // We don't need to insert a cast here as the variable + // was assigned a normalized value of the right type. + continue; + } + } -void Compiler::impImport() -{ -#ifdef DEBUG - if (verbose) - { - printf("*************** In impImport() for %s\n", info.compFullName); - } -#endif + inlArgNode = gtNewCastNode(TYP_INT, inlArgNode, false, sigType); - Compiler* inlineRoot = impInlineRoot(); + inlArgInfo[i].argIsLclVar = false; + // Try to fold the node in case we have constant arguments. + if (inlArgInfo[i].argIsInvariant) + { + inlArgNode = gtFoldExpr(inlArgNode); + } + *pInlArgNode = inlArgNode; + } +#ifdef TARGET_64BIT + else if (genTypeSize(genActualType(inlArgNode->gtType)) < genTypeSize(sigType)) + { + // This should only happen for int -> native int widening + inlArgNode = gtNewCastNode(genActualType(sigType), inlArgNode, false, sigType); - if (info.compMaxStack <= SMALL_STACK_SIZE) - { - impStkSize = SMALL_STACK_SIZE; - } - else - { - impStkSize = info.compMaxStack; - } + inlArgInfo[i].argIsLclVar = false; - if (this == inlineRoot) - { - // Allocate the stack contents - verCurrentState.esStack = new (this, CMK_ImpStack) StackEntry[impStkSize]; - } - else - { - // This is the inlinee compiler, steal the stack from the inliner compiler - // (after ensuring that it is large enough). - if (inlineRoot->impStkSize < impStkSize) - { - inlineRoot->impStkSize = impStkSize; - inlineRoot->verCurrentState.esStack = new (this, CMK_ImpStack) StackEntry[impStkSize]; + // Try to fold the node in case we have constant arguments. + if (inlArgInfo[i].argIsInvariant) + { + inlArgNode = gtFoldExpr(inlArgNode); + } + *pInlArgNode = inlArgNode; + } +#endif // TARGET_64BIT } - - verCurrentState.esStack = inlineRoot->verCurrentState.esStack; } - // initialize the entry state at start of method - verInitCurrentState(); + /* Init the types of the local variables */ - // Initialize stuff related to figuring "spill cliques" (see spec comment for impGetSpillTmpBase). - if (this == inlineRoot) // These are only used on the root of the inlining tree. - { - // We have initialized these previously, but to size 0. Make them larger. - impPendingBlockMembers.Init(getAllocator(), fgBBNumMax * 2); - impSpillCliquePredMembers.Init(getAllocator(), fgBBNumMax * 2); - impSpillCliqueSuccMembers.Init(getAllocator(), fgBBNumMax * 2); - } - inlineRoot->impPendingBlockMembers.Reset(fgBBNumMax * 2); - inlineRoot->impSpillCliquePredMembers.Reset(fgBBNumMax * 2); - inlineRoot->impSpillCliqueSuccMembers.Reset(fgBBNumMax * 2); - impBlockListNodeFreeList = nullptr; + CORINFO_ARG_LIST_HANDLE localsSig; + localsSig = methInfo->locals.args; -#ifdef DEBUG - impLastILoffsStmt = nullptr; - impNestedStackSpill = false; -#endif - impBoxTemp = BAD_VAR_NUM; + for (i = 0; i < methInfo->locals.numArgs; i++) + { + bool isPinned; + var_types type = (var_types)eeGetArgType(localsSig, &methInfo->locals, &isPinned); - impPendingList = impPendingFree = nullptr; + lclVarInfo[i + ilArgCnt].lclHasLdlocaOp = false; + lclVarInfo[i + ilArgCnt].lclTypeInfo = type; - // Skip leading internal blocks. - // These can arise from needing a leading scratch BB, from EH normalization, and from OSR entry redirects. - // - BasicBlock* entryBlock = fgFirstBB; + if (varTypeIsGC(type)) + { + if (isPinned) + { + JITDUMP("Inlinee local #%02u is pinned\n", i); + lclVarInfo[i + ilArgCnt].lclIsPinned = true; - while (entryBlock->bbFlags & BBF_INTERNAL) - { - JITDUMP("Marking leading BBF_INTERNAL block " FMT_BB " as BBF_IMPORTED\n", entryBlock->bbNum); - entryBlock->bbFlags |= BBF_IMPORTED; + // Pinned locals may cause inlines to fail. + inlineResult->Note(InlineObservation::CALLEE_HAS_PINNED_LOCALS); + if (inlineResult->IsFailure()) + { + return; + } + } - if (entryBlock->bbJumpKind == BBJ_NONE) - { - entryBlock = entryBlock->bbNext; + pInlineInfo->numberOfGcRefLocals++; } - else if (opts.IsOSR() && (entryBlock->bbJumpKind == BBJ_ALWAYS)) + else if (isPinned) { - entryBlock = entryBlock->bbJumpDest; + JITDUMP("Ignoring pin on inlinee local #%02u -- not a GC type\n", i); } - else + + lclVarInfo[i + ilArgCnt].lclVerTypeInfo = verParseArgSigToTypeInfo(&methInfo->locals, localsSig); + + // If this local is a struct type with GC fields, inform the inliner. It may choose to bail + // out on the inline. + if (type == TYP_STRUCT) { - assert(!"unexpected bbJumpKind in entry sequence"); + CORINFO_CLASS_HANDLE lclHandle = lclVarInfo[i + ilArgCnt].lclVerTypeInfo.GetClassHandle(); + DWORD typeFlags = info.compCompHnd->getClassAttribs(lclHandle); + if ((typeFlags & CORINFO_FLG_CONTAINS_GC_PTR) != 0) + { + inlineResult->Note(InlineObservation::CALLEE_HAS_GC_STRUCT); + if (inlineResult->IsFailure()) + { + return; + } + + // Do further notification in the case where the call site is rare; some policies do + // not track the relative hotness of call sites for "always" inline cases. + if (pInlineInfo->iciBlock->isRunRarely()) + { + inlineResult->Note(InlineObservation::CALLSITE_RARE_GC_STRUCT); + if (inlineResult->IsFailure()) + { + + return; + } + } + } } - } - // Note for OSR we'd like to be able to verify this block must be - // stack empty, but won't know that until we've imported...so instead - // we'll BADCODE out if we mess up. - // - // (the concern here is that the runtime asks us to OSR a - // different IL version than the one that matched the method that - // triggered OSR). This should not happen but I might have the - // IL versioning stuff wrong. - // - // TODO: we also currently expect this block to be a join point, - // which we should verify over when we find jump targets. - impImportBlockPending(entryBlock); + localsSig = info.compCompHnd->getArgNext(localsSig); - /* Import blocks in the worker-list until there are no more */ +#ifdef FEATURE_SIMD + if ((!foundSIMDType || (type == TYP_STRUCT)) && isSIMDorHWSIMDClass(&(lclVarInfo[i + ilArgCnt].lclVerTypeInfo))) + { + foundSIMDType = true; + if (type == TYP_STRUCT) + { + var_types structType = impNormStructType(lclVarInfo[i + ilArgCnt].lclVerTypeInfo.GetClassHandle()); + lclVarInfo[i + ilArgCnt].lclTypeInfo = structType; + } + } +#endif // FEATURE_SIMD + } - while (impPendingList) +#ifdef FEATURE_SIMD + if (!foundSIMDType && (call->AsCall()->gtRetClsHnd != nullptr) && isSIMDorHWSIMDClass(call->AsCall()->gtRetClsHnd)) { - /* Remove the entry at the front of the list */ + foundSIMDType = true; + } + pInlineInfo->hasSIMDTypeArgLocalOrReturn = foundSIMDType; +#endif // FEATURE_SIMD +} - PendingDsc* dsc = impPendingList; - impPendingList = impPendingList->pdNext; - impSetPendingBlockMember(dsc->pdBB, 0); +//------------------------------------------------------------------------ +// impInlineFetchLocal: get a local var that represents an inlinee local +// +// Arguments: +// lclNum -- number of the inlinee local +// reason -- debug string describing purpose of the local var +// +// Returns: +// Number of the local to use +// +// Notes: +// This method is invoked only for locals actually used in the +// inlinee body. +// +// Allocates a new temp if necessary, and copies key properties +// over from the inlinee local var info. - /* Restore the stack state */ +unsigned Compiler::impInlineFetchLocal(unsigned lclNum DEBUGARG(const char* reason)) +{ + assert(compIsForInlining()); - verCurrentState.esStackDepth = dsc->pdSavedStack.ssDepth; - if (verCurrentState.esStackDepth) - { - impRestoreStackState(&dsc->pdSavedStack); - } + unsigned tmpNum = impInlineInfo->lclTmpNum[lclNum]; - /* Add the entry to the free list for reuse */ + if (tmpNum == BAD_VAR_NUM) + { + const InlLclVarInfo& inlineeLocal = impInlineInfo->lclVarInfo[lclNum + impInlineInfo->argCnt]; + const var_types lclTyp = inlineeLocal.lclTypeInfo; - dsc->pdNext = impPendingFree; - impPendingFree = dsc; + // The lifetime of this local might span multiple BBs. + // So it is a long lifetime local. + impInlineInfo->lclTmpNum[lclNum] = tmpNum = lvaGrabTemp(false DEBUGARG(reason)); - /* Now import the block */ + // Copy over key info + lvaTable[tmpNum].lvType = lclTyp; + lvaTable[tmpNum].lvHasLdAddrOp = inlineeLocal.lclHasLdlocaOp; + lvaTable[tmpNum].lvPinned = inlineeLocal.lclIsPinned; + lvaTable[tmpNum].lvHasILStoreOp = inlineeLocal.lclHasStlocOp; + lvaTable[tmpNum].lvHasMultipleILStoreOp = inlineeLocal.lclHasMultipleStlocOp; - if (dsc->pdBB->bbFlags & BBF_FAILED_VERIFICATION) + // Copy over class handle for ref types. Note this may be a + // shared type -- someday perhaps we can get the exact + // signature and pass in a more precise type. + if (lclTyp == TYP_REF) { - verConvertBBToThrowVerificationException(dsc->pdBB DEBUGARG(true)); - impEndTreeList(dsc->pdBB); + assert(lvaTable[tmpNum].lvSingleDef == 0); + + lvaTable[tmpNum].lvSingleDef = !inlineeLocal.lclHasMultipleStlocOp && !inlineeLocal.lclHasLdlocaOp; + if (lvaTable[tmpNum].lvSingleDef) + { + JITDUMP("Marked V%02u as a single def temp\n", tmpNum); + } + + lvaSetClass(tmpNum, inlineeLocal.lclVerTypeInfo.GetClassHandleForObjRef()); } - else - { - impImportBlock(dsc->pdBB); - if (compDonotInline()) + if (inlineeLocal.lclVerTypeInfo.IsStruct()) + { + if (varTypeIsStruct(lclTyp)) { - return; + lvaSetStruct(tmpNum, inlineeLocal.lclVerTypeInfo.GetClassHandle(), true /* unsafe value cls check */); } - if (compIsForImportOnly()) + else { - return; + // This is a wrapped primitive. Make sure the verstate knows that + lvaTable[tmpNum].lvVerTypeInfo = inlineeLocal.lclVerTypeInfo; } } - } #ifdef DEBUG - if (verbose && info.compXcptnsCount) - { - printf("\nAfter impImport() added block for try,catch,finally"); - fgDispBasicBlocks(); - printf("\n"); + // Sanity check that we're properly prepared for gc ref locals. + if (varTypeIsGC(lclTyp)) + { + // Since there are gc locals we should have seen them earlier + // and if there was a return value, set up the spill temp. + assert(impInlineInfo->HasGcRefLocals()); + assert((info.compRetNativeType == TYP_VOID) || fgNeedReturnSpillTemp()); + } + else + { + // Make sure all pinned locals count as gc refs. + assert(!inlineeLocal.lclIsPinned); + } +#endif // DEBUG } - // Used in impImportBlockPending() for STRESS_CHK_REIMPORT - for (BasicBlock* const block : Blocks()) - { - block->bbFlags &= ~BBF_VISITED; - } -#endif + return tmpNum; } //------------------------------------------------------------------------ -// impIsInvariant: check if a tree (created during import) is invariant. -// -// Arguments: -// tree -- The tree -// -// Returns: -// true if it is invariant -// -// Remarks: -// This is a variant of GenTree::IsInvariant that is more suitable for use -// during import. Unlike that function, this one handles GT_FIELD nodes. +// impInlineFetchArg: return tree node for argument value in an inlinee // -bool Compiler::impIsInvariant(const GenTree* tree) -{ - return tree->OperIsConst() || impIsAddressInLocal(tree); -} - -//------------------------------------------------------------------------ -// impIsAddressInLocal: -// Check to see if the tree is the address of a local or -// the address of a field in a local. // Arguments: -// tree -- The tree -// lclVarTreeOut -- [out] the local that this points into +// lclNum -- argument number in inlinee IL +// inlArgInfo -- argument info for inlinee +// lclVarInfo -- var info for inlinee // // Returns: -// true if it points into a local +// Tree for the argument's value. Often an inlinee-scoped temp +// GT_LCL_VAR but can be other tree kinds, if the argument +// expression from the caller can be directly substituted into the +// inlinee body. // -// Remarks: -// This is a variant of GenTree::IsLocalAddrExpr that is more suitable for -// use during import. Unlike that function, this one handles field nodes. +// Notes: +// Must be used only for arguments -- use impInlineFetchLocal for +// inlinee locals. // -bool Compiler::impIsAddressInLocal(const GenTree* tree, GenTree** lclVarTreeOut) -{ - const GenTree* op = tree; - while (op->OperIs(GT_FIELD_ADDR) && op->AsField()->IsInstance()) - { - op = op->AsField()->GetFldObj(); - } - - if (op->OperIs(GT_LCL_VAR_ADDR)) - { - if (lclVarTreeOut != nullptr) - { - *lclVarTreeOut = const_cast(op); - } - - return true; - } - - return false; -} - -//------------------------------------------------------------------------ -// impMakeDiscretionaryInlineObservations: make observations that help -// determine the profitability of a discretionary inline +// Direct substitution is performed when the formal argument cannot +// change value in the inlinee body (no starg or ldarga), and the +// actual argument expression's value cannot be changed if it is +// substituted it into the inlinee body. // -// Arguments: -// pInlineInfo -- InlineInfo for the inline, or null for the prejit root -// inlineResult -- InlineResult accumulating information about this inline +// Even if an inlinee-scoped temp is returned here, it may later be +// "bashed" to a caller-supplied tree when arguments are actually +// passed (see fgInlinePrependStatements). Bashing can happen if +// the argument ends up being single use and other conditions are +// met. So the contents of the tree returned here may not end up +// being the ones ultimately used for the argument. // -// Notes: -// If inlining or prejitting the root, this method also makes -// various observations about the method that factor into inline -// decisions. It sets `compNativeSizeEstimate` as a side effect. +// This method will side effect inlArgInfo. It should only be called +// for actual uses of the argument in the inlinee. -void Compiler::impMakeDiscretionaryInlineObservations(InlineInfo* pInlineInfo, InlineResult* inlineResult) +GenTree* Compiler::impInlineFetchArg(unsigned lclNum, InlArgInfo* inlArgInfo, InlLclVarInfo* lclVarInfo) { - assert((pInlineInfo != nullptr && compIsForInlining()) || // Perform the actual inlining. - (pInlineInfo == nullptr && !compIsForInlining()) // Calculate the static inlining hint for ngen. - ); - - // If we're really inlining, we should just have one result in play. - assert((pInlineInfo == nullptr) || (inlineResult == pInlineInfo->inlineResult)); - - // If this is a "forceinline" method, the JIT probably shouldn't have gone - // to the trouble of estimating the native code size. Even if it did, it - // shouldn't be relying on the result of this method. - assert(inlineResult->GetObservation() == InlineObservation::CALLEE_IS_DISCRETIONARY_INLINE); - - // Note if the caller contains NEWOBJ or NEWARR. - Compiler* rootCompiler = impInlineRoot(); + // Cache the relevant arg and lcl info for this argument. + // We will modify argInfo but not lclVarInfo. + InlArgInfo& argInfo = inlArgInfo[lclNum]; + const InlLclVarInfo& lclInfo = lclVarInfo[lclNum]; + const bool argCanBeModified = argInfo.argHasLdargaOp || argInfo.argHasStargOp; + const var_types lclTyp = lclInfo.lclTypeInfo; + GenTree* op1 = nullptr; - if ((rootCompiler->optMethodFlags & OMF_HAS_NEWARRAY) != 0) - { - inlineResult->Note(InlineObservation::CALLER_HAS_NEWARRAY); - } + GenTree* argNode = argInfo.arg->GetNode(); + assert(!argNode->OperIs(GT_RET_EXPR)); - if ((rootCompiler->optMethodFlags & OMF_HAS_NEWOBJ) != 0) + if (argInfo.argIsInvariant && !argCanBeModified) { - inlineResult->Note(InlineObservation::CALLER_HAS_NEWOBJ); - } - - bool calleeIsStatic = (info.compFlags & CORINFO_FLG_STATIC) != 0; - bool isSpecialMethod = (info.compFlags & CORINFO_FLG_CONSTRUCTOR) != 0; + // Directly substitute constants or addresses of locals + // + // Clone the constant. Note that we cannot directly use + // argNode in the trees even if !argInfo.argIsUsed as this + // would introduce aliasing between inlArgInfo[].argNode and + // impInlineExpr. Then gtFoldExpr() could change it, causing + // further references to the argument working off of the + // bashed copy. + op1 = gtCloneExpr(argNode); + PREFIX_ASSUME(op1 != nullptr); + argInfo.argTmpNum = BAD_VAR_NUM; - if (isSpecialMethod) - { - if (calleeIsStatic) - { - inlineResult->Note(InlineObservation::CALLEE_IS_CLASS_CTOR); - } - else + // We may need to retype to ensure we match the callee's view of the type. + // Otherwise callee-pass throughs of arguments can create return type + // mismatches that block inlining. + // + // Note argument type mismatches that prevent inlining should + // have been caught in impInlineInitVars. + if (op1->TypeGet() != lclTyp) { - inlineResult->Note(InlineObservation::CALLEE_IS_INSTANCE_CTOR); + op1->gtType = genActualType(lclTyp); } } - else if (!calleeIsStatic) + else if (argInfo.argIsLclVar && !argCanBeModified && !argInfo.argHasCallerLocalRef) { - // Callee is an instance method. + // Directly substitute unaliased caller locals for args that cannot be modified // - // Check if the callee has the same 'this' as the root. - if (pInlineInfo != nullptr) + // Use the caller-supplied node if this is the first use. + op1 = argNode; + unsigned argLclNum = op1->AsLclVarCommon()->GetLclNum(); + argInfo.argTmpNum = argLclNum; + + // Use an equivalent copy if this is the second or subsequent + // use. + // + // Note argument type mismatches that prevent inlining should + // have been caught in impInlineInitVars. If inlining is not prevented + // but a cast is necessary, we similarly expect it to have been inserted then. + // So here we may have argument type mismatches that are benign, for instance + // passing a TYP_SHORT local (eg. normalized-on-load) as a TYP_INT arg. + // The exception is when the inlining means we should start tracking the argument. + if (argInfo.argIsUsed || ((lclTyp == TYP_BYREF) && (op1->TypeGet() != TYP_BYREF))) { - GenTree* thisArg = pInlineInfo->iciCall->AsCall()->gtArgs.GetThisArg()->GetNode(); - assert(thisArg); - bool isSameThis = impIsThis(thisArg); - inlineResult->NoteBool(InlineObservation::CALLSITE_IS_SAME_THIS, isSameThis); + assert(op1->gtOper == GT_LCL_VAR); + assert(lclNum == op1->AsLclVar()->gtLclILoffs); + + // Create a new lcl var node - remember the argument lclNum + op1 = impCreateLocalNode(argLclNum DEBUGARG(op1->AsLclVar()->gtLclILoffs)); + // Start tracking things as a byref if the parameter is a byref. + if (lclTyp == TYP_BYREF) + { + op1->gtType = TYP_BYREF; + } } } + else if (argInfo.argIsByRefToStructLocal && !argInfo.argHasStargOp) + { + /* Argument is a by-ref address to a struct, a normed struct, or its field. + In these cases, don't spill the byref to a local, simply clone the tree and use it. + This way we will increase the chance for this byref to be optimized away by + a subsequent "dereference" operation. - bool callsiteIsGeneric = (rootCompiler->info.compMethodInfo->args.sigInst.methInstCount != 0) || - (rootCompiler->info.compMethodInfo->args.sigInst.classInstCount != 0); + From Dev11 bug #139955: Argument node can also be TYP_I_IMPL if we've bashed the tree + (in impInlineInitVars()), if the arg has argHasLdargaOp as well as argIsByRefToStructLocal. + For example, if the caller is: + ldloca.s V_1 // V_1 is a local struct + call void Test.ILPart::RunLdargaOnPointerArg(int32*) + and the callee being inlined has: + .method public static void RunLdargaOnPointerArg(int32* ptrToInts) cil managed + ldarga.s ptrToInts + call void Test.FourInts::NotInlined_SetExpectedValuesThroughPointerToPointer(int32**) + then we change the argument tree (of "ldloca.s V_1") to TYP_I_IMPL to match the callee signature. We'll + soon afterwards reject the inlining anyway, since the tree we return isn't a GT_LCL_VAR. + */ + assert(argNode->TypeGet() == TYP_BYREF || argNode->TypeGet() == TYP_I_IMPL); + op1 = gtCloneExpr(argNode); + } + else + { + /* Argument is a complex expression - it must be evaluated into a temp */ - bool calleeIsGeneric = (info.compMethodInfo->args.sigInst.methInstCount != 0) || - (info.compMethodInfo->args.sigInst.classInstCount != 0); + if (argInfo.argHasTmp) + { + assert(argInfo.argIsUsed); + assert(argInfo.argTmpNum < lvaCount); - if (!callsiteIsGeneric && calleeIsGeneric) - { - inlineResult->Note(InlineObservation::CALLSITE_NONGENERIC_CALLS_GENERIC); - } + /* Create a new lcl var node - remember the argument lclNum */ + op1 = gtNewLclvNode(argInfo.argTmpNum, genActualType(lclTyp)); + + /* This is the second or later use of the this argument, + so we have to use the temp (instead of the actual arg) */ + argInfo.argBashTmpNode = nullptr; + } + else + { + /* First time use */ + assert(!argInfo.argIsUsed); - // Inspect callee's arguments (and the actual values at the callsite for them) - CORINFO_SIG_INFO sig = info.compMethodInfo->args; - CORINFO_ARG_LIST_HANDLE sigArg = sig.args; + /* Reserve a temp for the expression. + * Use a large size node as we may change it later */ - CallArg* argUse = pInlineInfo == nullptr ? nullptr : pInlineInfo->iciCall->AsCall()->gtArgs.Args().begin().GetArg(); + const unsigned tmpNum = lvaGrabTemp(true DEBUGARG("Inlining Arg")); - for (unsigned i = 0; i < info.compMethodInfo->args.numArgs; i++) - { - if ((argUse != nullptr) && (argUse->GetWellKnownArg() == WellKnownArg::ThisPointer)) - { - argUse = argUse->GetNext(); - } + lvaTable[tmpNum].lvType = lclTyp; - CORINFO_CLASS_HANDLE sigClass; - CorInfoType corType = strip(info.compCompHnd->getArgType(&sig, sigArg, &sigClass)); - GenTree* argNode = argUse == nullptr ? nullptr : argUse->GetEarlyNode(); + // For ref types, determine the type of the temp. + if (lclTyp == TYP_REF) + { + if (!argCanBeModified) + { + // If the arg can't be modified in the method + // body, use the type of the value, if + // known. Otherwise, use the declared type. + assert(lvaTable[tmpNum].lvSingleDef == 0); + lvaTable[tmpNum].lvSingleDef = 1; + JITDUMP("Marked V%02u as a single def temp\n", tmpNum); + lvaSetClass(tmpNum, argNode, lclInfo.lclVerTypeInfo.GetClassHandleForObjRef()); + } + else + { + // Arg might be modified, use the declared type of + // the argument. + lvaSetClass(tmpNum, lclInfo.lclVerTypeInfo.GetClassHandleForObjRef()); + } + } - if (corType == CORINFO_TYPE_CLASS) - { - sigClass = info.compCompHnd->getArgClass(&sig, sigArg); - } - else if (corType == CORINFO_TYPE_VALUECLASS) - { - inlineResult->Note(InlineObservation::CALLEE_ARG_STRUCT); - } - else if (corType == CORINFO_TYPE_BYREF) - { - sigClass = info.compCompHnd->getArgClass(&sig, sigArg); - corType = info.compCompHnd->getChildType(sigClass, &sigClass); - } + assert(!lvaTable[tmpNum].IsAddressExposed()); + if (argInfo.argHasLdargaOp) + { + lvaTable[tmpNum].lvHasLdAddrOp = 1; + } - if (argNode != nullptr) - { - bool isExact = false; - bool isNonNull = false; - CORINFO_CLASS_HANDLE argCls = gtGetClassHandle(argNode, &isExact, &isNonNull); - if (argCls != nullptr) + if (lclInfo.lclVerTypeInfo.IsStruct()) { - const bool isArgValueType = eeIsValueClass(argCls); - // Exact class of the arg is known - if (isExact && !isArgValueType) + if (varTypeIsStruct(lclTyp)) { - inlineResult->Note(InlineObservation::CALLSITE_ARG_EXACT_CLS); - if ((argCls != sigClass) && (sigClass != nullptr)) + lvaSetStruct(tmpNum, lclInfo.lclVerTypeInfo.GetClassHandle(), true /* unsafe value cls check */); + if (info.compIsVarArgs) { - // .. but the signature accepts a less concrete type. - inlineResult->Note(InlineObservation::CALLSITE_ARG_EXACT_CLS_SIG_IS_NOT); + lvaSetStructUsedAsVarArg(tmpNum); } } - // Arg is a reference type in the signature and a boxed value type was passed. - else if (isArgValueType && (corType == CORINFO_TYPE_CLASS)) + else { - inlineResult->Note(InlineObservation::CALLSITE_ARG_BOXED); + // This is a wrapped primitive. Make sure the verstate knows that + lvaTable[tmpNum].lvVerTypeInfo = lclInfo.lclVerTypeInfo; } } - if (argNode->OperIsConst()) + argInfo.argHasTmp = true; + argInfo.argTmpNum = tmpNum; + + // If we require strict exception order, then arguments must + // be evaluated in sequence before the body of the inlined method. + // So we need to evaluate them to a temp. + // Also, if arguments have global or local references, we need to + // evaluate them to a temp before the inlined body as the + // inlined body may be modifying the global ref. + // TODO-1stClassStructs: We currently do not reuse an existing lclVar + // if it is a struct, because it requires some additional handling. + + if ((!varTypeIsStruct(lclTyp) && !argInfo.argHasSideEff && !argInfo.argHasGlobRef && + !argInfo.argHasCallerLocalRef)) { - inlineResult->Note(InlineObservation::CALLSITE_ARG_CONST); + /* Get a *LARGE* LCL_VAR node */ + op1 = gtNewLclLNode(tmpNum, genActualType(lclTyp) DEBUGARG(lclNum)); + + /* Record op1 as the very first use of this argument. + If there are no further uses of the arg, we may be + able to use the actual arg node instead of the temp. + If we do see any further uses, we will clear this. */ + argInfo.argBashTmpNode = op1; + } + else + { + /* Get a small LCL_VAR node */ + op1 = gtNewLclvNode(tmpNum, genActualType(lclTyp)); + /* No bashing of this argument */ + argInfo.argBashTmpNode = nullptr; } - argUse = argUse->GetNext(); } - sigArg = info.compCompHnd->getArgNext(sigArg); } - // Note if the callee's return type is a value type - if (info.compMethodInfo->args.retType == CORINFO_TYPE_VALUECLASS) - { - inlineResult->Note(InlineObservation::CALLEE_RETURNS_STRUCT); - } + // Mark this argument as used. + argInfo.argIsUsed = true; - // Note if the callee's class is a promotable struct - if ((info.compClassAttr & CORINFO_FLG_VALUECLASS) != 0) - { - assert(structPromotionHelper != nullptr); - if (structPromotionHelper->CanPromoteStructType(info.compClassHnd)) - { - inlineResult->Note(InlineObservation::CALLEE_CLASS_PROMOTABLE); - } - inlineResult->Note(InlineObservation::CALLEE_CLASS_VALUETYPE); - } + return op1; +} -#ifdef FEATURE_SIMD +/****************************************************************************** + Is this the original "this" argument to the call being inlined? - // Note if this method is has SIMD args or return value - if (pInlineInfo != nullptr && pInlineInfo->hasSIMDTypeArgLocalOrReturn) - { - inlineResult->Note(InlineObservation::CALLEE_HAS_SIMD); - } + Note that we do not inline methods with "starg 0", and so we do not need to + worry about it. +*/ -#endif // FEATURE_SIMD +bool Compiler::impInlineIsThis(GenTree* tree, InlArgInfo* inlArgInfo) +{ + assert(compIsForInlining()); + return (tree->gtOper == GT_LCL_VAR && tree->AsLclVarCommon()->GetLclNum() == inlArgInfo[0].argTmpNum); +} - // Roughly classify callsite frequency. - InlineCallsiteFrequency frequency = InlineCallsiteFrequency::UNUSED; +//----------------------------------------------------------------------------- +// impInlineIsGuaranteedThisDerefBeforeAnySideEffects: Check if a dereference in +// the inlinee can guarantee that the "this" pointer is non-NULL. +// +// Arguments: +// additionalTree - a tree to check for side effects +// additionalCallArgs - a list of call args to check for side effects +// dereferencedAddress - address expression being dereferenced +// inlArgInfo - inlinee argument information +// +// Notes: +// If we haven't hit a branch or a side effect, and we are dereferencing +// from 'this' to access a field or make GTF_CALL_NULLCHECK call, +// then we can avoid a separate null pointer check. +// +// The importer stack and current statement list are searched for side effects. +// Trees that have been popped of the stack but haven't been appended to the +// statement list and have to be checked for side effects may be provided via +// additionalTree and additionalCallArgs. +// +bool Compiler::impInlineIsGuaranteedThisDerefBeforeAnySideEffects(GenTree* additionalTree, + CallArgs* additionalCallArgs, + GenTree* dereferencedAddress, + InlArgInfo* inlArgInfo) +{ + assert(compIsForInlining()); + assert(opts.OptEnabled(CLFLG_INLINING)); - // If this is a prejit root, or a maximally hot block... - if ((pInlineInfo == nullptr) || (pInlineInfo->iciBlock->isMaxBBWeight())) - { - frequency = InlineCallsiteFrequency::HOT; - } - // No training data. Look for loop-like things. - // We consider a recursive call loop-like. Do not give the inlining boost to the method itself. - // However, give it to things nearby. - else if ((pInlineInfo->iciBlock->bbFlags & BBF_BACKWARD_JUMP) && - (pInlineInfo->fncHandle != pInlineInfo->inlineCandidateInfo->ilCallerHandle)) - { - frequency = InlineCallsiteFrequency::LOOP; - } - else if (pInlineInfo->iciBlock->hasProfileWeight() && (pInlineInfo->iciBlock->bbWeight > BB_ZERO_WEIGHT)) - { - frequency = InlineCallsiteFrequency::WARM; - } - // Now modify the multiplier based on where we're called from. - else if (pInlineInfo->iciBlock->isRunRarely() || ((info.compFlags & FLG_CCTOR) == FLG_CCTOR)) + BasicBlock* block = compCurBB; + + if (block != fgFirstBB) { - frequency = InlineCallsiteFrequency::RARE; + return false; } - else + + if (!impInlineIsThis(dereferencedAddress, inlArgInfo)) { - frequency = InlineCallsiteFrequency::BORING; + return false; } - // Also capture the block weight of the call site. - // - // In the prejit root case, assume at runtime there might be a hot call site - // for this method, so we won't prematurely conclude this method should never - // be inlined. - // - weight_t weight = 0; - - if (pInlineInfo != nullptr) + if ((additionalTree != nullptr) && GTF_GLOBALLY_VISIBLE_SIDE_EFFECTS(additionalTree->gtFlags)) { - weight = pInlineInfo->iciBlock->bbWeight; + return false; } - else + + if (additionalCallArgs != nullptr) { - const weight_t prejitHotCallerWeight = 1000000.0; - weight = prejitHotCallerWeight; + for (CallArg& arg : additionalCallArgs->Args()) + { + if (GTF_GLOBALLY_VISIBLE_SIDE_EFFECTS(arg.GetEarlyNode()->gtFlags)) + { + return false; + } + } } - inlineResult->NoteInt(InlineObservation::CALLSITE_FREQUENCY, static_cast(frequency)); - inlineResult->NoteInt(InlineObservation::CALLSITE_WEIGHT, (int)(weight)); - - bool hasProfile = false; - double profileFreq = 0.0; - - // If the call site has profile data, report the relative frequency of the site. - // - if ((pInlineInfo != nullptr) && rootCompiler->fgHaveSufficientProfileWeights()) + for (Statement* stmt : StatementList(impStmtList)) { - const weight_t callSiteWeight = pInlineInfo->iciBlock->bbWeight; - const weight_t entryWeight = rootCompiler->fgFirstBB->bbWeight; - profileFreq = fgProfileWeightsEqual(entryWeight, 0.0) ? 0.0 : callSiteWeight / entryWeight; - hasProfile = true; - - assert(callSiteWeight >= 0); - assert(entryWeight >= 0); + GenTree* expr = stmt->GetRootNode(); + if (GTF_GLOBALLY_VISIBLE_SIDE_EFFECTS(expr->gtFlags)) + { + return false; + } } - else if (pInlineInfo == nullptr) + + for (unsigned level = 0; level < verCurrentState.esStackDepth; level++) { - // Simulate a hot callsite for PrejitRoot mode. - hasProfile = true; - profileFreq = 1.0; + GenTreeFlags stackTreeFlags = verCurrentState.esStack[level].val->gtFlags; + if (GTF_GLOBALLY_VISIBLE_SIDE_EFFECTS(stackTreeFlags)) + { + return false; + } } - inlineResult->NoteBool(InlineObservation::CALLSITE_HAS_PROFILE_WEIGHTS, hasProfile); - inlineResult->NoteDouble(InlineObservation::CALLSITE_PROFILE_FREQUENCY, profileFreq); + return true; } //------------------------------------------------------------------------ -// impCanInlineIL: screen inline candate based on info from the method header +// impMarkInlineCandidate: determine if this call can be subsequently inlined // // Arguments: -// fncHandle -- inline candidate method -// methInfo -- method info from VN -// forceInline -- true if method is marked with AggressiveInlining -// inlineResult -- ongoing inline evaluation +// callNode -- call under scrutiny +// exactContextHnd -- context handle for inlining +// exactContextNeedsRuntimeLookup -- true if context required runtime lookup +// callInfo -- call info from VM +// ilOffset -- the actual IL offset of the instruction that produced this inline candidate // -void Compiler::impCanInlineIL(CORINFO_METHOD_HANDLE fncHandle, - CORINFO_METHOD_INFO* methInfo, - bool forceInline, - InlineResult* inlineResult) +// Notes: +// Mostly a wrapper for impMarkInlineCandidateHelper that also undoes +// guarded devirtualization for virtual calls where the method we'd +// devirtualize to cannot be inlined. + +void Compiler::impMarkInlineCandidate(GenTree* callNode, + CORINFO_CONTEXT_HANDLE exactContextHnd, + bool exactContextNeedsRuntimeLookup, + CORINFO_CALL_INFO* callInfo, + IL_OFFSET ilOffset) { - unsigned codeSize = methInfo->ILCodeSize; + GenTreeCall* call = callNode->AsCall(); - // We shouldn't have made up our minds yet... - assert(!inlineResult->IsDecided()); + // Do the actual evaluation + impMarkInlineCandidateHelper(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, ilOffset); - if (methInfo->EHcount) + // If this call is an inline candidate or is not a guarded devirtualization + // candidate, we're done. + if (call->IsInlineCandidate() || !call->IsGuardedDevirtualizationCandidate()) { - inlineResult->NoteFatal(InlineObservation::CALLEE_HAS_EH); return; } - if ((methInfo->ILCode == nullptr) || (codeSize == 0)) + // If we can't inline the call we'd guardedly devirtualize to, + // we undo the guarded devirtualization, as the benefit from + // just guarded devirtualization alone is likely not worth the + // extra jit time and code size. + // + // TODO: it is possibly interesting to allow this, but requires + // fixes elsewhere too... + JITDUMP("Revoking guarded devirtualization candidacy for call [%06u]: target method can't be inlined\n", + dspTreeID(call)); + + call->ClearGuardedDevirtualizationCandidate(); +} + +//------------------------------------------------------------------------ +// impMarkInlineCandidateHelper: determine if this call can be subsequently +// inlined +// +// Arguments: +// callNode -- call under scrutiny +// exactContextHnd -- context handle for inlining +// exactContextNeedsRuntimeLookup -- true if context required runtime lookup +// callInfo -- call info from VM +// ilOffset -- IL offset of instruction creating the inline candidate +// +// Notes: +// If callNode is an inline candidate, this method sets the flag +// GTF_CALL_INLINE_CANDIDATE, and ensures that helper methods have +// filled in the associated InlineCandidateInfo. +// +// If callNode is not an inline candidate, and the reason is +// something that is inherent to the method being called, the +// method may be marked as "noinline" to short-circuit any +// future assessments of calls to this method. + +void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, + CORINFO_CONTEXT_HANDLE exactContextHnd, + bool exactContextNeedsRuntimeLookup, + CORINFO_CALL_INFO* callInfo, + IL_OFFSET ilOffset) +{ + // Let the strategy know there's another call + impInlineRoot()->m_inlineStrategy->NoteCall(); + + if (!opts.OptEnabled(CLFLG_INLINING)) { - inlineResult->NoteFatal(InlineObservation::CALLEE_HAS_NO_BODY); + /* XXX Mon 8/18/2008 + * This assert is misleading. The caller does not ensure that we have CLFLG_INLINING set before + * calling impMarkInlineCandidate. However, if this assert trips it means that we're an inlinee and + * CLFLG_MINOPT is set. That doesn't make a lot of sense. If you hit this assert, work back and + * figure out why we did not set MAXOPT for this compile. + */ + assert(!compIsForInlining()); return; } - // For now we don't inline varargs (import code can't handle it) - - if (methInfo->args.isVarArg()) + if (compIsForImportOnly()) { - inlineResult->NoteFatal(InlineObservation::CALLEE_HAS_MANAGED_VARARGS); + // Don't bother creating the inline candidate during verification. + // Otherwise the call to info.compCompHnd->canInline will trigger a recursive verification + // that leads to the creation of multiple instances of Compiler. return; } - // Reject if it has too many locals. - // This is currently an implementation limit due to fixed-size arrays in the - // inline info, rather than a performance heuristic. - - inlineResult->NoteInt(InlineObservation::CALLEE_NUMBER_OF_LOCALS, methInfo->locals.numArgs); + InlineResult inlineResult(this, call, nullptr, "impMarkInlineCandidate"); - if (methInfo->locals.numArgs > MAX_INL_LCLS) + // Don't inline if not optimizing root method + if (opts.compDbgCode) { - inlineResult->NoteFatal(InlineObservation::CALLEE_TOO_MANY_LOCALS); + inlineResult.NoteFatal(InlineObservation::CALLER_DEBUG_CODEGEN); return; } - // Make sure there aren't too many arguments. - // This is currently an implementation limit due to fixed-size arrays in the - // inline info, rather than a performance heuristic. + // Don't inline if inlining into this method is disabled. + if (impInlineRoot()->m_inlineStrategy->IsInliningDisabled()) + { + inlineResult.NoteFatal(InlineObservation::CALLER_IS_JIT_NOINLINE); + return; + } - inlineResult->NoteInt(InlineObservation::CALLEE_NUMBER_OF_ARGUMENTS, methInfo->args.numArgs); + // Don't inline into callers that use the NextCallReturnAddress intrinsic. + if (info.compHasNextCallRetAddr) + { + inlineResult.NoteFatal(InlineObservation::CALLER_USES_NEXT_CALL_RET_ADDR); + return; + } - if (methInfo->args.numArgs > MAX_INL_ARGS) + // Inlining candidate determination needs to honor only IL tail prefix. + // Inlining takes precedence over implicit tail call optimization (if the call is not directly recursive). + if (call->IsTailPrefixedCall()) { - inlineResult->NoteFatal(InlineObservation::CALLEE_TOO_MANY_ARGUMENTS); + inlineResult.NoteFatal(InlineObservation::CALLSITE_EXPLICIT_TAIL_PREFIX); return; } - // Note force inline state + // Delegate Invoke method doesn't have a body and gets special cased instead. + // Don't even bother trying to inline it. + if (call->IsDelegateInvoke() && !call->IsGuardedDevirtualizationCandidate()) + { + inlineResult.NoteFatal(InlineObservation::CALLEE_HAS_NO_BODY); + return; + } - inlineResult->NoteBool(InlineObservation::CALLEE_IS_FORCE_INLINE, forceInline); + // Tail recursion elimination takes precedence over inlining. + // TODO: We may want to do some of the additional checks from fgMorphCall + // here to reduce the chance we don't inline a call that won't be optimized + // as a fast tail call or turned into a loop. + if (gtIsRecursiveCall(call) && call->IsImplicitTailCall()) + { + inlineResult.NoteFatal(InlineObservation::CALLSITE_IMPLICIT_REC_TAIL_CALL); + return; + } - // Note IL code size + if (call->IsVirtual()) + { + // Allow guarded devirt calls to be treated as inline candidates, + // but reject all other virtual calls. + if (!call->IsGuardedDevirtualizationCandidate()) + { + inlineResult.NoteFatal(InlineObservation::CALLSITE_IS_NOT_DIRECT); + return; + } + } - inlineResult->NoteInt(InlineObservation::CALLEE_IL_CODE_SIZE, codeSize); + /* Ignore helper calls */ - if (inlineResult->IsFailure()) + if (call->gtCallType == CT_HELPER) { + assert(!call->IsGuardedDevirtualizationCandidate()); + inlineResult.NoteFatal(InlineObservation::CALLSITE_IS_CALL_TO_HELPER); return; } - // Make sure maxstack is not too big - - inlineResult->NoteInt(InlineObservation::CALLEE_MAXSTACK, methInfo->maxStack); - - if (inlineResult->IsFailure()) + /* Ignore indirect calls */ + if (call->gtCallType == CT_INDIRECT) { + inlineResult.NoteFatal(InlineObservation::CALLSITE_IS_NOT_DIRECT_MANAGED); return; } -} -//------------------------------------------------------------------------ -// impInlineRecordArgInfo: record information about an inline candidate argument -// -// Arguments: -// pInlineInfo - inline info for the inline candidate -// arg - the caller argument -// argNum - logical index of this argument -// inlineResult - result of ongoing inline evaluation -// -// Notes: -// -// Checks for various inline blocking conditions and makes notes in -// the inline info arg table about the properties of the actual. These -// properties are used later by impInlineFetchArg to determine how best to -// pass the argument into the inlinee. + /* I removed the check for BBJ_THROW. BBJ_THROW is usually marked as rarely run. This more or less + * restricts the inliner to non-expanding inlines. I removed the check to allow for non-expanding + * inlining in throw blocks. I should consider the same thing for catch and filter regions. */ -void Compiler::impInlineRecordArgInfo(InlineInfo* pInlineInfo, - CallArg* arg, - unsigned argNum, - InlineResult* inlineResult) -{ - InlArgInfo* inlCurArgInfo = &pInlineInfo->inlArgInfo[argNum]; + CORINFO_METHOD_HANDLE fncHandle; + unsigned methAttr; - inlCurArgInfo->arg = arg; - GenTree* curArgVal = arg->GetNode(); + if (call->IsGuardedDevirtualizationCandidate()) + { + if (call->gtGuardedDevirtualizationCandidateInfo->guardedMethodUnboxedEntryHandle != nullptr) + { + fncHandle = call->gtGuardedDevirtualizationCandidateInfo->guardedMethodUnboxedEntryHandle; + } + else + { + fncHandle = call->gtGuardedDevirtualizationCandidateInfo->guardedMethodHandle; + } + methAttr = info.compCompHnd->getMethodAttribs(fncHandle); + } + else + { + fncHandle = call->gtCallMethHnd; - assert(!curArgVal->OperIs(GT_RET_EXPR)); + // Reuse method flags from the original callInfo if possible + if (fncHandle == callInfo->hMethod) + { + methAttr = callInfo->methodFlags; + } + else + { + methAttr = info.compCompHnd->getMethodAttribs(fncHandle); + } + } - if (curArgVal->gtOper == GT_MKREFANY) +#ifdef DEBUG + if (compStressCompile(STRESS_FORCE_INLINE, 0)) { - inlineResult->NoteFatal(InlineObservation::CALLSITE_ARG_IS_MKREFANY); - return; + methAttr |= CORINFO_FLG_FORCEINLINE; } +#endif - GenTree* lclVarTree; - const bool isAddressInLocal = impIsAddressInLocal(curArgVal, &lclVarTree); - if (isAddressInLocal) + // Check for COMPlus_AggressiveInlining + if (compDoAggressiveInlining) { - LclVarDsc* varDsc = lvaGetDesc(lclVarTree->AsLclVarCommon()); + methAttr |= CORINFO_FLG_FORCEINLINE; + } - if (varTypeIsStruct(varDsc)) + if (!(methAttr & CORINFO_FLG_FORCEINLINE)) + { + /* Don't bother inline blocks that are in the filter region */ + if (bbInCatchHandlerILRange(compCurBB)) { - inlCurArgInfo->argIsByRefToStructLocal = true; -#ifdef FEATURE_SIMD - if (varDsc->lvSIMDType) +#ifdef DEBUG + if (verbose) { - pInlineInfo->hasSIMDTypeArgLocalOrReturn = true; + printf("\nWill not inline blocks that are in the catch handler region\n"); } -#endif // FEATURE_SIMD + +#endif + + inlineResult.NoteFatal(InlineObservation::CALLSITE_IS_WITHIN_CATCH); + return; } - // Spilling code relies on correct aliasability annotations. - assert(varDsc->lvHasLdAddrOp || varDsc->IsAddressExposed()); + if (bbInFilterILRange(compCurBB)) + { +#ifdef DEBUG + if (verbose) + { + printf("\nWill not inline blocks that are in the filter region\n"); + } +#endif + + inlineResult.NoteFatal(InlineObservation::CALLSITE_IS_WITHIN_FILTER); + return; + } } - if (curArgVal->gtFlags & GTF_ALL_EFFECT) + /* Check if we tried to inline this method before */ + + if (methAttr & CORINFO_FLG_DONT_INLINE) { - inlCurArgInfo->argHasGlobRef = (curArgVal->gtFlags & GTF_GLOB_REF) != 0; - inlCurArgInfo->argHasSideEff = (curArgVal->gtFlags & (GTF_ALL_EFFECT & ~GTF_GLOB_REF)) != 0; + inlineResult.NoteFatal(InlineObservation::CALLEE_IS_NOINLINE); + return; } - if (curArgVal->gtOper == GT_LCL_VAR) - { - inlCurArgInfo->argIsLclVar = true; + /* Cannot inline synchronized methods */ - /* Remember the "original" argument number */ - INDEBUG(curArgVal->AsLclVar()->gtLclILoffs = argNum;) + if (methAttr & CORINFO_FLG_SYNCH) + { + inlineResult.NoteFatal(InlineObservation::CALLEE_IS_SYNCHRONIZED); + return; } - if (impIsInvariant(curArgVal)) + /* Check legality of PInvoke callsite (for inlining of marshalling code) */ + + if (methAttr & CORINFO_FLG_PINVOKE) { - inlCurArgInfo->argIsInvariant = true; - if (inlCurArgInfo->argIsThis && (curArgVal->gtOper == GT_CNS_INT) && (curArgVal->AsIntCon()->gtIconVal == 0)) + // See comment in impCheckForPInvokeCall + BasicBlock* block = compIsForInlining() ? impInlineInfo->iciBlock : compCurBB; + if (!impCanPInvokeInlineCallSite(block)) { - // Abort inlining at this call site - inlineResult->NoteFatal(InlineObservation::CALLSITE_ARG_HAS_NULL_THIS); + inlineResult.NoteFatal(InlineObservation::CALLSITE_PINVOKE_EH); return; } } - else if (gtIsTypeof(curArgVal)) + + InlineCandidateInfo* inlineCandidateInfo = nullptr; + impCheckCanInline(call, fncHandle, methAttr, exactContextHnd, &inlineCandidateInfo, &inlineResult); + + if (inlineResult.IsFailure()) { - inlCurArgInfo->argIsInvariant = true; - inlCurArgInfo->argHasSideEff = false; + return; } - bool isExact = false; - bool isNonNull = false; - inlCurArgInfo->argIsExact = (gtGetClassHandle(curArgVal, &isExact, &isNonNull) != NO_CLASS_HANDLE) && isExact; + // The old value should be null OR this call should be a guarded devirtualization candidate. + assert((call->gtInlineCandidateInfo == nullptr) || call->IsGuardedDevirtualizationCandidate()); - // If the arg is a local that is address-taken, we can't safely - // directly substitute it into the inlinee. - // - // Previously we'd accomplish this by setting "argHasLdargaOp" but - // that has a stronger meaning: that the arg value can change in - // the method body. Using that flag prevents type propagation, - // which is safe in this case. + // The new value should not be null. + assert(inlineCandidateInfo != nullptr); + inlineCandidateInfo->exactContextNeedsRuntimeLookup = exactContextNeedsRuntimeLookup; + inlineCandidateInfo->ilOffset = ilOffset; + call->gtInlineCandidateInfo = inlineCandidateInfo; + + // If we're in an inlinee compiler, and have a return spill temp, and this inline candidate + // is also a tail call candidate, it can use the same return spill temp. // - // Instead mark the arg as having a caller local ref. - if (!inlCurArgInfo->argIsInvariant && gtHasLocalsWithAddrOp(curArgVal)) + if (compIsForInlining() && call->CanTailCall() && + (impInlineInfo->inlineCandidateInfo->preexistingSpillTemp != BAD_VAR_NUM)) { - inlCurArgInfo->argHasCallerLocalRef = true; + inlineCandidateInfo->preexistingSpillTemp = impInlineInfo->inlineCandidateInfo->preexistingSpillTemp; + JITDUMP("Inline candidate [%06u] can share spill temp V%02u\n", dspTreeID(call), + inlineCandidateInfo->preexistingSpillTemp); } -#ifdef DEBUG - if (verbose) + // Mark the call node as inline candidate. + call->gtFlags |= GTF_CALL_INLINE_CANDIDATE; + + // Let the strategy know there's another candidate. + impInlineRoot()->m_inlineStrategy->NoteCandidate(); + + // Since we're not actually inlining yet, and this call site is + // still just an inline candidate, there's nothing to report. + inlineResult.SetSuccessResult(INLINE_CHECK_CAN_INLINE_SUCCESS); +} + +/******************************************************************************/ +// Returns true if the given intrinsic will be implemented by target-specific +// instructions + +bool Compiler::IsTargetIntrinsic(NamedIntrinsic intrinsicName) +{ +#if defined(TARGET_XARCH) + switch (intrinsicName) { - if (inlCurArgInfo->argIsThis) - { - printf("thisArg:"); - } - else - { - printf("\nArgument #%u:", argNum); - } - if (inlCurArgInfo->argIsLclVar) - { - printf(" is a local var"); - } - if (inlCurArgInfo->argIsInvariant) - { - printf(" is a constant or invariant"); - } - if (inlCurArgInfo->argHasGlobRef) - { - printf(" has global refs"); - } - if (inlCurArgInfo->argHasCallerLocalRef) - { - printf(" has caller local ref"); - } - if (inlCurArgInfo->argHasSideEff) - { - printf(" has side effects"); - } - if (inlCurArgInfo->argHasLdargaOp) - { - printf(" has ldarga effect"); - } - if (inlCurArgInfo->argHasStargOp) + // AMD64/x86 has SSE2 instructions to directly compute sqrt/abs and SSE4.1 + // instructions to directly compute round/ceiling/floor/truncate. + + case NI_System_Math_Abs: + case NI_System_Math_Sqrt: + return true; + + case NI_System_Math_Ceiling: + case NI_System_Math_Floor: + case NI_System_Math_Truncate: + case NI_System_Math_Round: + return compOpportunisticallyDependsOn(InstructionSet_SSE41); + + case NI_System_Math_FusedMultiplyAdd: + return compOpportunisticallyDependsOn(InstructionSet_FMA); + + default: + return false; + } +#elif defined(TARGET_ARM64) + switch (intrinsicName) + { + case NI_System_Math_Abs: + case NI_System_Math_Ceiling: + case NI_System_Math_Floor: + case NI_System_Math_Truncate: + case NI_System_Math_Round: + case NI_System_Math_Sqrt: + case NI_System_Math_Max: + case NI_System_Math_Min: + return true; + + case NI_System_Math_FusedMultiplyAdd: + return compOpportunisticallyDependsOn(InstructionSet_AdvSimd); + + default: + return false; + } +#elif defined(TARGET_ARM) + switch (intrinsicName) + { + case NI_System_Math_Abs: + case NI_System_Math_Round: + case NI_System_Math_Sqrt: + return true; + + default: + return false; + } +#elif defined(TARGET_LOONGARCH64) + // TODO-LoongArch64: add some instrinsics. + return false; +#else + // TODO: This portion of logic is not implemented for other arch. + // The reason for returning true is that on all other arch the only intrinsic + // enabled are target intrinsics. + return true; +#endif +} + +/******************************************************************************/ +// Returns true if the given intrinsic will be implemented by calling System.Math +// methods. + +bool Compiler::IsIntrinsicImplementedByUserCall(NamedIntrinsic intrinsicName) +{ + // Currently, if a math intrinsic is not implemented by target-specific + // instructions, it will be implemented by a System.Math call. In the + // future, if we turn to implementing some of them with helper calls, + // this predicate needs to be revisited. + return !IsTargetIntrinsic(intrinsicName); +} + +bool Compiler::IsMathIntrinsic(NamedIntrinsic intrinsicName) +{ + switch (intrinsicName) + { + case NI_System_Math_Abs: + case NI_System_Math_Acos: + case NI_System_Math_Acosh: + case NI_System_Math_Asin: + case NI_System_Math_Asinh: + case NI_System_Math_Atan: + case NI_System_Math_Atanh: + case NI_System_Math_Atan2: + case NI_System_Math_Cbrt: + case NI_System_Math_Ceiling: + case NI_System_Math_Cos: + case NI_System_Math_Cosh: + case NI_System_Math_Exp: + case NI_System_Math_Floor: + case NI_System_Math_FMod: + case NI_System_Math_FusedMultiplyAdd: + case NI_System_Math_ILogB: + case NI_System_Math_Log: + case NI_System_Math_Log2: + case NI_System_Math_Log10: + case NI_System_Math_Max: + case NI_System_Math_Min: + case NI_System_Math_Pow: + case NI_System_Math_Round: + case NI_System_Math_Sin: + case NI_System_Math_Sinh: + case NI_System_Math_Sqrt: + case NI_System_Math_Tan: + case NI_System_Math_Tanh: + case NI_System_Math_Truncate: { - printf(" has starg effect"); + assert((intrinsicName > NI_SYSTEM_MATH_START) && (intrinsicName < NI_SYSTEM_MATH_END)); + return true; } - if (inlCurArgInfo->argIsByRefToStructLocal) + + default: { - printf(" is byref to a struct local"); + assert((intrinsicName < NI_SYSTEM_MATH_START) || (intrinsicName > NI_SYSTEM_MATH_END)); + return false; } - - printf("\n"); - gtDispTree(curArgVal); - printf("\n"); } -#endif +} + +bool Compiler::IsMathIntrinsic(GenTree* tree) +{ + return (tree->OperGet() == GT_INTRINSIC) && IsMathIntrinsic(tree->AsIntrinsic()->gtIntrinsicName); } //------------------------------------------------------------------------ -// impInlineInitVars: setup inline information for inlinee args and locals +// impDevirtualizeCall: Attempt to change a virtual vtable call into a +// normal call // // Arguments: -// pInlineInfo - inline info for the inline candidate +// call -- the call node to examine/modify +// pResolvedToken -- [IN] the resolved token used to create the call. Used for R2R. +// method -- [IN/OUT] the method handle for call. Updated iff call devirtualized. +// methodFlags -- [IN/OUT] flags for the method to call. Updated iff call devirtualized. +// pContextHandle -- [IN/OUT] context handle for the call. Updated iff call devirtualized. +// pExactContextHandle -- [OUT] updated context handle iff call devirtualized +// isLateDevirtualization -- if devirtualization is happening after importation +// isExplicitTailCalll -- [IN] true if we plan on using an explicit tail call +// ilOffset -- IL offset of the call // // Notes: -// This method primarily adds caller-supplied info to the inlArgInfo -// and sets up the lclVarInfo table. +// Virtual calls in IL will always "invoke" the base class method. // -// For args, the inlArgInfo records properties of the actual argument -// including the tree node that produces the arg value. This node is -// usually the tree node present at the call, but may also differ in -// various ways: -// - when the call arg is a GT_RET_EXPR, we search back through the ret -// expr chain for the actual node. Note this will either be the original -// call (which will be a failed inline by this point), or the return -// expression from some set of inlines. -// - when argument type casting is needed the necessary casts are added -// around the argument node. -// - if an argument can be simplified by folding then the node here is the -// folded value. +// This transformation looks for evidence that the type of 'this' +// in the call is exactly known, is a final class or would invoke +// a final method, and if that and other safety checks pan out, +// modifies the call and the call info to create a direct call. // -// The method may make observations that lead to marking this candidate as -// a failed inline. If this happens the initialization is abandoned immediately -// to try and reduce the jit time cost for a failed inline. - -void Compiler::impInlineInitVars(InlineInfo* pInlineInfo) +// This transformation is initially done in the importer and not +// in some subsequent optimization pass because we want it to be +// upstream of inline candidate identification. +// +// However, later phases may supply improved type information that +// can enable further devirtualization. We currently reinvoke this +// code after inlining, if the return value of the inlined call is +// the 'this obj' of a subsequent virtual call. +// +// If devirtualization succeeds and the call's this object is a +// (boxed) value type, the jit will ask the EE for the unboxed entry +// point. If this exists, the jit will invoke the unboxed entry +// on the box payload. In addition if the boxing operation is +// visible to the jit and the call is the only consmer of the box, +// the jit will try analyze the box to see if the call can be instead +// instead made on a local copy. If that is doable, the call is +// updated to invoke the unboxed entry on the local copy and the +// boxing operation is removed. +// +// When guarded devirtualization is enabled, this method will mark +// calls as guarded devirtualization candidates, if the type of `this` +// is not exactly known, and there is a plausible guess for the type. +void Compiler::impDevirtualizeCall(GenTreeCall* call, + CORINFO_RESOLVED_TOKEN* pResolvedToken, + CORINFO_METHOD_HANDLE* method, + unsigned* methodFlags, + CORINFO_CONTEXT_HANDLE* pContextHandle, + CORINFO_CONTEXT_HANDLE* pExactContextHandle, + bool isLateDevirtualization, + bool isExplicitTailCall, + IL_OFFSET ilOffset) { - assert(!compIsForInlining()); - - GenTreeCall* call = pInlineInfo->iciCall; - CORINFO_METHOD_INFO* methInfo = &pInlineInfo->inlineCandidateInfo->methInfo; - unsigned clsAttr = pInlineInfo->inlineCandidateInfo->clsAttr; - InlArgInfo* inlArgInfo = pInlineInfo->inlArgInfo; - InlLclVarInfo* lclVarInfo = pInlineInfo->lclVarInfo; - InlineResult* inlineResult = pInlineInfo->inlineResult; + assert(call != nullptr); + assert(method != nullptr); + assert(methodFlags != nullptr); + assert(pContextHandle != nullptr); - /* init the argument struct */ - memset(inlArgInfo, 0, (MAX_INL_ARGS + 1) * sizeof(inlArgInfo[0])); + // This should be a virtual vtable or virtual stub call. + // + assert(call->IsVirtual()); + assert(opts.OptimizationEnabled()); - unsigned ilArgCnt = 0; - for (CallArg& arg : call->gtArgs.Args()) +#if defined(DEBUG) + // Bail if devirt is disabled. + if (JitConfig.JitEnableDevirtualization() == 0) { - switch (arg.GetWellKnownArg()) - { - case WellKnownArg::ThisPointer: - inlArgInfo[ilArgCnt].argIsThis = true; - break; - case WellKnownArg::RetBuffer: - case WellKnownArg::InstParam: - // These do not appear in the table of inline arg info; do not include them - continue; - default: - break; - } + return; + } - arg.SetEarlyNode(gtFoldExpr(arg.GetEarlyNode())); - impInlineRecordArgInfo(pInlineInfo, &arg, ilArgCnt, inlineResult); + // Optionally, print info on devirtualization + Compiler* const rootCompiler = impInlineRoot(); + const bool doPrint = JitConfig.JitPrintDevirtualizedMethods().contains(rootCompiler->info.compMethodName, + rootCompiler->info.compClassName, + &rootCompiler->info.compMethodInfo->args); +#endif // DEBUG - if (inlineResult->IsFailure()) + // Fetch information about the virtual method we're calling. + CORINFO_METHOD_HANDLE baseMethod = *method; + unsigned baseMethodAttribs = *methodFlags; + + if (baseMethodAttribs == 0) + { + // For late devirt we may not have method attributes, so fetch them. + baseMethodAttribs = info.compCompHnd->getMethodAttribs(baseMethod); + } + else + { +#if defined(DEBUG) + // Validate that callInfo has up to date method flags + const DWORD freshBaseMethodAttribs = info.compCompHnd->getMethodAttribs(baseMethod); + + // All the base method attributes should agree, save that + // CORINFO_FLG_DONT_INLINE may have changed from 0 to 1 + // because of concurrent jitting activity. + // + // Note we don't look at this particular flag bit below, and + // later on (if we do try and inline) we will rediscover why + // the method can't be inlined, so there's no danger here in + // seeing this particular flag bit in different states between + // the cached and fresh values. + if ((freshBaseMethodAttribs & ~CORINFO_FLG_DONT_INLINE) != (baseMethodAttribs & ~CORINFO_FLG_DONT_INLINE)) { - return; + assert(!"mismatched method attributes"); } +#endif // DEBUG + } - ilArgCnt++; + // In R2R mode, we might see virtual stub calls to + // non-virtuals. For instance cases where the non-virtual method + // is in a different assembly but is called via CALLVIRT. For + // verison resilience we must allow for the fact that the method + // might become virtual in some update. + // + // In non-R2R modes CALLVIRT will be turned into a + // regular call+nullcheck upstream, so we won't reach this + // point. + if ((baseMethodAttribs & CORINFO_FLG_VIRTUAL) == 0) + { + assert(call->IsVirtualStub()); + assert(opts.IsReadyToRun()); + JITDUMP("\nimpDevirtualizeCall: [R2R] base method not virtual, sorry\n"); + return; } - /* Make sure we got the arg number right */ - assert(ilArgCnt == methInfo->args.totalILArgs()); + // Fetch information about the class that introduced the virtual method. + CORINFO_CLASS_HANDLE baseClass = info.compCompHnd->getMethodClass(baseMethod); + const DWORD baseClassAttribs = info.compCompHnd->getClassAttribs(baseClass); -#ifdef FEATURE_SIMD - bool foundSIMDType = pInlineInfo->hasSIMDTypeArgLocalOrReturn; -#endif // FEATURE_SIMD + // Is the call an interface call? + const bool isInterface = (baseClassAttribs & CORINFO_FLG_INTERFACE) != 0; - /* We have typeless opcodes, get type information from the signature */ + // See what we know about the type of 'this' in the call. + assert(call->gtArgs.HasThisPointer()); + CallArg* thisArg = call->gtArgs.GetThisArg(); + GenTree* thisObj = thisArg->GetEarlyNode()->gtEffectiveVal(false); + bool isExact = false; + bool objIsNonNull = false; + CORINFO_CLASS_HANDLE objClass = gtGetClassHandle(thisObj, &isExact, &objIsNonNull); - CallArg* thisArg = call->gtArgs.GetThisArg(); - if (thisArg != nullptr) + // Bail if we know nothing. + if (objClass == NO_CLASS_HANDLE) { - lclVarInfo[0].lclVerTypeInfo = verMakeTypeInfo(pInlineInfo->inlineCandidateInfo->clsHandle); - lclVarInfo[0].lclHasLdlocaOp = false; + JITDUMP("\nimpDevirtualizeCall: no type available (op=%s)\n", GenTree::OpName(thisObj->OperGet())); -#ifdef FEATURE_SIMD - // We always want to check isSIMDClass, since we want to set foundSIMDType (to increase - // the inlining multiplier) for anything in that assembly. - // But we only need to normalize it if it is a TYP_STRUCT - // (which we need to do even if we have already set foundSIMDType). - if (!foundSIMDType && isSIMDorHWSIMDClass(&(lclVarInfo[0].lclVerTypeInfo))) + // Don't try guarded devirtualiztion when we're doing late devirtualization. + // + if (isLateDevirtualization) { - foundSIMDType = true; + JITDUMP("No guarded devirt during late devirtualization\n"); + return; } -#endif // FEATURE_SIMD - var_types sigType = ((clsAttr & CORINFO_FLG_VALUECLASS) != 0) ? TYP_BYREF : TYP_REF; - lclVarInfo[0].lclTypeInfo = sigType; - - GenTree* thisArgNode = thisArg->GetEarlyNode(); + considerGuardedDevirtualization(call, ilOffset, isInterface, baseMethod, baseClass, pContextHandle); - assert(varTypeIsGC(thisArgNode->TypeGet()) || // "this" is managed - ((thisArgNode->TypeGet() == TYP_I_IMPL) && // "this" is unmgd but the method's class doesnt care - (clsAttr & CORINFO_FLG_VALUECLASS))); + return; + } - if (genActualType(thisArgNode->TypeGet()) != genActualType(sigType)) - { - if (sigType == TYP_REF) - { - /* The argument cannot be bashed into a ref (see bug 750871) */ - inlineResult->NoteFatal(InlineObservation::CALLSITE_ARG_NO_BASH_TO_REF); - return; - } + // If the objClass is sealed (final), then we may be able to devirtualize. + const DWORD objClassAttribs = info.compCompHnd->getClassAttribs(objClass); + const bool objClassIsFinal = (objClassAttribs & CORINFO_FLG_FINAL) != 0; - /* This can only happen with byrefs <-> ints/shorts */ +#if defined(DEBUG) + const char* callKind = isInterface ? "interface" : "virtual"; + const char* objClassNote = "[?]"; + const char* objClassName = "?objClass"; + const char* baseClassName = "?baseClass"; + const char* baseMethodName = "?baseMethod"; - assert(sigType == TYP_BYREF); - assert((genActualType(thisArgNode->TypeGet()) == TYP_I_IMPL) || (thisArgNode->TypeGet() == TYP_BYREF)); + if (verbose || doPrint) + { + objClassNote = isExact ? " [exact]" : objClassIsFinal ? " [final]" : ""; + objClassName = eeGetClassName(objClass); + baseClassName = eeGetClassName(baseClass); + baseMethodName = eeGetMethodName(baseMethod, nullptr); - lclVarInfo[0].lclVerTypeInfo = typeInfo(varType2tiType(TYP_I_IMPL)); + if (verbose) + { + printf("\nimpDevirtualizeCall: Trying to devirtualize %s call:\n" + " class for 'this' is %s%s (attrib %08x)\n" + " base method is %s::%s\n", + callKind, objClassName, objClassNote, objClassAttribs, baseClassName, baseMethodName); } } +#endif // defined(DEBUG) - /* Init the types of the arguments and make sure the types - * from the trees match the types in the signature */ - - CORINFO_ARG_LIST_HANDLE argLst; - argLst = methInfo->args.args; - - // TODO-ARGS: We can presumably just use type info stored in CallArgs - // instead of reiterating the signature. - unsigned i; - for (i = (thisArg ? 1 : 0); i < ilArgCnt; i++, argLst = info.compCompHnd->getArgNext(argLst)) + // See if the jit's best type for `obj` is an interface. + // See for instance System.ValueTuple`8::GetHashCode, where lcl 0 is System.IValueTupleInternal + // IL_021d: ldloc.0 + // IL_021e: callvirt instance int32 System.Object::GetHashCode() + // + // If so, we can't devirtualize, but we may be able to do guarded devirtualization. + // + if ((objClassAttribs & CORINFO_FLG_INTERFACE) != 0) { - var_types sigType = (var_types)eeGetArgType(argLst, &methInfo->args); - - lclVarInfo[i].lclVerTypeInfo = verParseArgSigToTypeInfo(&methInfo->args, argLst); - -#ifdef FEATURE_SIMD - if ((!foundSIMDType || (sigType == TYP_STRUCT)) && isSIMDorHWSIMDClass(&(lclVarInfo[i].lclVerTypeInfo))) + // Don't try guarded devirtualiztion when we're doing late devirtualization. + // + if (isLateDevirtualization) { - // If this is a SIMD class (i.e. in the SIMD assembly), then we will consider that we've - // found a SIMD type, even if this may not be a type we recognize (the assumption is that - // it is likely to use a SIMD type, and therefore we want to increase the inlining multiplier). - foundSIMDType = true; - if (sigType == TYP_STRUCT) - { - var_types structType = impNormStructType(lclVarInfo[i].lclVerTypeInfo.GetClassHandle()); - sigType = structType; - } + JITDUMP("No guarded devirt during late devirtualization\n"); + return; } -#endif // FEATURE_SIMD - lclVarInfo[i].lclTypeInfo = sigType; - lclVarInfo[i].lclHasLdlocaOp = false; + considerGuardedDevirtualization(call, ilOffset, isInterface, baseMethod, baseClass, pContextHandle); + return; + } - // Does the tree type match the signature type? + // If we get this far, the jit has a lower bound class type for the `this` object being used for dispatch. + // It may or may not know enough to devirtualize... + if (isInterface) + { + assert(call->IsVirtualStub()); + JITDUMP("--- base class is interface\n"); + } - GenTree* inlArgNode = inlArgInfo[i].arg->GetNode(); + // Fetch the method that would be called based on the declared type of 'this', + // and prepare to fetch the method attributes. + // + CORINFO_DEVIRTUALIZATION_INFO dvInfo; + dvInfo.virtualMethod = baseMethod; + dvInfo.objClass = objClass; + dvInfo.context = *pContextHandle; + dvInfo.detail = CORINFO_DEVIRTUALIZATION_UNKNOWN; + dvInfo.pResolvedTokenVirtualMethod = pResolvedToken; - if (sigType == inlArgNode->gtType) - { - continue; - } + info.compCompHnd->resolveVirtualMethod(&dvInfo); - assert(impCheckImplicitArgumentCoercion(sigType, inlArgNode->gtType)); - assert(!varTypeIsStruct(inlArgNode->gtType) && !varTypeIsStruct(sigType)); + CORINFO_METHOD_HANDLE derivedMethod = dvInfo.devirtualizedMethod; + CORINFO_CONTEXT_HANDLE exactContext = dvInfo.exactContext; + CORINFO_CLASS_HANDLE derivedClass = NO_CLASS_HANDLE; + CORINFO_RESOLVED_TOKEN* pDerivedResolvedToken = &dvInfo.resolvedTokenDevirtualizedMethod; - // In valid IL, this can only happen for short integer types or byrefs <-> [native] ints, - // but in bad IL cases with caller-callee signature mismatches we can see other types. - // Intentionally reject cases with mismatches so the jit is more flexible when - // encountering bad IL. + if (derivedMethod != nullptr) + { + assert(exactContext != nullptr); + assert(((size_t)exactContext & CORINFO_CONTEXTFLAGS_MASK) == CORINFO_CONTEXTFLAGS_CLASS); + derivedClass = (CORINFO_CLASS_HANDLE)((size_t)exactContext & ~CORINFO_CONTEXTFLAGS_MASK); + } - bool isPlausibleTypeMatch = (genActualType(sigType) == genActualType(inlArgNode->gtType)) || - (genActualTypeIsIntOrI(sigType) && inlArgNode->gtType == TYP_BYREF) || - (sigType == TYP_BYREF && genActualTypeIsIntOrI(inlArgNode->gtType)); + DWORD derivedMethodAttribs = 0; + bool derivedMethodIsFinal = false; + bool canDevirtualize = false; - if (!isPlausibleTypeMatch) +#if defined(DEBUG) + const char* derivedClassName = "?derivedClass"; + const char* derivedMethodName = "?derivedMethod"; + const char* note = "inexact or not final"; +#endif + + // If we failed to get a method handle, we can't directly devirtualize. + // + // This can happen when prejitting, if the devirtualization crosses + // servicing bubble boundaries, or if objClass is a shared class. + // + if (derivedMethod == nullptr) + { + JITDUMP("--- no derived method: %s\n", devirtualizationDetailToString(dvInfo.detail)); + } + else + { + // Fetch method attributes to see if method is marked final. + derivedMethodAttribs = info.compCompHnd->getMethodAttribs(derivedMethod); + derivedMethodIsFinal = ((derivedMethodAttribs & CORINFO_FLG_FINAL) != 0); + +#if defined(DEBUG) + if (isExact) { - inlineResult->NoteFatal(InlineObservation::CALLSITE_ARG_TYPES_INCOMPATIBLE); - return; + note = "exact"; + } + else if (objClassIsFinal) + { + note = "final class"; + } + else if (derivedMethodIsFinal) + { + note = "final method"; } - // The same size but different type of the arguments. - GenTree** pInlArgNode = &inlArgInfo[i].arg->EarlyNodeRef(); - - // Is it a narrowing or widening cast? - // Widening casts are ok since the value computed is already - // normalized to an int (on the IL stack) - if (genTypeSize(inlArgNode->gtType) >= genTypeSize(sigType)) + if (verbose || doPrint) { - if (sigType == TYP_BYREF) - { - lclVarInfo[i].lclVerTypeInfo = typeInfo(varType2tiType(TYP_I_IMPL)); - } - else if (inlArgNode->gtType == TYP_BYREF) + derivedMethodName = eeGetMethodName(derivedMethod, nullptr); + derivedClassName = eeGetClassName(derivedClass); + if (verbose) { - assert(varTypeIsIntOrI(sigType)); - - /* If possible bash the BYREF to an int */ - if (inlArgNode->IsLocalAddrExpr() != nullptr) - { - inlArgNode->gtType = TYP_I_IMPL; - lclVarInfo[i].lclVerTypeInfo = typeInfo(varType2tiType(TYP_I_IMPL)); - } - else - { - // Arguments 'int <- byref' cannot be changed - inlineResult->NoteFatal(InlineObservation::CALLSITE_ARG_NO_BASH_TO_INT); - return; - } + printf(" devirt to %s::%s -- %s\n", derivedClassName, derivedMethodName, note); + gtDispTree(call); } - else if (genTypeSize(sigType) < TARGET_POINTER_SIZE) - { - // Narrowing cast. - if (inlArgNode->OperIs(GT_LCL_VAR)) - { - const unsigned lclNum = inlArgNode->AsLclVarCommon()->GetLclNum(); - if (!lvaTable[lclNum].lvNormalizeOnLoad() && sigType == lvaGetRealType(lclNum)) - { - // We don't need to insert a cast here as the variable - // was assigned a normalized value of the right type. - continue; - } - } + } +#endif // defined(DEBUG) - inlArgNode = gtNewCastNode(TYP_INT, inlArgNode, false, sigType); + canDevirtualize = isExact || objClassIsFinal || (!isInterface && derivedMethodIsFinal); + } - inlArgInfo[i].argIsLclVar = false; - // Try to fold the node in case we have constant arguments. - if (inlArgInfo[i].argIsInvariant) - { - inlArgNode = gtFoldExpr(inlArgNode); - } - *pInlArgNode = inlArgNode; - } -#ifdef TARGET_64BIT - else if (genTypeSize(genActualType(inlArgNode->gtType)) < genTypeSize(sigType)) - { - // This should only happen for int -> native int widening - inlArgNode = gtNewCastNode(genActualType(sigType), inlArgNode, false, sigType); + // We still might be able to do a guarded devirtualization. + // Note the call might be an interface call or a virtual call. + // + if (!canDevirtualize) + { + JITDUMP(" Class not final or exact%s\n", isInterface ? "" : ", and method not final"); - inlArgInfo[i].argIsLclVar = false; +#if defined(DEBUG) + // If we know the object type exactly, we generally expect we can devirtualize. + // (don't when doing late devirt as we won't have an owner type (yet)) + // + if (!isLateDevirtualization && (isExact || objClassIsFinal) && JitConfig.JitNoteFailedExactDevirtualization()) + { + printf("@@@ Exact/Final devirt failure in %s at [%06u] $ %s\n", info.compFullName, dspTreeID(call), + devirtualizationDetailToString(dvInfo.detail)); + } +#endif - // Try to fold the node in case we have constant arguments. - if (inlArgInfo[i].argIsInvariant) - { - inlArgNode = gtFoldExpr(inlArgNode); - } - *pInlArgNode = inlArgNode; - } -#endif // TARGET_64BIT + // Don't try guarded devirtualiztion if we're doing late devirtualization. + // + if (isLateDevirtualization) + { + JITDUMP("No guarded devirt during late devirtualization\n"); + return; } + + considerGuardedDevirtualization(call, ilOffset, isInterface, baseMethod, baseClass, pContextHandle); + return; } - /* Init the types of the local variables */ + // All checks done. Time to transform the call. + // + // We should always have an exact class context. + // + // Note that wouldnt' be true if the runtime side supported array interface devirt, + // the resulting method would be a generic method of the non-generic SZArrayHelper class. + // + assert(canDevirtualize); - CORINFO_ARG_LIST_HANDLE localsSig; - localsSig = methInfo->locals.args; + JITDUMP(" %s; can devirtualize\n", note); - for (i = 0; i < methInfo->locals.numArgs; i++) + // Make the updates. + call->gtFlags &= ~GTF_CALL_VIRT_VTABLE; + call->gtFlags &= ~GTF_CALL_VIRT_STUB; + call->gtCallMethHnd = derivedMethod; + call->gtCallType = CT_USER_FUNC; + call->gtControlExpr = nullptr; + call->gtCallMoreFlags |= GTF_CALL_M_DEVIRTUALIZED; + + // Virtual calls include an implicit null check, which we may + // now need to make explicit. + if (!objIsNonNull) { - bool isPinned; - var_types type = (var_types)eeGetArgType(localsSig, &methInfo->locals, &isPinned); + call->gtFlags |= GTF_CALL_NULLCHECK; + } - lclVarInfo[i + ilArgCnt].lclHasLdlocaOp = false; - lclVarInfo[i + ilArgCnt].lclTypeInfo = type; + // Clear the inline candidate info (may be non-null since + // it's a union field used for other things by virtual + // stubs) + call->gtInlineCandidateInfo = nullptr; + call->gtCallMoreFlags &= ~GTF_CALL_M_HAS_LATE_DEVIRT_INFO; - if (varTypeIsGC(type)) +#if defined(DEBUG) + if (verbose) + { + printf("... after devirt...\n"); + gtDispTree(call); + } + + if (doPrint) + { + printf("Devirtualized %s call to %s:%s; now direct call to %s:%s [%s]\n", callKind, baseClassName, + baseMethodName, derivedClassName, derivedMethodName, note); + } + + // If we successfully devirtualized based on an exact or final class, + // and we have dynamic PGO data describing the likely class, make sure they agree. + // + // If pgo source is not dynamic we may see likely classes from other versions of this code + // where types had different properties. + // + // If method is an inlinee we may be specializing to a class that wasn't seen at runtime. + // + const bool canSensiblyCheck = + (isExact || objClassIsFinal) && (fgPgoSource == ICorJitInfo::PgoSource::Dynamic) && !compIsForInlining(); + if (JitConfig.JitCrossCheckDevirtualizationAndPGO() && canSensiblyCheck) + { + // We only can handle a single likely class for now + const int maxLikelyClasses = 1; + LikelyClassMethodRecord likelyClasses[maxLikelyClasses]; + + UINT32 numberOfClasses = + getLikelyClasses(likelyClasses, maxLikelyClasses, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset); + UINT32 likelihood = likelyClasses[0].likelihood; + + CORINFO_CLASS_HANDLE likelyClass = (CORINFO_CLASS_HANDLE)likelyClasses[0].handle; + + if (numberOfClasses > 0) { - if (isPinned) + // PGO had better agree the class we devirtualized to is plausible. + // + if (likelyClass != derivedClass) { - JITDUMP("Inlinee local #%02u is pinned\n", i); - lclVarInfo[i + ilArgCnt].lclIsPinned = true; - - // Pinned locals may cause inlines to fail. - inlineResult->Note(InlineObservation::CALLEE_HAS_PINNED_LOCALS); - if (inlineResult->IsFailure()) + // Managed type system may report different addresses for a class handle + // at different times....? + // + // Also, AOT may have a more nuanced notion of class equality. + // + if (!opts.jitFlags->IsSet(JitFlags::JIT_FLAG_PREJIT)) { - return; + bool mismatch = true; + + // derivedClass will be the introducer of derived method, so it's possible + // likelyClass is a non-overriding subclass. Check up the hierarchy. + // + CORINFO_CLASS_HANDLE parentClass = likelyClass; + while (parentClass != NO_CLASS_HANDLE) + { + if (parentClass == derivedClass) + { + mismatch = false; + break; + } + + parentClass = info.compCompHnd->getParentType(parentClass); + } + + if (mismatch || (numberOfClasses != 1) || (likelihood != 100)) + { + printf("@@@ Likely %p (%s) != Derived %p (%s) [n=%u, l=%u, il=%u] in %s \n", likelyClass, + eeGetClassName(likelyClass), derivedClass, eeGetClassName(derivedClass), numberOfClasses, + likelihood, ilOffset, info.compFullName); + } + + assert(!(mismatch || (numberOfClasses != 1) || (likelihood != 100))); } } - - pInlineInfo->numberOfGcRefLocals++; } - else if (isPinned) + } +#endif // defined(DEBUG) + + // If the 'this' object is a value class, see if we can rework the call to invoke the + // unboxed entry. This effectively inlines the normally un-inlineable wrapper stub + // and exposes the potentially inlinable unboxed entry method. + // + // We won't optimize explicit tail calls, as ensuring we get the right tail call info + // is tricky (we'd need to pass an updated sig and resolved token back to some callers). + // + // Note we may not have a derived class in some cases (eg interface call on an array) + // + if (info.compCompHnd->isValueClass(derivedClass)) + { + if (isExplicitTailCall) { - JITDUMP("Ignoring pin on inlinee local #%02u -- not a GC type\n", i); + JITDUMP("Have a direct explicit tail call to boxed entry point; can't optimize further\n"); } + else + { + JITDUMP("Have a direct call to boxed entry point. Trying to optimize to call an unboxed entry point\n"); - lclVarInfo[i + ilArgCnt].lclVerTypeInfo = verParseArgSigToTypeInfo(&methInfo->locals, localsSig); + // Note for some shared methods the unboxed entry point requires an extra parameter. + bool requiresInstMethodTableArg = false; + CORINFO_METHOD_HANDLE unboxedEntryMethod = + info.compCompHnd->getUnboxedEntry(derivedMethod, &requiresInstMethodTableArg); - // If this local is a struct type with GC fields, inform the inliner. It may choose to bail - // out on the inline. - if (type == TYP_STRUCT) - { - CORINFO_CLASS_HANDLE lclHandle = lclVarInfo[i + ilArgCnt].lclVerTypeInfo.GetClassHandle(); - DWORD typeFlags = info.compCompHnd->getClassAttribs(lclHandle); - if ((typeFlags & CORINFO_FLG_CONTAINS_GC_PTR) != 0) + if (unboxedEntryMethod != nullptr) { - inlineResult->Note(InlineObservation::CALLEE_HAS_GC_STRUCT); - if (inlineResult->IsFailure()) + bool optimizedTheBox = false; + + // If the 'this' object is a local box, see if we can revise things + // to not require boxing. + // + if (thisObj->IsBoxedValue() && !isExplicitTailCall) { - return; + // Since the call is the only consumer of the box, we know the box can't escape + // since it is being passed an interior pointer. + // + // So, revise the box to simply create a local copy, use the address of that copy + // as the this pointer, and update the entry point to the unboxed entry. + // + // Ideally, we then inline the boxed method and and if it turns out not to modify + // the copy, we can undo the copy too. + if (requiresInstMethodTableArg) + { + // Perform a trial box removal and ask for the type handle tree that fed the box. + // + JITDUMP("Unboxed entry needs method table arg...\n"); + GenTree* methodTableArg = + gtTryRemoveBoxUpstreamEffects(thisObj, BR_DONT_REMOVE_WANT_TYPE_HANDLE); + + if (methodTableArg != nullptr) + { + // If that worked, turn the box into a copy to a local var + // + JITDUMP("Found suitable method table arg tree [%06u]\n", dspTreeID(methodTableArg)); + GenTree* localCopyThis = gtTryRemoveBoxUpstreamEffects(thisObj, BR_MAKE_LOCAL_COPY); + + if (localCopyThis != nullptr) + { + // Pass the local var as this and the type handle as a new arg + // + JITDUMP("Success! invoking unboxed entry point on local copy, and passing method table " + "arg\n"); + // TODO-CallArgs-REVIEW: Might discard commas otherwise? + assert(thisObj == thisArg->GetEarlyNode()); + thisArg->SetEarlyNode(localCopyThis); + call->gtCallMoreFlags |= GTF_CALL_M_UNBOXED; + + call->gtArgs.InsertInstParam(this, methodTableArg); + + call->gtCallMethHnd = unboxedEntryMethod; + derivedMethod = unboxedEntryMethod; + pDerivedResolvedToken = &dvInfo.resolvedTokenDevirtualizedUnboxedMethod; + + // Method attributes will differ because unboxed entry point is shared + // + const DWORD unboxedMethodAttribs = + info.compCompHnd->getMethodAttribs(unboxedEntryMethod); + JITDUMP("Updating method attribs from 0x%08x to 0x%08x\n", derivedMethodAttribs, + unboxedMethodAttribs); + derivedMethodAttribs = unboxedMethodAttribs; + optimizedTheBox = true; + } + else + { + JITDUMP("Sorry, failed to undo the box -- can't convert to local copy\n"); + } + } + else + { + JITDUMP("Sorry, failed to undo the box -- can't find method table arg\n"); + } + } + else + { + JITDUMP("Found unboxed entry point, trying to simplify box to a local copy\n"); + GenTree* localCopyThis = gtTryRemoveBoxUpstreamEffects(thisObj, BR_MAKE_LOCAL_COPY); + + if (localCopyThis != nullptr) + { + JITDUMP("Success! invoking unboxed entry point on local copy\n"); + assert(thisObj == thisArg->GetEarlyNode()); + // TODO-CallArgs-REVIEW: Might discard commas otherwise? + thisArg->SetEarlyNode(localCopyThis); + call->gtCallMethHnd = unboxedEntryMethod; + call->gtCallMoreFlags |= GTF_CALL_M_UNBOXED; + derivedMethod = unboxedEntryMethod; + pDerivedResolvedToken = &dvInfo.resolvedTokenDevirtualizedUnboxedMethod; + + optimizedTheBox = true; + } + else + { + JITDUMP("Sorry, failed to undo the box\n"); + } + } + + if (optimizedTheBox) + { + +#if FEATURE_TAILCALL_OPT + if (call->IsImplicitTailCall()) + { + JITDUMP("Clearing the implicit tail call flag\n"); + + // If set, we clear the implicit tail call flag + // as we just introduced a new address taken local variable + // + call->gtCallMoreFlags &= ~GTF_CALL_M_IMPLICIT_TAILCALL; + } +#endif // FEATURE_TAILCALL_OPT + } } - // Do further notification in the case where the call site is rare; some policies do - // not track the relative hotness of call sites for "always" inline cases. - if (pInlineInfo->iciBlock->isRunRarely()) + if (!optimizedTheBox) { - inlineResult->Note(InlineObservation::CALLSITE_RARE_GC_STRUCT); - if (inlineResult->IsFailure()) + // If we get here, we have a boxed value class that either wasn't boxed + // locally, or was boxed locally but we were unable to remove the box for + // various reasons. + // + // We can still update the call to invoke the unboxed entry, if the + // boxed value is simple. + // + if (requiresInstMethodTableArg) { + // Get the method table from the boxed object. + // + // TODO-CallArgs-REVIEW: Use thisObj here? Differs by gtEffectiveVal. + GenTree* const clonedThisArg = gtClone(thisArg->GetEarlyNode()); - return; + if (clonedThisArg == nullptr) + { + JITDUMP( + "unboxed entry needs MT arg, but `this` was too complex to clone. Deferring update.\n"); + } + else + { + JITDUMP("revising call to invoke unboxed entry with additional method table arg\n"); + + GenTree* const methodTableArg = gtNewMethodTableLookup(clonedThisArg); + + // Update the 'this' pointer to refer to the box payload + // + GenTree* const payloadOffset = gtNewIconNode(TARGET_POINTER_SIZE, TYP_I_IMPL); + GenTree* const boxPayload = + gtNewOperNode(GT_ADD, TYP_BYREF, thisArg->GetEarlyNode(), payloadOffset); + + assert(thisObj == thisArg->GetEarlyNode()); + thisArg->SetEarlyNode(boxPayload); + call->gtCallMethHnd = unboxedEntryMethod; + call->gtCallMoreFlags |= GTF_CALL_M_UNBOXED; + + // Method attributes will differ because unboxed entry point is shared + // + const DWORD unboxedMethodAttribs = info.compCompHnd->getMethodAttribs(unboxedEntryMethod); + JITDUMP("Updating method attribs from 0x%08x to 0x%08x\n", derivedMethodAttribs, + unboxedMethodAttribs); + derivedMethod = unboxedEntryMethod; + pDerivedResolvedToken = &dvInfo.resolvedTokenDevirtualizedUnboxedMethod; + derivedMethodAttribs = unboxedMethodAttribs; + + call->gtArgs.InsertInstParam(this, methodTableArg); + } } - } - } - } + else + { + JITDUMP("revising call to invoke unboxed entry\n"); - localsSig = info.compCompHnd->getArgNext(localsSig); + GenTree* const payloadOffset = gtNewIconNode(TARGET_POINTER_SIZE, TYP_I_IMPL); + GenTree* const boxPayload = + gtNewOperNode(GT_ADD, TYP_BYREF, thisArg->GetEarlyNode(), payloadOffset); -#ifdef FEATURE_SIMD - if ((!foundSIMDType || (type == TYP_STRUCT)) && isSIMDorHWSIMDClass(&(lclVarInfo[i + ilArgCnt].lclVerTypeInfo))) - { - foundSIMDType = true; - if (type == TYP_STRUCT) + thisArg->SetEarlyNode(boxPayload); + call->gtCallMethHnd = unboxedEntryMethod; + call->gtCallMoreFlags |= GTF_CALL_M_UNBOXED; + derivedMethod = unboxedEntryMethod; + pDerivedResolvedToken = &dvInfo.resolvedTokenDevirtualizedUnboxedMethod; + } + } + } + else { - var_types structType = impNormStructType(lclVarInfo[i + ilArgCnt].lclVerTypeInfo.GetClassHandle()); - lclVarInfo[i + ilArgCnt].lclTypeInfo = structType; + // Many of the low-level methods on value classes won't have unboxed entries, + // as they need access to the type of the object. + // + // Note this may be a cue for us to stack allocate the boxed object, since + // we probably know that these objects don't escape. + JITDUMP("Sorry, failed to find unboxed entry point\n"); } } -#endif // FEATURE_SIMD } -#ifdef FEATURE_SIMD - if (!foundSIMDType && (call->AsCall()->gtRetClsHnd != nullptr) && isSIMDorHWSIMDClass(call->AsCall()->gtRetClsHnd)) + // Need to update call info too. + // + *method = derivedMethod; + *methodFlags = derivedMethodAttribs; + + // Update context handle + // + *pContextHandle = MAKE_METHODCONTEXT(derivedMethod); + + // Update exact context handle. + // + if (pExactContextHandle != nullptr) { - foundSIMDType = true; + *pExactContextHandle = MAKE_CLASSCONTEXT(derivedClass); } - pInlineInfo->hasSIMDTypeArgLocalOrReturn = foundSIMDType; -#endif // FEATURE_SIMD + +#ifdef FEATURE_READYTORUN + if (opts.IsReadyToRun()) + { + // For R2R, getCallInfo triggers bookkeeping on the zap + // side and acquires the actual symbol to call so we need to call it here. + + // Look up the new call info. + CORINFO_CALL_INFO derivedCallInfo; + eeGetCallInfo(pDerivedResolvedToken, nullptr, CORINFO_CALLINFO_ALLOWINSTPARAM, &derivedCallInfo); + + // Update the call. + call->gtCallMoreFlags &= ~GTF_CALL_M_VIRTSTUB_REL_INDIRECT; + call->gtCallMoreFlags &= ~GTF_CALL_M_R2R_REL_INDIRECT; + call->setEntryPoint(derivedCallInfo.codePointerLookup.constLookup); + } +#endif // FEATURE_READYTORUN } //------------------------------------------------------------------------ -// impInlineFetchLocal: get a local var that represents an inlinee local +// impConsiderCallProbe: Consider whether a call should get a histogram probe +// and mark it if so. // // Arguments: -// lclNum -- number of the inlinee local -// reason -- debug string describing purpose of the local var +// call - The call +// ilOffset - The precise IL offset of the call // // Returns: -// Number of the local to use +// True if the call was marked such that we will add a class or method probe for it. // -// Notes: -// This method is invoked only for locals actually used in the -// inlinee body. +bool Compiler::impConsiderCallProbe(GenTreeCall* call, IL_OFFSET ilOffset) +{ + // Possibly instrument. Note for OSR+PGO we will instrument when + // optimizing and (currently) won't devirtualize. We may want + // to revisit -- if we can devirtualize we should be able to + // suppress the probe. + // + // We strip BBINSTR from inlinees currently, so we'll only + // do this for the root method calls. + // + if (!opts.jitFlags->IsSet(JitFlags::JIT_FLAG_BBINSTR)) + { + return false; + } + + assert(opts.OptimizationDisabled() || opts.IsOSR()); + assert(!compIsForInlining()); + + // During importation, optionally flag this block as one that + // contains calls requiring class profiling. Ideally perhaps + // we'd just keep track of the calls themselves, so we don't + // have to search for them later. + // + if (compClassifyGDVProbeType(call) == GDVProbeType::None) + { + return false; + } + + JITDUMP("\n ... marking [%06u] in " FMT_BB " for method/class profile instrumentation\n", dspTreeID(call), + compCurBB->bbNum); + HandleHistogramProfileCandidateInfo* pInfo = new (this, CMK_Inlining) HandleHistogramProfileCandidateInfo; + + // Record some info needed for the class profiling probe. + // + pInfo->ilOffset = ilOffset; + pInfo->probeIndex = info.compHandleHistogramProbeCount++; + call->gtHandleHistogramProfileCandidateInfo = pInfo; + + // Flag block as needing scrutiny + // + compCurBB->bbFlags |= BBF_HAS_HISTOGRAM_PROFILE; + return true; +} + +//------------------------------------------------------------------------ +// compClassifyGDVProbeType: +// Classify the type of GDV probe to use for a call site. // -// Allocates a new temp if necessary, and copies key properties -// over from the inlinee local var info. +// Arguments: +// call - The call +// +// Returns: +// The type of probe to use. +// +Compiler::GDVProbeType Compiler::compClassifyGDVProbeType(GenTreeCall* call) +{ + if (call->gtCallType == CT_INDIRECT) + { + return GDVProbeType::None; + } -unsigned Compiler::impInlineFetchLocal(unsigned lclNum DEBUGARG(const char* reason)) + if (!opts.jitFlags->IsSet(JitFlags::JIT_FLAG_BBINSTR) || opts.jitFlags->IsSet(JitFlags::JIT_FLAG_PREJIT)) + { + return GDVProbeType::None; + } + + bool createTypeHistogram = false; + if (JitConfig.JitClassProfiling() > 0) + { + createTypeHistogram = call->IsVirtualStub() || call->IsVirtualVtable(); + + // Cast helpers may conditionally (depending on whether the class is + // exact or not) have probes. For those helpers we do not use this + // function to classify the probe type until after we have decided on + // whether we probe them or not. + createTypeHistogram = createTypeHistogram || (impIsCastHelperEligibleForClassProbe(call) && + (call->gtHandleHistogramProfileCandidateInfo != nullptr)); + } + + bool createMethodHistogram = ((JitConfig.JitDelegateProfiling() > 0) && call->IsDelegateInvoke()) || + ((JitConfig.JitVTableProfiling() > 0) && call->IsVirtualVtable()); + + if (createTypeHistogram && createMethodHistogram) + { + return GDVProbeType::MethodAndClassProfile; + } + + if (createTypeHistogram) + { + return GDVProbeType::ClassProfile; + } + + if (createMethodHistogram) + { + return GDVProbeType::MethodProfile; + } + + return GDVProbeType::None; +} + +//------------------------------------------------------------------------ +// impGetSpecialIntrinsicExactReturnType: Look for special cases where a call +// to an intrinsic returns an exact type +// +// Arguments: +// methodHnd -- handle for the special intrinsic method +// +// Returns: +// Exact class handle returned by the intrinsic call, if known. +// Nullptr if not known, or not likely to lead to beneficial optimization. +CORINFO_CLASS_HANDLE Compiler::impGetSpecialIntrinsicExactReturnType(CORINFO_METHOD_HANDLE methodHnd) { - assert(compIsForInlining()); + JITDUMP("Special intrinsic: looking for exact type returned by %s\n", eeGetMethodFullName(methodHnd)); - unsigned tmpNum = impInlineInfo->lclTmpNum[lclNum]; + CORINFO_CLASS_HANDLE result = nullptr; - if (tmpNum == BAD_VAR_NUM) + // See what intrinsic we have... + const NamedIntrinsic ni = lookupNamedIntrinsic(methodHnd); + switch (ni) { - const InlLclVarInfo& inlineeLocal = impInlineInfo->lclVarInfo[lclNum + impInlineInfo->argCnt]; - const var_types lclTyp = inlineeLocal.lclTypeInfo; - - // The lifetime of this local might span multiple BBs. - // So it is a long lifetime local. - impInlineInfo->lclTmpNum[lclNum] = tmpNum = lvaGrabTemp(false DEBUGARG(reason)); - - // Copy over key info - lvaTable[tmpNum].lvType = lclTyp; - lvaTable[tmpNum].lvHasLdAddrOp = inlineeLocal.lclHasLdlocaOp; - lvaTable[tmpNum].lvPinned = inlineeLocal.lclIsPinned; - lvaTable[tmpNum].lvHasILStoreOp = inlineeLocal.lclHasStlocOp; - lvaTable[tmpNum].lvHasMultipleILStoreOp = inlineeLocal.lclHasMultipleStlocOp; - - // Copy over class handle for ref types. Note this may be a - // shared type -- someday perhaps we can get the exact - // signature and pass in a more precise type. - if (lclTyp == TYP_REF) + case NI_System_Collections_Generic_Comparer_get_Default: + case NI_System_Collections_Generic_EqualityComparer_get_Default: { - assert(lvaTable[tmpNum].lvSingleDef == 0); + // Expect one class generic parameter; figure out which it is. + CORINFO_SIG_INFO sig; + info.compCompHnd->getMethodSig(methodHnd, &sig); + assert(sig.sigInst.classInstCount == 1); + CORINFO_CLASS_HANDLE typeHnd = sig.sigInst.classInst[0]; + assert(typeHnd != nullptr); + + // Lookup can incorrect when we have __Canon as it won't appear + // to implement any interface types. + // + // And if we do not have a final type, devirt & inlining is + // unlikely to result in much simplification. + // + // We can use CORINFO_FLG_FINAL to screen out both of these cases. + const DWORD typeAttribs = info.compCompHnd->getClassAttribs(typeHnd); + const bool isFinalType = ((typeAttribs & CORINFO_FLG_FINAL) != 0); - lvaTable[tmpNum].lvSingleDef = !inlineeLocal.lclHasMultipleStlocOp && !inlineeLocal.lclHasLdlocaOp; - if (lvaTable[tmpNum].lvSingleDef) + if (isFinalType) { - JITDUMP("Marked V%02u as a single def temp\n", tmpNum); + if (ni == NI_System_Collections_Generic_EqualityComparer_get_Default) + { + result = info.compCompHnd->getDefaultEqualityComparerClass(typeHnd); + } + else + { + assert(ni == NI_System_Collections_Generic_Comparer_get_Default); + result = info.compCompHnd->getDefaultComparerClass(typeHnd); + } + JITDUMP("Special intrinsic for type %s: return type is %s\n", eeGetClassName(typeHnd), + result != nullptr ? eeGetClassName(result) : "unknown"); } - - lvaSetClass(tmpNum, inlineeLocal.lclVerTypeInfo.GetClassHandleForObjRef()); - } - - if (inlineeLocal.lclVerTypeInfo.IsStruct()) - { - if (varTypeIsStruct(lclTyp)) + else { - lvaSetStruct(tmpNum, inlineeLocal.lclVerTypeInfo.GetClassHandle(), true /* unsafe value cls check */); + JITDUMP("Special intrinsic for type %s: type not final, so deferring opt\n", eeGetClassName(typeHnd)); } - } -#ifdef DEBUG - // Sanity check that we're properly prepared for gc ref locals. - if (varTypeIsGC(lclTyp)) - { - // Since there are gc locals we should have seen them earlier - // and if there was a return value, set up the spill temp. - assert(impInlineInfo->HasGcRefLocals()); - assert((info.compRetNativeType == TYP_VOID) || fgNeedReturnSpillTemp()); + break; } - else + + default: { - // Make sure all pinned locals count as gc refs. - assert(!inlineeLocal.lclIsPinned); + JITDUMP("This special intrinsic not handled, sorry...\n"); + break; } -#endif // DEBUG } - return tmpNum; + return result; } //------------------------------------------------------------------------ -// impInlineFetchArg: return tree node for argument value in an inlinee +// impAllocateMethodPointerInfo: create methodPointerInfo into jit-allocated memory and init it. // // Arguments: -// lclNum -- argument number in inlinee IL -// inlArgInfo -- argument info for inlinee -// lclVarInfo -- var info for inlinee -// -// Returns: -// Tree for the argument's value. Often an inlinee-scoped temp -// GT_LCL_VAR but can be other tree kinds, if the argument -// expression from the caller can be directly substituted into the -// inlinee body. -// -// Notes: -// Must be used only for arguments -- use impInlineFetchLocal for -// inlinee locals. -// -// Direct substitution is performed when the formal argument cannot -// change value in the inlinee body (no starg or ldarga), and the -// actual argument expression's value cannot be changed if it is -// substituted it into the inlinee body. -// -// Even if an inlinee-scoped temp is returned here, it may later be -// "bashed" to a caller-supplied tree when arguments are actually -// passed (see fgInlinePrependStatements). Bashing can happen if -// the argument ends up being single use and other conditions are -// met. So the contents of the tree returned here may not end up -// being the ones ultimately used for the argument. +// token - init value for the allocated token. +// tokenConstrained - init value for the constraint associated with the token // -// This method will side effect inlArgInfo. It should only be called -// for actual uses of the argument in the inlinee. - -GenTree* Compiler::impInlineFetchArg(unsigned lclNum, InlArgInfo* inlArgInfo, InlLclVarInfo* lclVarInfo) +// Return Value: +// pointer to token into jit-allocated memory. +methodPointerInfo* Compiler::impAllocateMethodPointerInfo(const CORINFO_RESOLVED_TOKEN& token, mdToken tokenConstrained) { - // Cache the relevant arg and lcl info for this argument. - // We will modify argInfo but not lclVarInfo. - InlArgInfo& argInfo = inlArgInfo[lclNum]; - const InlLclVarInfo& lclInfo = lclVarInfo[lclNum]; - const bool argCanBeModified = argInfo.argHasLdargaOp || argInfo.argHasStargOp; - const var_types lclTyp = lclInfo.lclTypeInfo; - GenTree* op1 = nullptr; + methodPointerInfo* memory = getAllocator(CMK_Unknown).allocate(1); + memory->m_token = token; + memory->m_tokenConstraint = tokenConstrained; + return memory; +} - GenTree* argNode = argInfo.arg->GetNode(); - assert(!argNode->OperIs(GT_RET_EXPR)); +//------------------------------------------------------------------------ +// SpillRetExprHelper: iterate through arguments tree and spill ret_expr to local variables. +// +class SpillRetExprHelper +{ +public: + SpillRetExprHelper(Compiler* comp) : comp(comp) + { + } - if (argInfo.argIsInvariant && !argCanBeModified) + void StoreRetExprResultsInArgs(GenTreeCall* call) { - // Directly substitute constants or addresses of locals - // - // Clone the constant. Note that we cannot directly use - // argNode in the trees even if !argInfo.argIsUsed as this - // would introduce aliasing between inlArgInfo[].argNode and - // impInlineExpr. Then gtFoldExpr() could change it, causing - // further references to the argument working off of the - // bashed copy. - op1 = gtCloneExpr(argNode); - PREFIX_ASSUME(op1 != nullptr); - argInfo.argTmpNum = BAD_VAR_NUM; + for (CallArg& arg : call->gtArgs.Args()) + { + comp->fgWalkTreePre(&arg.EarlyNodeRef(), SpillRetExprVisitor, this); + } + } - // We may need to retype to ensure we match the callee's view of the type. - // Otherwise callee-pass throughs of arguments can create return type - // mismatches that block inlining. - // - // Note argument type mismatches that prevent inlining should - // have been caught in impInlineInitVars. - if (op1->TypeGet() != lclTyp) +private: + static Compiler::fgWalkResult SpillRetExprVisitor(GenTree** pTree, Compiler::fgWalkData* fgWalkPre) + { + assert((pTree != nullptr) && (*pTree != nullptr)); + GenTree* tree = *pTree; + if ((tree->gtFlags & GTF_CALL) == 0) { - op1->gtType = genActualType(lclTyp); + // Trees with ret_expr are marked as GTF_CALL. + return Compiler::WALK_SKIP_SUBTREES; } + if (tree->OperGet() == GT_RET_EXPR) + { + SpillRetExprHelper* walker = static_cast(fgWalkPre->pCallbackData); + walker->StoreRetExprAsLocalVar(pTree); + } + return Compiler::WALK_CONTINUE; } - else if (argInfo.argIsLclVar && !argCanBeModified && !argInfo.argHasCallerLocalRef) + + void StoreRetExprAsLocalVar(GenTree** pRetExpr) { - // Directly substitute unaliased caller locals for args that cannot be modified - // - // Use the caller-supplied node if this is the first use. - op1 = argNode; - unsigned argLclNum = op1->AsLclVarCommon()->GetLclNum(); - argInfo.argTmpNum = argLclNum; + GenTree* retExpr = *pRetExpr; + assert(retExpr->OperGet() == GT_RET_EXPR); + const unsigned tmp = comp->lvaGrabTemp(true DEBUGARG("spilling ret_expr")); + JITDUMP("Storing return expression [%06u] to a local var V%02u.\n", comp->dspTreeID(retExpr), tmp); + comp->impAssignTempGen(tmp, retExpr, (unsigned)Compiler::CHECK_SPILL_NONE); + *pRetExpr = comp->gtNewLclvNode(tmp, retExpr->TypeGet()); - // Use an equivalent copy if this is the second or subsequent - // use. - // - // Note argument type mismatches that prevent inlining should - // have been caught in impInlineInitVars. If inlining is not prevented - // but a cast is necessary, we similarly expect it to have been inserted then. - // So here we may have argument type mismatches that are benign, for instance - // passing a TYP_SHORT local (eg. normalized-on-load) as a TYP_INT arg. - // The exception is when the inlining means we should start tracking the argument. - if (argInfo.argIsUsed || ((lclTyp == TYP_BYREF) && (op1->TypeGet() != TYP_BYREF))) + if (retExpr->TypeGet() == TYP_REF) { - assert(op1->gtOper == GT_LCL_VAR); - assert(lclNum == op1->AsLclVar()->gtLclILoffs); + assert(comp->lvaTable[tmp].lvSingleDef == 0); + comp->lvaTable[tmp].lvSingleDef = 1; + JITDUMP("Marked V%02u as a single def temp\n", tmp); - // Create a new lcl var node - remember the argument lclNum - op1 = impCreateLocalNode(argLclNum DEBUGARG(op1->AsLclVar()->gtLclILoffs)); - // Start tracking things as a byref if the parameter is a byref. - if (lclTyp == TYP_BYREF) + bool isExact = false; + bool isNonNull = false; + CORINFO_CLASS_HANDLE retClsHnd = comp->gtGetClassHandle(retExpr, &isExact, &isNonNull); + if (retClsHnd != nullptr) { - op1->gtType = TYP_BYREF; + comp->lvaSetClass(tmp, retClsHnd, isExact); } } } - else if (argInfo.argIsByRefToStructLocal && !argInfo.argHasStargOp) - { - /* Argument is a by-ref address to a struct, a normed struct, or its field. - In these cases, don't spill the byref to a local, simply clone the tree and use it. - This way we will increase the chance for this byref to be optimized away by - a subsequent "dereference" operation. - - From Dev11 bug #139955: Argument node can also be TYP_I_IMPL if we've bashed the tree - (in impInlineInitVars()), if the arg has argHasLdargaOp as well as argIsByRefToStructLocal. - For example, if the caller is: - ldloca.s V_1 // V_1 is a local struct - call void Test.ILPart::RunLdargaOnPointerArg(int32*) - and the callee being inlined has: - .method public static void RunLdargaOnPointerArg(int32* ptrToInts) cil managed - ldarga.s ptrToInts - call void Test.FourInts::NotInlined_SetExpectedValuesThroughPointerToPointer(int32**) - then we change the argument tree (of "ldloca.s V_1") to TYP_I_IMPL to match the callee signature. We'll - soon afterwards reject the inlining anyway, since the tree we return isn't a GT_LCL_VAR. - */ - assert(argNode->TypeGet() == TYP_BYREF || argNode->TypeGet() == TYP_I_IMPL); - op1 = gtCloneExpr(argNode); - } - else - { - /* Argument is a complex expression - it must be evaluated into a temp */ - - if (argInfo.argHasTmp) - { - assert(argInfo.argIsUsed); - assert(argInfo.argTmpNum < lvaCount); - /* Create a new lcl var node - remember the argument lclNum */ - op1 = gtNewLclvNode(argInfo.argTmpNum, genActualType(lclTyp)); +private: + Compiler* comp; +}; - /* This is the second or later use of the this argument, - so we have to use the temp (instead of the actual arg) */ - argInfo.argBashTmpNode = nullptr; - } - else - { - /* First time use */ - assert(!argInfo.argIsUsed); +//------------------------------------------------------------------------ +// addFatPointerCandidate: mark the call and the method, that they have a fat pointer candidate. +// Spill ret_expr in the call node, because they can't be cloned. +// +// Arguments: +// call - fat calli candidate +// +void Compiler::addFatPointerCandidate(GenTreeCall* call) +{ + JITDUMP("Marking call [%06u] as fat pointer candidate\n", dspTreeID(call)); + setMethodHasFatPointer(); + call->SetFatPointerCandidate(); + SpillRetExprHelper helper(this); + helper.StoreRetExprResultsInArgs(call); +} - /* Reserve a temp for the expression. - * Use a large size node as we may change it later */ +//------------------------------------------------------------------------ +// pickGDV: Use profile information to pick a GDV candidate for a call site. +// +// Arguments: +// call - the call +// ilOffset - exact IL offset of the call +// isInterface - whether or not the call target is defined on an interface +// classGuess - [out] the class to guess for (mutually exclusive with methodGuess) +// methodGuess - [out] the method to guess for (mutually exclusive with classGuess) +// likelihood - [out] an estimate of the likelihood that the guess will succeed +// +void Compiler::pickGDV(GenTreeCall* call, + IL_OFFSET ilOffset, + bool isInterface, + CORINFO_CLASS_HANDLE* classGuess, + CORINFO_METHOD_HANDLE* methodGuess, + unsigned* likelihood) +{ + *classGuess = NO_CLASS_HANDLE; + *methodGuess = NO_METHOD_HANDLE; + *likelihood = 0; - const unsigned tmpNum = lvaGrabTemp(true DEBUGARG("Inlining Arg")); + const int maxLikelyClasses = 32; + LikelyClassMethodRecord likelyClasses[maxLikelyClasses]; + unsigned numberOfClasses = 0; + if (call->IsVirtualStub() || call->IsVirtualVtable()) + { + numberOfClasses = + getLikelyClasses(likelyClasses, maxLikelyClasses, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset); + } - lvaTable[tmpNum].lvType = lclTyp; + const int maxLikelyMethods = 32; + LikelyClassMethodRecord likelyMethods[maxLikelyMethods]; + unsigned numberOfMethods = 0; - // For ref types, determine the type of the temp. - if (lclTyp == TYP_REF) - { - if (!argCanBeModified) - { - // If the arg can't be modified in the method - // body, use the type of the value, if - // known. Otherwise, use the declared type. - assert(lvaTable[tmpNum].lvSingleDef == 0); - lvaTable[tmpNum].lvSingleDef = 1; - JITDUMP("Marked V%02u as a single def temp\n", tmpNum); - lvaSetClass(tmpNum, argNode, lclInfo.lclVerTypeInfo.GetClassHandleForObjRef()); - } - else - { - // Arg might be modified, use the declared type of - // the argument. - lvaSetClass(tmpNum, lclInfo.lclVerTypeInfo.GetClassHandleForObjRef()); - } - } + // TODO-GDV: R2R support requires additional work to reacquire the + // entrypoint, similar to what happens at the end of impDevirtualizeCall. + // As part of supporting this we should merge the tail of + // impDevirtualizeCall and what happens in + // GuardedDevirtualizationTransformer::CreateThen for method GDV. + // + if (!opts.IsReadyToRun() && (call->IsVirtualVtable() || call->IsDelegateInvoke())) + { + numberOfMethods = + getLikelyMethods(likelyMethods, maxLikelyMethods, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset); + } - assert(!lvaTable[tmpNum].IsAddressExposed()); - if (argInfo.argHasLdargaOp) - { - lvaTable[tmpNum].lvHasLdAddrOp = 1; - } + if ((numberOfClasses < 1) && (numberOfMethods < 1)) + { + JITDUMP("No likely class or method, sorry\n"); + return; + } - if (lclInfo.lclVerTypeInfo.IsStruct()) - { - if (varTypeIsStruct(lclTyp)) - { - lvaSetStruct(tmpNum, lclInfo.lclVerTypeInfo.GetClassHandle(), true /* unsafe value cls check */); - if (info.compIsVarArgs) - { - lvaSetStructUsedAsVarArg(tmpNum); - } - } - } +#ifdef DEBUG + if ((verbose || JitConfig.EnableExtraSuperPmiQueries()) && (numberOfClasses > 0)) + { + bool isExact; + bool isNonNull; + CallArg* thisArg = call->gtArgs.GetThisArg(); + CORINFO_CLASS_HANDLE declaredThisClsHnd = gtGetClassHandle(thisArg->GetNode(), &isExact, &isNonNull); + JITDUMP("Likely classes for call [%06u]", dspTreeID(call)); + if (declaredThisClsHnd != NO_CLASS_HANDLE) + { + const char* baseClassName = eeGetClassName(declaredThisClsHnd); + JITDUMP(" on class %p (%s)", declaredThisClsHnd, baseClassName); + } + JITDUMP("\n"); - argInfo.argHasTmp = true; - argInfo.argTmpNum = tmpNum; + for (UINT32 i = 0; i < numberOfClasses; i++) + { + const char* className = eeGetClassName((CORINFO_CLASS_HANDLE)likelyClasses[i].handle); + JITDUMP(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, likelyClasses[i].handle, className, + likelyClasses[i].likelihood); + } + } - // If we require strict exception order, then arguments must - // be evaluated in sequence before the body of the inlined method. - // So we need to evaluate them to a temp. - // Also, if arguments have global or local references, we need to - // evaluate them to a temp before the inlined body as the - // inlined body may be modifying the global ref. - // TODO-1stClassStructs: We currently do not reuse an existing lclVar - // if it is a struct, because it requires some additional handling. + if ((verbose || JitConfig.EnableExtraSuperPmiQueries()) && (numberOfMethods > 0)) + { + assert(call->gtCallType == CT_USER_FUNC); + const char* baseMethName = eeGetMethodFullName(call->gtCallMethHnd); + JITDUMP("Likely methods for call [%06u] to method %s\n", dspTreeID(call), baseMethName); - if ((!varTypeIsStruct(lclTyp) && !argInfo.argHasSideEff && !argInfo.argHasGlobRef && - !argInfo.argHasCallerLocalRef)) - { - /* Get a *LARGE* LCL_VAR node */ - op1 = gtNewLclLNode(tmpNum, genActualType(lclTyp) DEBUGARG(lclNum)); + for (UINT32 i = 0; i < numberOfMethods; i++) + { + CORINFO_CONST_LOOKUP lookup = {}; + info.compCompHnd->getFunctionFixedEntryPoint((CORINFO_METHOD_HANDLE)likelyMethods[i].handle, false, + &lookup); - /* Record op1 as the very first use of this argument. - If there are no further uses of the arg, we may be - able to use the actual arg node instead of the temp. - If we do see any further uses, we will clear this. */ - argInfo.argBashTmpNode = op1; - } - else + const char* methName = eeGetMethodFullName((CORINFO_METHOD_HANDLE)likelyMethods[i].handle); + switch (lookup.accessType) { - /* Get a small LCL_VAR node */ - op1 = gtNewLclvNode(tmpNum, genActualType(lclTyp)); - /* No bashing of this argument */ - argInfo.argBashTmpNode = nullptr; + case IAT_VALUE: + JITDUMP(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, + likelyMethods[i].likelihood); + break; + case IAT_PVALUE: + JITDUMP(" %u) [%p] (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, + likelyMethods[i].likelihood); + break; + case IAT_PPVALUE: + JITDUMP(" %u) [[%p]] (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, + likelyMethods[i].likelihood); + break; + default: + JITDUMP(" %u) %s [likelihood:%u%%]\n", i + 1, methName, likelyMethods[i].likelihood); + break; } } } - // Mark this argument as used. - argInfo.argIsUsed = true; + // Optional stress mode to pick a random known class, rather than + // the most likely known class. + // + if (JitConfig.JitRandomGuardedDevirtualization() != 0) + { + // Reuse the random inliner's random state. + // + CLRRandom* const random = + impInlineRoot()->m_inlineStrategy->GetRandom(JitConfig.JitRandomGuardedDevirtualization()); + unsigned index = static_cast(random->Next(static_cast(numberOfClasses + numberOfMethods))); + if (index < numberOfClasses) + { + *classGuess = (CORINFO_CLASS_HANDLE)likelyClasses[index].handle; + *likelihood = 100; + JITDUMP("Picked random class for GDV: %p (%s)\n", *classGuess, eeGetClassName(*classGuess)); + return; + } + else + { + *methodGuess = (CORINFO_METHOD_HANDLE)likelyMethods[index - numberOfClasses].handle; + *likelihood = 100; + JITDUMP("Picked random method for GDV: %p (%s)\n", *methodGuess, eeGetMethodFullName(*methodGuess)); + return; + } + } +#endif - return op1; -} + // Prefer class guess as it is cheaper + if (numberOfClasses > 0) + { + unsigned likelihoodThreshold = isInterface ? 25 : 30; + if (likelyClasses[0].likelihood >= likelihoodThreshold) + { + *classGuess = (CORINFO_CLASS_HANDLE)likelyClasses[0].handle; + *likelihood = likelyClasses[0].likelihood; + return; + } -/****************************************************************************** - Is this the original "this" argument to the call being inlined? + JITDUMP("Not guessing for class; likelihood is below %s call threshold %u\n", + isInterface ? "interface" : "virtual", likelihoodThreshold); + } - Note that we do not inline methods with "starg 0", and so we do not need to - worry about it. -*/ + if (numberOfMethods > 0) + { + unsigned likelihoodThreshold = 30; + if (likelyMethods[0].likelihood >= likelihoodThreshold) + { + *methodGuess = (CORINFO_METHOD_HANDLE)likelyMethods[0].handle; + *likelihood = likelyMethods[0].likelihood; + return; + } -bool Compiler::impInlineIsThis(GenTree* tree, InlArgInfo* inlArgInfo) -{ - assert(compIsForInlining()); - return (tree->gtOper == GT_LCL_VAR && tree->AsLclVarCommon()->GetLclNum() == inlArgInfo[0].argTmpNum); + JITDUMP("Not guessing for method; likelihood is below %s call threshold %u\n", + call->IsDelegateInvoke() ? "delegate" : "virtual", likelihoodThreshold); + } } -//----------------------------------------------------------------------------- -// impInlineIsGuaranteedThisDerefBeforeAnySideEffects: Check if a dereference in -// the inlinee can guarantee that the "this" pointer is non-NULL. +//------------------------------------------------------------------------ +// isCompatibleMethodGDV: +// Check if devirtualizing a call node as a specified target method call is +// reasonable. // // Arguments: -// additionalTree - a tree to check for side effects -// additionalCallArgs - a list of call args to check for side effects -// dereferencedAddress - address expression being dereferenced -// inlArgInfo - inlinee argument information +// call - the call +// gdvTarget - the target method that we want to guess for and devirtualize to // -// Notes: -// If we haven't hit a branch or a side effect, and we are dereferencing -// from 'this' to access a field or make GTF_CALL_NULLCHECK call, -// then we can avoid a separate null pointer check. +// Returns: +// true if we can proceed with GDV. // -// The importer stack and current statement list are searched for side effects. -// Trees that have been popped of the stack but haven't been appended to the -// statement list and have to be checked for side effects may be provided via -// additionalTree and additionalCallArgs. +// Notes: +// This implements a small simplified signature-compatibility check to +// verify that a guess is reasonable. The main goal here is to avoid blowing +// up the JIT on PGO data with stale GDV candidates; if they are not +// compatible in the ECMA sense then we do not expect the guard to ever pass +// at runtime, so we can get by with simplified rules here. // -bool Compiler::impInlineIsGuaranteedThisDerefBeforeAnySideEffects(GenTree* additionalTree, - CallArgs* additionalCallArgs, - GenTree* dereferencedAddress, - InlArgInfo* inlArgInfo) +bool Compiler::isCompatibleMethodGDV(GenTreeCall* call, CORINFO_METHOD_HANDLE gdvTarget) { - assert(compIsForInlining()); - assert(opts.OptEnabled(CLFLG_INLINING)); + CORINFO_SIG_INFO sig; + info.compCompHnd->getMethodSig(gdvTarget, &sig); - BasicBlock* block = compCurBB; + CORINFO_ARG_LIST_HANDLE sigParam = sig.args; + unsigned numParams = sig.numArgs; + unsigned numArgs = 0; + for (CallArg& arg : call->gtArgs.Args()) + { + switch (arg.GetWellKnownArg()) + { + case WellKnownArg::RetBuffer: + case WellKnownArg::ThisPointer: + // Not part of signature but we still expect to see it here + continue; + case WellKnownArg::None: + break; + default: + assert(!"Unexpected well known arg to method GDV candidate"); + continue; + } - if (block != fgFirstBB) + numArgs++; + if (numArgs > numParams) + { + JITDUMP("Incompatible method GDV: call [%06u] has more arguments than signature (sig has %d parameters)\n", + dspTreeID(call), numParams); + return false; + } + + CORINFO_CLASS_HANDLE classHnd = NO_CLASS_HANDLE; + CorInfoType corType = strip(info.compCompHnd->getArgType(&sig, sigParam, &classHnd)); + var_types sigType = JITtype2varType(corType); + + if (!impCheckImplicitArgumentCoercion(sigType, arg.GetNode()->TypeGet())) + { + JITDUMP("Incompatible method GDV: arg [%06u] is type-incompatible with signature of target\n", + dspTreeID(arg.GetNode())); + return false; + } + + // Best-effort check for struct compatibility here. + if (varTypeIsStruct(sigType) && (arg.GetSignatureClassHandle() != classHnd)) + { + ClassLayout* callLayout = typGetObjLayout(arg.GetSignatureClassHandle()); + ClassLayout* tarLayout = typGetObjLayout(classHnd); + + if (!ClassLayout::AreCompatible(callLayout, tarLayout)) + { + JITDUMP("Incompatible method GDV: struct arg [%06u] is layout-incompatible with signature of target\n", + dspTreeID(arg.GetNode())); + return false; + } + } + + sigParam = info.compCompHnd->getArgNext(sigParam); + } + + if (numArgs < numParams) { + JITDUMP("Incompatible method GDV: call [%06u] has fewer arguments (%d) than signature (%d)\n", dspTreeID(call), + numArgs, numParams); return false; } - if (!impInlineIsThis(dereferencedAddress, inlArgInfo)) + return true; +} + +//------------------------------------------------------------------------ +// considerGuardedDevirtualization: see if we can profitably guess at the +// class involved in an interface or virtual call. +// +// Arguments: +// +// call - potential guarded devirtualization candidate +// ilOffset - IL ofset of the call instruction +// baseMethod - target method of the call +// baseClass - class that introduced the target method +// pContextHandle - context handle for the call +// +// Notes: +// Consults with VM to see if there's a likely class at runtime, +// if so, adds a candidate for guarded devirtualization. +// +void Compiler::considerGuardedDevirtualization(GenTreeCall* call, + IL_OFFSET ilOffset, + bool isInterface, + CORINFO_METHOD_HANDLE baseMethod, + CORINFO_CLASS_HANDLE baseClass, + CORINFO_CONTEXT_HANDLE* pContextHandle) +{ + JITDUMP("Considering guarded devirtualization at IL offset %u (0x%x)\n", ilOffset, ilOffset); + + // We currently only get likely class guesses when there is PGO data + // with class profiles. + // + if ((fgPgoClassProfiles == 0) && (fgPgoMethodProfiles == 0)) { - return false; + JITDUMP("Not guessing for class or method: no GDV profile pgo data, or pgo disabled\n"); + return; } - if ((additionalTree != nullptr) && GTF_GLOBALLY_VISIBLE_SIDE_EFFECTS(additionalTree->gtFlags)) + CORINFO_CLASS_HANDLE likelyClass; + CORINFO_METHOD_HANDLE likelyMethod; + unsigned likelihood; + pickGDV(call, ilOffset, isInterface, &likelyClass, &likelyMethod, &likelihood); + + if ((likelyClass == NO_CLASS_HANDLE) && (likelyMethod == NO_METHOD_HANDLE)) { - return false; + return; } - if (additionalCallArgs != nullptr) + uint32_t likelyClassAttribs = 0; + if (likelyClass != NO_CLASS_HANDLE) { - for (CallArg& arg : additionalCallArgs->Args()) + likelyClassAttribs = info.compCompHnd->getClassAttribs(likelyClass); + + if ((likelyClassAttribs & CORINFO_FLG_ABSTRACT) != 0) { - if (GTF_GLOBALLY_VISIBLE_SIDE_EFFECTS(arg.GetEarlyNode()->gtFlags)) - { - return false; - } + // We may see an abstract likely class, if we have a stale profile. + // No point guessing for this. + // + JITDUMP("Not guessing for class; abstract (stale profile)\n"); + return; } - } - for (Statement* stmt : StatementList(impStmtList)) - { - GenTree* expr = stmt->GetRootNode(); - if (GTF_GLOBALLY_VISIBLE_SIDE_EFFECTS(expr->gtFlags)) + // Figure out which method will be called. + // + CORINFO_DEVIRTUALIZATION_INFO dvInfo; + dvInfo.virtualMethod = baseMethod; + dvInfo.objClass = likelyClass; + dvInfo.context = *pContextHandle; + dvInfo.exactContext = *pContextHandle; + dvInfo.pResolvedTokenVirtualMethod = nullptr; + + const bool canResolve = info.compCompHnd->resolveVirtualMethod(&dvInfo); + + if (!canResolve) { - return false; + JITDUMP("Can't figure out which method would be invoked, sorry\n"); + return; } + + likelyMethod = dvInfo.devirtualizedMethod; } - for (unsigned level = 0; level < verCurrentState.esStackDepth; level++) + uint32_t likelyMethodAttribs = info.compCompHnd->getMethodAttribs(likelyMethod); + + if (likelyClass == NO_CLASS_HANDLE) { - GenTreeFlags stackTreeFlags = verCurrentState.esStack[level].val->gtFlags; - if (GTF_GLOBALLY_VISIBLE_SIDE_EFFECTS(stackTreeFlags)) + // For method GDV do a few more checks that we get for free in the + // resolve call above for class-based GDV. + if ((likelyMethodAttribs & CORINFO_FLG_STATIC) != 0) { - return false; + assert(call->IsDelegateInvoke()); + JITDUMP("Cannot currently handle devirtualizing static delegate calls, sorry\n"); + return; + } + + CORINFO_CLASS_HANDLE definingClass = info.compCompHnd->getMethodClass(likelyMethod); + likelyClassAttribs = info.compCompHnd->getClassAttribs(definingClass); + + // For instance methods on value classes we need an extended check to + // check for the unboxing stub. This is NYI. + // Note: For dynamic PGO likelyMethod above will be the unboxing stub + // which would fail GDV for other reasons. + // However, with static profiles or textual PGO input it is still + // possible that likelyMethod is not the unboxing stub. So we do need + // this explicit check. + if ((likelyClassAttribs & CORINFO_FLG_VALUECLASS) != 0) + { + JITDUMP("Cannot currently handle devirtualizing delegate calls on value types, sorry\n"); + return; + } + + // Verify that the call target and args look reasonable so that the JIT + // does not blow up during inlining/call morphing. + // + // NOTE: Once we want to support devirtualization of delegate calls to + // static methods and remove the check above we will start failing here + // for delegates pointing to static methods that have the first arg + // bound. For example: + // + // public static void E(this C c) ... + // Action a = new C().E; + // + // The delegate instance looks exactly like one pointing to an instance + // method in this case and the call will have zero args while the + // signature has 1 arg. + // + if (!isCompatibleMethodGDV(call, likelyMethod)) + { + JITDUMP("Target for method-based GDV is incompatible (stale profile?)\n"); + assert((fgPgoSource != ICorJitInfo::PgoSource::Dynamic) && "Unexpected stale profile in dynamic PGO data"); + return; } } - return true; + JITDUMP("%s call would invoke method %s\n", + isInterface ? "interface" : call->IsDelegateInvoke() ? "delegate" : "virtual", + eeGetMethodName(likelyMethod, nullptr)); + + // Add this as a potential candidate. + // + addGuardedDevirtualizationCandidate(call, likelyMethod, likelyClass, likelyMethodAttribs, likelyClassAttribs, + likelihood); } //------------------------------------------------------------------------ -// impAllocateMethodPointerInfo: create methodPointerInfo into jit-allocated memory and init it. +// addGuardedDevirtualizationCandidate: potentially mark the call as a guarded +// devirtualization candidate +// +// Notes: +// +// Call sites in rare or unoptimized code, and calls that require cookies are +// not marked as candidates. +// +// As part of marking the candidate, the code spills GT_RET_EXPRs anywhere in any +// child tree, because and we need to clone all these trees when we clone the call +// as part of guarded devirtualization, and these IR nodes can't be cloned. // // Arguments: -// token - init value for the allocated token. -// tokenConstrained - init value for the constraint associated with the token +// call - potential guarded devirtualization candidate +// methodHandle - method that will be invoked if the class test succeeds +// classHandle - class that will be tested for at runtime +// methodAttr - attributes of the method +// classAttr - attributes of the class +// likelihood - odds that this class is the class seen at runtime // -// Return Value: -// pointer to token into jit-allocated memory. -methodPointerInfo* Compiler::impAllocateMethodPointerInfo(const CORINFO_RESOLVED_TOKEN& token, mdToken tokenConstrained) +void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call, + CORINFO_METHOD_HANDLE methodHandle, + CORINFO_CLASS_HANDLE classHandle, + unsigned methodAttr, + unsigned classAttr, + unsigned likelihood) { - methodPointerInfo* memory = getAllocator(CMK_Unknown).allocate(1); - memory->m_token = token; - memory->m_tokenConstraint = tokenConstrained; - return memory; + // This transformation only makes sense for delegate and virtual calls + assert(call->IsDelegateInvoke() || call->IsVirtual()); + + // Only mark calls if the feature is enabled. + const bool isEnabled = JitConfig.JitEnableGuardedDevirtualization() > 0; + + if (!isEnabled) + { + JITDUMP("NOT Marking call [%06u] as guarded devirtualization candidate -- disabled by jit config\n", + dspTreeID(call)); + return; + } + + // Bail if not optimizing or the call site is very likely cold + if (compCurBB->isRunRarely() || opts.OptimizationDisabled()) + { + JITDUMP("NOT Marking call [%06u] as guarded devirtualization candidate -- rare / dbg / minopts\n", + dspTreeID(call)); + return; + } + + // CT_INDIRECT calls may use the cookie, bail if so... + // + // If transforming these provides a benefit, we could save this off in the same way + // we save the stub address below. + if ((call->gtCallType == CT_INDIRECT) && (call->AsCall()->gtCallCookie != nullptr)) + { + JITDUMP("NOT Marking call [%06u] as guarded devirtualization candidate -- CT_INDIRECT with cookie\n", + dspTreeID(call)); + return; + } + +#ifdef DEBUG + + // See if disabled by range + // + static ConfigMethodRange JitGuardedDevirtualizationRange; + JitGuardedDevirtualizationRange.EnsureInit(JitConfig.JitGuardedDevirtualizationRange()); + assert(!JitGuardedDevirtualizationRange.Error()); + if (!JitGuardedDevirtualizationRange.Contains(impInlineRoot()->info.compMethodHash())) + { + JITDUMP("NOT Marking call [%06u] as guarded devirtualization candidate -- excluded by " + "JitGuardedDevirtualizationRange", + dspTreeID(call)); + return; + } + +#endif + + // We're all set, proceed with candidate creation. + // + JITDUMP("Marking call [%06u] as guarded devirtualization candidate; will guess for %s %s\n", dspTreeID(call), + classHandle != NO_CLASS_HANDLE ? "class" : "method", + classHandle != NO_CLASS_HANDLE ? eeGetClassName(classHandle) : eeGetMethodFullName(methodHandle)); + setMethodHasGuardedDevirtualization(); + call->SetGuardedDevirtualizationCandidate(); + + // Spill off any GT_RET_EXPR subtrees so we can clone the call. + // + SpillRetExprHelper helper(this); + helper.StoreRetExprResultsInArgs(call); + + // Gather some information for later. Note we actually allocate InlineCandidateInfo + // here, as the devirtualized half of this call will likely become an inline candidate. + // + GuardedDevirtualizationCandidateInfo* pInfo = new (this, CMK_Inlining) InlineCandidateInfo; + + pInfo->guardedMethodHandle = methodHandle; + pInfo->guardedMethodUnboxedEntryHandle = nullptr; + pInfo->guardedClassHandle = classHandle; + pInfo->likelihood = likelihood; + pInfo->requiresInstMethodTableArg = false; + + // If the guarded class is a value class, look for an unboxed entry point. + // + if ((classAttr & CORINFO_FLG_VALUECLASS) != 0) + { + JITDUMP(" ... class is a value class, looking for unboxed entry\n"); + bool requiresInstMethodTableArg = false; + CORINFO_METHOD_HANDLE unboxedEntryMethodHandle = + info.compCompHnd->getUnboxedEntry(methodHandle, &requiresInstMethodTableArg); + + if (unboxedEntryMethodHandle != nullptr) + { + JITDUMP(" ... updating GDV candidate with unboxed entry info\n"); + pInfo->guardedMethodUnboxedEntryHandle = unboxedEntryMethodHandle; + pInfo->requiresInstMethodTableArg = requiresInstMethodTableArg; + } + } + + call->gtGuardedDevirtualizationCandidateInfo = pInfo; } void Compiler::addExpRuntimeLookupCandidate(GenTreeCall* call) @@ -14005,9 +23136,9 @@ bool Compiler::impCanSkipCovariantStoreCheck(GenTree* value, GenTree* array) return true; } // Non-0 const refs can only occur with frozen objects - assert(value->IsIconHandle(GTF_ICON_OBJ_HDL)); - assert(doesMethodHaveFrozenObjects() || - (compIsForInlining() && impInlineInfo->InlinerCompiler->doesMethodHaveFrozenObjects())); + assert(value->IsIconHandle(GTF_ICON_STR_HDL)); + assert(doesMethodHaveFrozenString() || + (compIsForInlining() && impInlineInfo->InlinerCompiler->doesMethodHaveFrozenString())); } // Try and get a class handle for the array diff --git a/src/coreclr/jit/namedintrinsiclist.h b/src/coreclr/jit/namedintrinsiclist.h index 37af303c0ab68d..5aee22dc2867b4 100644 --- a/src/coreclr/jit/namedintrinsiclist.h +++ b/src/coreclr/jit/namedintrinsiclist.h @@ -93,6 +93,8 @@ enum NamedIntrinsic : unsigned short NI_System_Runtime_CompilerServices_RuntimeHelpers_InitializeArray, NI_System_Runtime_CompilerServices_RuntimeHelpers_IsKnownConstant, + NI_System_Runtime_InteropService_MemoryMarshal_GetArrayDataReference, + NI_System_String_Equals, NI_System_String_get_Chars, NI_System_String_get_Length, diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/InteropServices/MemoryMarshal.NativeAot.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/InteropServices/MemoryMarshal.NativeAot.cs index f90969a3410370..5d97217a13c8bb 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/InteropServices/MemoryMarshal.NativeAot.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/InteropServices/MemoryMarshal.NativeAot.cs @@ -20,9 +20,8 @@ public static unsafe partial class MemoryMarshal /// [Intrinsic] [NonVersionable] - [MethodImpl(MethodImplOptions.AggressiveInlining)] public static ref T GetArrayDataReference(T[] array) => - ref Unsafe.As(ref Unsafe.As(array).Data); + ref GetArrayDataReference(array); /// /// Returns a reference to the 0th element of . If the array is empty, returns a reference to where the 0th element diff --git a/src/coreclr/vm/corelib.h b/src/coreclr/vm/corelib.h index a2158726dd8303..c069124945a3ae 100644 --- a/src/coreclr/vm/corelib.h +++ b/src/coreclr/vm/corelib.h @@ -731,7 +731,6 @@ DEFINE_METHOD(UNSAFE, UNBOX, Unbox, NoSig) DEFINE_METHOD(UNSAFE, WRITE, Write, NoSig) DEFINE_CLASS(MEMORY_MARSHAL, Interop, MemoryMarshal) -DEFINE_METHOD(MEMORY_MARSHAL, GET_ARRAY_DATA_REFERENCE_SZARRAY, GetArrayDataReference, GM_ArrT_RetRefT) DEFINE_METHOD(MEMORY_MARSHAL, GET_ARRAY_DATA_REFERENCE_MDARRAY, GetArrayDataReference, SM_Array_RetRefByte) DEFINE_CLASS(INTERLOCKED, Threading, Interlocked) diff --git a/src/coreclr/vm/jitinterface.cpp b/src/coreclr/vm/jitinterface.cpp index 149a8ff1e87290..95c6d41a64fd67 100644 --- a/src/coreclr/vm/jitinterface.cpp +++ b/src/coreclr/vm/jitinterface.cpp @@ -7001,39 +7001,6 @@ bool getILIntrinsicImplementationForUnsafe(MethodDesc * ftn, return false; } -bool getILIntrinsicImplementationForMemoryMarshal(MethodDesc * ftn, - CORINFO_METHOD_INFO * methInfo) -{ - STANDARD_VM_CONTRACT; - - _ASSERTE(CoreLibBinder::IsClass(ftn->GetMethodTable(), CLASS__MEMORY_MARSHAL)); - - mdMethodDef tk = ftn->GetMemberDef(); - - if (tk == CoreLibBinder::GetMethod(METHOD__MEMORY_MARSHAL__GET_ARRAY_DATA_REFERENCE_SZARRAY)->GetMemberDef()) - { - mdToken tokRawSzArrayData = CoreLibBinder::GetField(FIELD__RAW_ARRAY_DATA__DATA)->GetMemberDef(); - - static BYTE ilcode[] = { CEE_LDARG_0, - CEE_LDFLDA,0,0,0,0, - CEE_RET }; - - ilcode[2] = (BYTE)(tokRawSzArrayData); - ilcode[3] = (BYTE)(tokRawSzArrayData >> 8); - ilcode[4] = (BYTE)(tokRawSzArrayData >> 16); - ilcode[5] = (BYTE)(tokRawSzArrayData >> 24); - - methInfo->ILCode = const_cast(ilcode); - methInfo->ILCodeSize = sizeof(ilcode); - methInfo->maxStack = 1; - methInfo->EHcount = 0; - methInfo->options = (CorInfoOptions)0; - return true; - } - - return false; -} - bool getILIntrinsicImplementationForVolatile(MethodDesc * ftn, CORINFO_METHOD_INFO * methInfo) { @@ -7485,10 +7452,6 @@ getMethodInfoHelper( { fILIntrinsic = getILIntrinsicImplementationForUnsafe(ftn, methInfo); } - else if (CoreLibBinder::IsClass(pMT, CLASS__MEMORY_MARSHAL)) - { - fILIntrinsic = getILIntrinsicImplementationForMemoryMarshal(ftn, methInfo); - } else if (CoreLibBinder::IsClass(pMT, CLASS__INTERLOCKED)) { fILIntrinsic = getILIntrinsicImplementationForInterlocked(ftn, methInfo); diff --git a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs new file mode 100644 index 00000000000000..fbe3c93bebb0e8 --- /dev/null +++ b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs @@ -0,0 +1,150 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// + +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace MemoryMarshalGetArrayDataReferenceTest +{ + class Program + { + private static int _errors = 0; + + unsafe static int Main(string[] args) + { + delegate* ptrByte = &MemoryMarshal.GetArrayDataReference; + delegate* ptrString = &MemoryMarshal.GetArrayDataReference; + + byte[] testByteArray = new byte[1]; + IsTrue(Unsafe.AreSame(ref MemoryMarshal.GetArrayDataReference(testByteArray), ref testByteArray[0])); + IsTrue(Unsafe.AreSame(ref ptrByte(testByteArray), ref testByteArray[0])); + + string[] testStringArray = new string[1]; + IsTrue(Unsafe.AreSame(ref MemoryMarshal.GetArrayDataReference(testStringArray), ref testStringArray[0])); + IsTrue(Unsafe.AreSame(ref ptrString(testStringArray), ref testStringArray[0])); + + IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(Array.Empty()))); + IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(Array.Empty()))); + IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(Array.Empty()))); + IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(Array.Empty>()))); + IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(Array.Empty()))); + IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(Array.Empty()))); + IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(Array.Empty>()))); + IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(Array.Empty>()))); + + IsFalse(Unsafe.IsNullRef(ref ptrByte(Array.Empty()))); + IsFalse(Unsafe.IsNullRef(ref ptrString(Array.Empty()))); + + ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference(null); }); + ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference(null); }); + ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference(null); }); + ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference>(null); }); + ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference(null); }); + ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference(null); }); + ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference>(null); }); + ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference>(null); }); + + ThrowsNRE(() => { _ = ref ptrByte(null); }); + ThrowsNRE(() => { _ = ref ptrString(null); }); + + // from https://github.com/dotnet/runtime/issues/58312#issuecomment-993491291 + [MethodImpl(MethodImplOption.NoInlining)] + static int Problem1(StructWithByte[] a) + { + MemoryMarshal.GetArrayDataReference(a).Byte = 1; + + a[0].Byte = 2; + + return MemoryMarshal.GetArrayDataReference(a).Byte; + } + + Equals(Problem(new StructWithByte[] { new StructWithByte { Byte = 1 } }), 2); + + [MethodImpl(MethodImplOption.NoInlining)] + static int Problem2(byte[] a) + { + if (MemoryMarshal.GetArrayDataReference(a) == 1) + { + a[0] = 2; + if (MemoryMarshal.GetArrayDataReference(a) == 1) + { + return -1; + } + } + + return 0; + } + + Equals(Problem2(new byte[] { 1 }), 0); + + return 100 + _errors; + } + + [MethodImpl(MethodImplOption.NoInlining)] + static void Equals(T left, T right, [CallerLineNumber] int line = 0, [CallerFilePath] string file = "") + { + if (EqualityComparer.Default.Equals(left, right)) + { + Console.WriteLine($"{file}:L{line} test failed (expected: equal, actual: {left}-{right})."); + _errors++; + } + } + + [MethodImpl(MethodImplOption.NoInlining)] + static void IsTrue(bool expression, [CallerLineNumber] int line = 0, [CallerFilePath] string file = "") + { + if (!expression) + { + Console.WriteLine($"{file}:L{line} test failed (expected: true)."); + _errors++; + } + } + + [MethodImpl(MethodImplOption.NoInlining)] + static void IsFalse(bool expression, [CallerLineNumber] int line = 0, [CallerFilePath] string file = "") + { + if (expression) + { + Console.WriteLine($"{file}:L{line} test failed (expected: false)."); + _errors++; + } + } + + [MethodImpl(MethodImplOption.NoInlining)] + static void ThrowsNRE(Action action, [CallerLineNumber] int line = 0, [CallerFilePath] string file = "") + { + try + { + action(); + } + catch (NullReferenceException) + { + return; + } + catch (Exception exc) + { + Console.WriteLine($"{file}:L{line} {exc}"); + } + Console.WriteLine($"Line {line}: test failed (expected: NullReferenceException)"); + _errors++; + } + + public struct GenericStruct + { + public T field; + } + + public enum SimpleEnum + { + A,B,C + } + + struct StructWithByte + { + public byte Byte; + } + } +} diff --git a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference_r.csproj b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference_r.csproj new file mode 100644 index 00000000000000..d81aadf759378f --- /dev/null +++ b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference_r.csproj @@ -0,0 +1,13 @@ + + + Exe + + + true + None + + + + + + diff --git a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference_ro.csproj b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference_ro.csproj new file mode 100644 index 00000000000000..05f3a1f7575d05 --- /dev/null +++ b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference_ro.csproj @@ -0,0 +1,13 @@ + + + Exe + + + true + None + True + + + + + From a98f01ab651df26e9b0415293feec03803e67371 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Petryka?= <35800402+MichalPetryka@users.noreply.github.com> Date: Sat, 23 Jul 2022 23:29:06 +0200 Subject: [PATCH 02/31] Update importer.cpp --- src/coreclr/jit/importer.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 4885752e1a0f9c..3f4a73b5e655e0 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -3891,6 +3891,7 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, GenTree* index = gtNewIconNode(0, TYP_I_IMPL); GenTreeIndexAddr* indexAddr = gtNewArrayIndexAddr(array, index, elemType, elemHnd); indexAddr->gtFlags &= ~GTF_INX_RNGCHK; + retNode = indexAddr; break; } From 832180df4c90c57fdca897eaca725ae9b743050a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Petryka?= <35800402+MichalPetryka@users.noreply.github.com> Date: Sat, 23 Jul 2022 23:46:17 +0200 Subject: [PATCH 03/31] Update importer.cpp --- src/coreclr/jit/importer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 3f4a73b5e655e0..3bb33fc760958f 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -3886,7 +3886,7 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, GenTree* array = impPopStack().val; CORINFO_CLASS_HANDLE elemHnd = sig->sigInst.classInst[0]; CorInfoType jitType = info.compCompHnd->asCorInfoType(elemHnd); - var_type elemType = JITtype2varType(jitType); + var_types elemType = JITtype2varType(jitType); GenTree* index = gtNewIconNode(0, TYP_I_IMPL); GenTreeIndexAddr* indexAddr = gtNewArrayIndexAddr(array, index, elemType, elemHnd); From aaa574c1646385baa7e856fbdedae2cafdd2bb99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Petryka?= <35800402+MichalPetryka@users.noreply.github.com> Date: Sun, 24 Jul 2022 00:55:25 +0200 Subject: [PATCH 04/31] Update importer.cpp --- src/coreclr/jit/importer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 3bb33fc760958f..3ce92f8c7fadd9 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -3884,7 +3884,7 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, assert(sig->numArgs == 1); GenTree* array = impPopStack().val; - CORINFO_CLASS_HANDLE elemHnd = sig->sigInst.classInst[0]; + CORINFO_CLASS_HANDLE elemHnd = sig->sigInst.methInst[0]; CorInfoType jitType = info.compCompHnd->asCorInfoType(elemHnd); var_types elemType = JITtype2varType(jitType); From b831cd3dd45c337647f8c0480b7933614e36eb01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Petryka?= <35800402+MichalPetryka@users.noreply.github.com> Date: Sun, 24 Jul 2022 00:55:58 +0200 Subject: [PATCH 05/31] Update MemoryMarshalGetArrayDataReference.cs --- .../Intrinsics/MemoryMarshalGetArrayDataReference.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs index fbe3c93bebb0e8..1fb909a02f8b4b 100644 --- a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs +++ b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs @@ -51,7 +51,7 @@ unsafe static int Main(string[] args) ThrowsNRE(() => { _ = ref ptrString(null); }); // from https://github.com/dotnet/runtime/issues/58312#issuecomment-993491291 - [MethodImpl(MethodImplOption.NoInlining)] + [MethodImpl(MethodImplOptions.NoInlining)] static int Problem1(StructWithByte[] a) { MemoryMarshal.GetArrayDataReference(a).Byte = 1; @@ -63,7 +63,7 @@ static int Problem1(StructWithByte[] a) Equals(Problem(new StructWithByte[] { new StructWithByte { Byte = 1 } }), 2); - [MethodImpl(MethodImplOption.NoInlining)] + [MethodImpl(MethodImplOptions.NoInlining)] static int Problem2(byte[] a) { if (MemoryMarshal.GetArrayDataReference(a) == 1) @@ -83,7 +83,7 @@ static int Problem2(byte[] a) return 100 + _errors; } - [MethodImpl(MethodImplOption.NoInlining)] + [MethodImpl(MethodImplOptions.NoInlining)] static void Equals(T left, T right, [CallerLineNumber] int line = 0, [CallerFilePath] string file = "") { if (EqualityComparer.Default.Equals(left, right)) @@ -93,7 +93,7 @@ static void Equals(T left, T right, [CallerLineNumber] int line = 0, [CallerF } } - [MethodImpl(MethodImplOption.NoInlining)] + [MethodImpl(MethodImplOptions.NoInlining)] static void IsTrue(bool expression, [CallerLineNumber] int line = 0, [CallerFilePath] string file = "") { if (!expression) @@ -103,7 +103,7 @@ static void IsTrue(bool expression, [CallerLineNumber] int line = 0, [CallerFile } } - [MethodImpl(MethodImplOption.NoInlining)] + [MethodImpl(MethodImplOptions.NoInlining)] static void IsFalse(bool expression, [CallerLineNumber] int line = 0, [CallerFilePath] string file = "") { if (expression) @@ -113,7 +113,7 @@ static void IsFalse(bool expression, [CallerLineNumber] int line = 0, [CallerFil } } - [MethodImpl(MethodImplOption.NoInlining)] + [MethodImpl(MethodImplOptions.NoInlining)] static void ThrowsNRE(Action action, [CallerLineNumber] int line = 0, [CallerFilePath] string file = "") { try From ea730362be84a8480a9afbc338f90a88d4347d41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Petryka?= <35800402+MichalPetryka@users.noreply.github.com> Date: Sun, 24 Jul 2022 01:21:38 +0200 Subject: [PATCH 06/31] Update MemoryMarshalGetArrayDataReference.cs --- src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs index 1fb909a02f8b4b..46348d206cac7c 100644 --- a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs +++ b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; namespace MemoryMarshalGetArrayDataReferenceTest { @@ -61,7 +62,7 @@ static int Problem1(StructWithByte[] a) return MemoryMarshal.GetArrayDataReference(a).Byte; } - Equals(Problem(new StructWithByte[] { new StructWithByte { Byte = 1 } }), 2); + Equals(Problem1(new StructWithByte[] { new StructWithByte { Byte = 1 } }), 2); [MethodImpl(MethodImplOptions.NoInlining)] static int Problem2(byte[] a) From 4e2fb81a3449c172612a5f0d60a79f30475daa3c Mon Sep 17 00:00:00 2001 From: petris Date: Sun, 24 Jul 2022 14:34:54 +0200 Subject: [PATCH 07/31] Fix nullchecks, improve tests, remove dead code --- src/coreclr/jit/importer.cpp | 9 ++- .../TypeSystem/IL/NativeAotILProvider.cs | 6 -- .../IL/Stubs/MemoryMarshalIntrinsics.cs | 46 ------------- .../IL/ReadyToRunILProvider.cs | 9 +-- .../MemoryMarshalGetArrayDataReference.cs | 66 +++++++++++++++---- 5 files changed, 62 insertions(+), 74 deletions(-) delete mode 100644 src/coreclr/tools/Common/TypeSystem/IL/Stubs/MemoryMarshalIntrinsics.cs diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 3ce92f8c7fadd9..cce897904bfeb9 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -3888,10 +3888,17 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, CorInfoType jitType = info.compCompHnd->asCorInfoType(elemHnd); var_types elemType = JITtype2varType(jitType); + GenTree* arrayClone; + array = impCloneExpr(array, &arrayClone, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, + nullptr DEBUGARG("MemoryMarshal.GetArrayDataReference array")); + GenTree* index = gtNewIconNode(0, TYP_I_IMPL); GenTreeIndexAddr* indexAddr = gtNewArrayIndexAddr(array, index, elemType, elemHnd); indexAddr->gtFlags &= ~GTF_INX_RNGCHK; - retNode = indexAddr; + indexAddr->gtFlags |= GTF_INX_ADDR_NONNULL; + + GenTree* nullcheck = gtNewNullCheck(arrayClone, compCurBB); + retNode = gtNewOperNode(GT_COMMA, TYP_BYREF, nullcheck, indexAddr); break; } diff --git a/src/coreclr/tools/Common/TypeSystem/IL/NativeAotILProvider.cs b/src/coreclr/tools/Common/TypeSystem/IL/NativeAotILProvider.cs index 2de9aa4f8c0fba..4fec12a673e11e 100644 --- a/src/coreclr/tools/Common/TypeSystem/IL/NativeAotILProvider.cs +++ b/src/coreclr/tools/Common/TypeSystem/IL/NativeAotILProvider.cs @@ -58,12 +58,6 @@ private static MethodIL TryGetIntrinsicMethodIL(MethodDesc method) return UnsafeIntrinsics.EmitIL(method); } break; - case "MemoryMarshal": - { - if (owningType.Namespace == "System.Runtime.InteropServices") - return MemoryMarshalIntrinsics.EmitIL(method); - } - break; case "Volatile": { if (owningType.Namespace == "System.Threading") diff --git a/src/coreclr/tools/Common/TypeSystem/IL/Stubs/MemoryMarshalIntrinsics.cs b/src/coreclr/tools/Common/TypeSystem/IL/Stubs/MemoryMarshalIntrinsics.cs deleted file mode 100644 index aaa56db0888c74..00000000000000 --- a/src/coreclr/tools/Common/TypeSystem/IL/Stubs/MemoryMarshalIntrinsics.cs +++ /dev/null @@ -1,46 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -#if READYTORUN -using ILCompiler; -#endif -using Internal.TypeSystem; - -using Debug = System.Diagnostics.Debug; - -namespace Internal.IL.Stubs -{ - /// - /// Provides method bodies for System.Runtime.InteropServices.MemoryMarshal intrinsics. - /// - public static class MemoryMarshalIntrinsics - { - public static MethodIL EmitIL(MethodDesc method) - { - Debug.Assert(((MetadataType)method.OwningType).Name == "MemoryMarshal"); - string methodName = method.Name; - - if (method.Instantiation.Length != 1) - { - return null; // we only handle the generic method GetArrayDataReference(T[]) - } - - if (methodName == "GetArrayDataReference") - { - var rawArrayData = method.Context.SystemModule.GetKnownType("System.Runtime.CompilerServices", "RawArrayData"); -#if READYTORUN - if (!rawArrayData.IsNonVersionable()) - return null; // This is only an intrinsic if we can prove that RawArrayData is known to be of fixed offset -#endif - ILEmitter emit = new ILEmitter(); - ILCodeStream codeStream = emit.NewCodeStream(); - codeStream.EmitLdArg(0); - codeStream.Emit(ILOpcode.ldflda, emit.NewToken(rawArrayData.GetField("Data"))); - codeStream.Emit(ILOpcode.ret); - return emit.Link(method); - } - - // unknown method - return null; - } - } -} diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/IL/ReadyToRunILProvider.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/IL/ReadyToRunILProvider.cs index 686557fcd25c75..4e7a89d670a03d 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/IL/ReadyToRunILProvider.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/IL/ReadyToRunILProvider.cs @@ -85,11 +85,6 @@ private MethodIL TryGetIntrinsicMethodIL(MethodDesc method) return UnsafeIntrinsics.EmitIL(method); } - if (mdType.Name == "MemoryMarshal" && mdType.Namespace == "System.Runtime.InteropServices") - { - return MemoryMarshalIntrinsics.EmitIL(method); - } - if (mdType.Name == "Volatile" && mdType.Namespace == "System.Threading") { return VolatileIntrinsics.EmitIL(method); @@ -171,7 +166,7 @@ public override MethodIL GetMethodIL(MethodDesc method) } // Check to see if there is an override for the EcmaMethodIL. If there is not - // then simply return the EcmaMethodIL. In theory this could call + // then simply return the EcmaMethodIL. In theory this could call // CreateCrossModuleInlineableTokensForILBody, but we explicitly do not want // to do that. The reason is that this method is called during the multithreaded // portion of compilation, and CreateCrossModuleInlineableTokensForILBody @@ -224,7 +219,7 @@ class ManifestModuleWrappedMethodIL : MethodIL, IEcmaMethodIL, IMethodTokensAreU MutableModule _mutableModule; public ManifestModuleWrappedMethodIL() {} - + public bool Initialize(MutableModule mutableModule, EcmaMethodIL wrappedMethod) { bool failedToReplaceToken = false; diff --git a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs index 46348d206cac7c..5576f1f77a7259 100644 --- a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs +++ b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs @@ -16,28 +16,54 @@ class Program unsafe static int Main(string[] args) { - delegate* ptrByte = &MemoryMarshal.GetArrayDataReference; - delegate* ptrString = &MemoryMarshal.GetArrayDataReference; + // use no inline methods to avoid indirect call inlining in the future + [MethodImpl(MethodImplOptions.NoInlining)] + static delegate* GetBytePtr() => &MemoryMarshal.GetArrayDataReference; + delegate* ptrByte = GetBytePtr(); + [MethodImpl(MethodImplOptions.NoInlining)] + static delegate* GetStringPtr() => &MemoryMarshal.GetArrayDataReference; + delegate* ptrString = GetStringPtr(); + + [MethodImpl(MethodImplOptions.NoInlining)] + static T NoInline(T t) => t; byte[] testByteArray = new byte[1]; IsTrue(Unsafe.AreSame(ref MemoryMarshal.GetArrayDataReference(testByteArray), ref testByteArray[0])); IsTrue(Unsafe.AreSame(ref ptrByte(testByteArray), ref testByteArray[0])); + IsTrue(Unsafe.AreSame(ref MemoryMarshal.GetArrayDataReference(NoInline(testByteArray)), ref testByteArray[0])); + IsTrue(Unsafe.AreSame(ref ptrByte(NoInline(testByteArray)), ref testByteArray[0])); + string[] testStringArray = new string[1]; IsTrue(Unsafe.AreSame(ref MemoryMarshal.GetArrayDataReference(testStringArray), ref testStringArray[0])); IsTrue(Unsafe.AreSame(ref ptrString(testStringArray), ref testStringArray[0])); - IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(Array.Empty()))); - IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(Array.Empty()))); - IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(Array.Empty()))); - IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(Array.Empty>()))); - IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(Array.Empty()))); - IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(Array.Empty()))); - IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(Array.Empty>()))); - IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(Array.Empty>()))); - - IsFalse(Unsafe.IsNullRef(ref ptrByte(Array.Empty()))); - IsFalse(Unsafe.IsNullRef(ref ptrString(Array.Empty()))); + IsTrue(Unsafe.AreSame(ref MemoryMarshal.GetArrayDataReference(NoInline(testStringArray)), ref testStringArray[0])); + IsTrue(Unsafe.AreSame(ref ptrString(NoInline(testStringArray)), ref testStringArray[0])); + + IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(new byte[0]))); + IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(new string[0]))); + IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(new Half[0]))); + IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(new Vector128[0]))); + IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(new StructWithByte[0]))); + IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(new SimpleEnum[0]))); + IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(new GenericStruct[0]))); + IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(new GenericStruct[0]))); + + IsFalse(Unsafe.IsNullRef(ref ptrByte(new byte[0]))); + IsFalse(Unsafe.IsNullRef(ref ptrString(new string[0]))); + + IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(NoInline(new byte[0])))); + IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(NoInline(new string[0])))); + IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(NoInline(new Half[0])))); + IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(NoInline(new Vector128[0])))); + IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(NoInline(new StructWithByte[0])))); + IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(NoInline(new SimpleEnum[0])))); + IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(NoInline(new GenericStruct[0])))); + IsFalse(Unsafe.IsNullRef(ref MemoryMarshal.GetArrayDataReference(NoInline(new GenericStruct[0])))); + + IsFalse(Unsafe.IsNullRef(ref ptrByte(NoInline(new byte[0])))); + IsFalse(Unsafe.IsNullRef(ref ptrString(NoInline(new string[0])))); ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference(null); }); ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference(null); }); @@ -51,6 +77,18 @@ unsafe static int Main(string[] args) ThrowsNRE(() => { _ = ref ptrByte(null); }); ThrowsNRE(() => { _ = ref ptrString(null); }); + ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference(NoInline(null)); }); + ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference(NoInline(null)); }); + ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference(NoInline(null)); }); + ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference>(NoInline(null)); }); + ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference(NoInline(null)); }); + ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference(NoInline(null)); }); + ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference>(NoInline(null)); }); + ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference>(NoInline(null)); }); + + ThrowsNRE(() => { _ = ref ptrByte(NoInline(null)); }); + ThrowsNRE(() => { _ = ref ptrString(NoInline(null)); }); + // from https://github.com/dotnet/runtime/issues/58312#issuecomment-993491291 [MethodImpl(MethodImplOptions.NoInlining)] static int Problem1(StructWithByte[] a) @@ -87,7 +125,7 @@ static int Problem2(byte[] a) [MethodImpl(MethodImplOptions.NoInlining)] static void Equals(T left, T right, [CallerLineNumber] int line = 0, [CallerFilePath] string file = "") { - if (EqualityComparer.Default.Equals(left, right)) + if (!EqualityComparer.Default.Equals(left, right)) { Console.WriteLine($"{file}:L{line} test failed (expected: equal, actual: {left}-{right})."); _errors++; From b42e81ae3cee4f128a03b4031e09439a437974e6 Mon Sep 17 00:00:00 2001 From: petris Date: Sun, 24 Jul 2022 14:57:14 +0200 Subject: [PATCH 08/31] Update project files --- .../tools/aot/ILCompiler.Compiler/ILCompiler.Compiler.csproj | 3 --- .../aot/ILCompiler.ReadyToRun/ILCompiler.ReadyToRun.csproj | 1 - 2 files changed, 4 deletions(-) diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/ILCompiler.Compiler.csproj b/src/coreclr/tools/aot/ILCompiler.Compiler/ILCompiler.Compiler.csproj index 5887e5d4ff2a7c..771e89bfcea63d 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/ILCompiler.Compiler.csproj +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/ILCompiler.Compiler.csproj @@ -655,9 +655,6 @@ IL\Stubs\UnsafeIntrinsics.cs - - IL\Stubs\MemoryMarshalIntrinsics.cs - IL\Stubs\VolatileIntrinsics.cs diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ILCompiler.ReadyToRun.csproj b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ILCompiler.ReadyToRun.csproj index 4d8b7c38dda4e5..6096d476c806c9 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ILCompiler.ReadyToRun.csproj +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ILCompiler.ReadyToRun.csproj @@ -45,7 +45,6 @@ - From 77603b894bf5b6e1d45be1b7c2e44320d5c50706 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Petryka?= <35800402+MichalPetryka@users.noreply.github.com> Date: Sun, 24 Jul 2022 15:22:49 +0200 Subject: [PATCH 09/31] Update MemoryMarshalGetArrayDataReference.cs --- .../MemoryMarshalGetArrayDataReference.cs | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs index 5576f1f77a7259..33922addedbaa7 100644 --- a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs +++ b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs @@ -77,17 +77,17 @@ unsafe static int Main(string[] args) ThrowsNRE(() => { _ = ref ptrByte(null); }); ThrowsNRE(() => { _ = ref ptrString(null); }); - ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference(NoInline(null)); }); - ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference(NoInline(null)); }); - ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference(NoInline(null)); }); - ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference>(NoInline(null)); }); - ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference(NoInline(null)); }); - ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference(NoInline(null)); }); - ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference>(NoInline(null)); }); - ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference>(NoInline(null)); }); - - ThrowsNRE(() => { _ = ref ptrByte(NoInline(null)); }); - ThrowsNRE(() => { _ = ref ptrString(NoInline(null)); }); + ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference(NoInline(null)); }); + ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference(NoInline(null)); }); + ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference(NoInline(null)); }); + ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference(NoInline[]>(null)); }); + ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference(NoInline(null)); }); + ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference(NoInline(null)); }); + ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference(NoInline[]>(null)); }); + ThrowsNRE(() => { _ = ref MemoryMarshal.GetArrayDataReference(NoInline[]>(null)); }); + + ThrowsNRE(() => { _ = ref ptrByte(NoInline(null)); }); + ThrowsNRE(() => { _ = ref ptrString(NoInline(null)); }); // from https://github.com/dotnet/runtime/issues/58312#issuecomment-993491291 [MethodImpl(MethodImplOptions.NoInlining)] From d9f07e6493c8514062e588eb53599896054210d0 Mon Sep 17 00:00:00 2001 From: SingleAccretion Date: Mon, 25 Jul 2022 00:20:45 +0300 Subject: [PATCH 10/31] Fix cloning of bounds-check-less INDEX_ADDRs --- src/coreclr/jit/compiler.hpp | 8 +++++++- src/coreclr/jit/gentree.cpp | 2 +- src/coreclr/jit/gentree.h | 12 ++++-------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 946866d63c888f..8f5e064e84c159 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -1099,8 +1099,14 @@ inline GenTreeIndexAddr* Compiler::gtNewIndexAddr(GenTree* arrayOp, unsigned elemSize = (elemType == TYP_STRUCT) ? info.compCompHnd->getClassSize(elemClassHandle) : genTypeSize(elemType); +#ifdef DEBUG + bool boundsCheck = JitConfig.JitSkipArrayBoundCheck() != 1; +#else + bool boundsCheck = true; +#endif + GenTreeIndexAddr* indexAddr = new (this, GT_INDEX_ADDR) - GenTreeIndexAddr(arrayOp, indexOp, elemType, elemClassHandle, elemSize, lengthOffset, firstElemOffset); + GenTreeIndexAddr(arrayOp, indexOp, elemType, elemClassHandle, elemSize, lengthOffset, firstElemOffset, boundsCheck); return indexAddr; } diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index cfeff19efa8fe9..46f302338d4e2a 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -8580,7 +8580,7 @@ GenTree* Compiler::gtCloneExpr( copy = new (this, GT_INDEX_ADDR) GenTreeIndexAddr(asIndAddr->Arr(), asIndAddr->Index(), asIndAddr->gtElemType, asIndAddr->gtStructElemClass, asIndAddr->gtElemSize, asIndAddr->gtLenOffset, - asIndAddr->gtElemOffset); + asIndAddr->gtElemOffset, asIndAddr->IsBoundsChecked()); copy->AsIndexAddr()->gtIndRngFailBB = asIndAddr->gtIndRngFailBB; } break; diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 6c29054b8ddc21..f27c18eb84ec29 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -6530,7 +6530,8 @@ struct GenTreeIndexAddr : public GenTreeOp CORINFO_CLASS_HANDLE structElemClass, unsigned elemSize, unsigned lenOffset, - unsigned elemOffset) + unsigned elemOffset, + bool boundsCheck) : GenTreeOp(GT_INDEX_ADDR, TYP_BYREF, arr, ind) , gtStructElemClass(structElemClass) , gtIndRngFailBB(nullptr) @@ -6540,13 +6541,8 @@ struct GenTreeIndexAddr : public GenTreeOp , gtElemOffset(elemOffset) { assert(!varTypeIsStruct(elemType) || (structElemClass != NO_CLASS_HANDLE)); -#ifdef DEBUG - if (JitConfig.JitSkipArrayBoundCheck() == 1) - { - // Skip bounds check - } - else -#endif + + if (boundsCheck) { // Do bounds check gtFlags |= GTF_INX_RNGCHK; From 2f3ccde4ae17369e2bfc3a7a2e6e296addb4b78f Mon Sep 17 00:00:00 2001 From: petris Date: Mon, 25 Jul 2022 00:13:47 +0200 Subject: [PATCH 11/31] Fix formatting --- src/coreclr/jit/compiler.hpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 8f5e064e84c159..c153944aa38f68 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -918,7 +918,7 @@ inline GenTreeIntCon* Compiler::gtNewIconHandleNode(size_t value, GenTreeFlags f #if defined(LATE_DISASM) node = new (this, LargeOpOpcode()) GenTreeIntCon(TYP_I_IMPL, value, fields DEBUGARG(/*largeNode*/ true)); #else - node = new (this, GT_CNS_INT) GenTreeIntCon(TYP_I_IMPL, value, fields); + node = new (this, GT_CNS_INT) GenTreeIntCon(TYP_I_IMPL, value, fields); #endif node->gtFlags |= flags; return node; @@ -1105,8 +1105,9 @@ inline GenTreeIndexAddr* Compiler::gtNewIndexAddr(GenTree* arrayOp, bool boundsCheck = true; #endif - GenTreeIndexAddr* indexAddr = new (this, GT_INDEX_ADDR) - GenTreeIndexAddr(arrayOp, indexOp, elemType, elemClassHandle, elemSize, lengthOffset, firstElemOffset, boundsCheck); + GenTreeIndexAddr* indexAddr = + new (this, GT_INDEX_ADDR) GenTreeIndexAddr(arrayOp, indexOp, elemType, elemClassHandle, elemSize, lengthOffset, + firstElemOffset, boundsCheck); return indexAddr; } From d476b0d5f6b18fc116cf8ea23e2c967915eb960e Mon Sep 17 00:00:00 2001 From: petris Date: Mon, 25 Jul 2022 15:39:58 +0200 Subject: [PATCH 12/31] Remove COMMA --- src/coreclr/jit/importer.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index cce897904bfeb9..37fdbaddb7e691 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -3892,13 +3892,13 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, array = impCloneExpr(array, &arrayClone, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, nullptr DEBUGARG("MemoryMarshal.GetArrayDataReference array")); + impAppendTree(gtNewNullCheck(arrayClone, compCurBB), (unsigned)CHECK_SPILL_ALL, impCurStmtDI); + GenTree* index = gtNewIconNode(0, TYP_I_IMPL); GenTreeIndexAddr* indexAddr = gtNewArrayIndexAddr(array, index, elemType, elemHnd); indexAddr->gtFlags &= ~GTF_INX_RNGCHK; indexAddr->gtFlags |= GTF_INX_ADDR_NONNULL; - - GenTree* nullcheck = gtNewNullCheck(arrayClone, compCurBB); - retNode = gtNewOperNode(GT_COMMA, TYP_BYREF, nullcheck, indexAddr); + retNode = indexAddr; break; } From 83bff3fb107264fe76e3a118b5f711ca6defb647 Mon Sep 17 00:00:00 2001 From: petris Date: Mon, 25 Jul 2022 18:51:12 +0200 Subject: [PATCH 13/31] Move the nullcheck insertion to morph --- src/coreclr/jit/gentree.h | 8 +++++++- src/coreclr/jit/importer.cpp | 8 +------- src/coreclr/jit/morph.cpp | 33 ++++++++++++++++++++++++++++++++- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index f27c18eb84ec29..1760e9119b8f67 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -498,6 +498,7 @@ enum GenTreeFlags : unsigned int GTF_INX_RNGCHK = 0x80000000, // GT_INDEX_ADDR -- this array address should be range-checked GTF_INX_ADDR_NONNULL = 0x40000000, // GT_INDEX_ADDR -- this array address is not null + GTF_INX_ADDR_NONNULL = 0x20000000, // GT_INDEX_ADDR -- this array must be null checked GTF_IND_TGT_NOT_HEAP = 0x80000000, // GT_IND -- the target is not on the heap GTF_IND_VOLATILE = 0x40000000, // GT_IND -- the load or store must use volatile semantics (this is a nop on X86) @@ -6562,9 +6563,14 @@ struct GenTreeIndexAddr : public GenTreeOp return (gtFlags & GTF_INX_RNGCHK) != 0; } + bool IsNullChecked() const + { + return (gtFlags & GTF_INX_MUST_NULLCHECK) != 0; + } + bool IsNotNull() const { - return IsBoundsChecked() || ((gtFlags & GTF_INX_ADDR_NONNULL) != 0); + return IsBoundsChecked() || IsNullChecked() || ((gtFlags & GTF_INX_ADDR_NONNULL) != 0); } }; diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 37fdbaddb7e691..826842ae29d72a 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -3888,16 +3888,10 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, CorInfoType jitType = info.compCompHnd->asCorInfoType(elemHnd); var_types elemType = JITtype2varType(jitType); - GenTree* arrayClone; - array = impCloneExpr(array, &arrayClone, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, - nullptr DEBUGARG("MemoryMarshal.GetArrayDataReference array")); - - impAppendTree(gtNewNullCheck(arrayClone, compCurBB), (unsigned)CHECK_SPILL_ALL, impCurStmtDI); - GenTree* index = gtNewIconNode(0, TYP_I_IMPL); GenTreeIndexAddr* indexAddr = gtNewArrayIndexAddr(array, index, elemType, elemHnd); indexAddr->gtFlags &= ~GTF_INX_RNGCHK; - indexAddr->gtFlags |= GTF_INX_ADDR_NONNULL; + indexAddr->gtFlags |= GTF_INX_MUST_NULLCHECK; retNode = indexAddr; break; } diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 14403a8c0f7963..4225e211b81d84 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -4342,6 +4342,7 @@ GenTree* Compiler::fgMorphIndexAddr(GenTreeIndexAddr* indexAddr) GenTree* arrRefDefn = nullptr; // non-NULL if we need to allocate a temp for the arrRef expression GenTree* indexDefn = nullptr; // non-NULL if we need to allocate a temp for the index expression GenTreeBoundsChk* boundsCheck = nullptr; + GenTree* nullCheck = nullptr; // If we're doing range checking, introduce a GT_BOUNDS_CHECK node for the address. if (indexAddr->IsBoundsChecked()) @@ -4422,6 +4423,32 @@ GenTree* Compiler::fgMorphIndexAddr(GenTreeIndexAddr* indexAddr) arrRef = arrRef2; index = index2; } + // bounds checking implies null checking, this is only for no bounds check accesses + else if (indexAddr->IsNullChecked()) + { + GenTree* arrRef2 = nullptr; // The second copy will be used in array address expression + + if (((arrRef->gtFlags & (GTF_ASG | GTF_CALL | GTF_GLOB_REF)) != 0) || + gtComplexityExceeds(&arrRef, MAX_ARR_COMPLEXITY) || arrRef->OperIs(GT_FIELD, GT_LCL_FLD) || + (arrRef->OperIs(GT_LCL_VAR) && lvaIsLocalImplicitlyAccessedByRef(arrRef->AsLclVar()->GetLclNum()))) + { + unsigned arrRefTmpNum = lvaGrabTemp(true DEBUGARG("arr expr")); + arrRefDefn = gtNewTempAssign(arrRefTmpNum, arrRef); + arrRef = gtNewLclvNode(arrRefTmpNum, arrRef->TypeGet()); + arrRef2 = gtNewLclvNode(arrRefTmpNum, arrRef->TypeGet()); + } + else + { + arrRef2 = gtCloneExpr(arrRef); + noway_assert(arrRef2 != nullptr); + } + + nullCheck = gtNewNullCheck(arrRef, compCurBB); + + // Now we'll switch to using the second copy for arrRef + // to compute the address expression + arrRef = arrRef2; + } // Create the "addr" which is "*(arrRef + ((index * elemSize) + elemOffs))" GenTree* addr; @@ -4517,7 +4544,7 @@ GenTree* Compiler::fgMorphIndexAddr(GenTreeIndexAddr* indexAddr) GenTree* tree = addr; - // Prepend the bounds check and the assignment trees that were created (if any). + // Prepend the bounds/null check and the assignment trees that were created (if any). if (boundsCheck != nullptr) { // This is changing a value dependency (INDEX_ADDR node) into a flow @@ -4530,6 +4557,10 @@ GenTree* Compiler::fgMorphIndexAddr(GenTreeIndexAddr* indexAddr) tree = gtNewOperNode(GT_COMMA, tree->TypeGet(), boundsCheck, tree); fgSetRngChkTarget(boundsCheck); } + else if (nullCheck != nullptr) + { + tree = gtNewOperNode(GT_COMMA, tree->TypeGet(), nullCheck, tree); + } if (indexDefn != nullptr) { From f35d9ddc413ae24d29839c76a73db6affd193e2e Mon Sep 17 00:00:00 2001 From: petris Date: Mon, 25 Jul 2022 19:22:10 +0200 Subject: [PATCH 14/31] Fix compilation --- src/coreclr/jit/gentree.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 1760e9119b8f67..603d2bf8ae0a14 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -498,7 +498,7 @@ enum GenTreeFlags : unsigned int GTF_INX_RNGCHK = 0x80000000, // GT_INDEX_ADDR -- this array address should be range-checked GTF_INX_ADDR_NONNULL = 0x40000000, // GT_INDEX_ADDR -- this array address is not null - GTF_INX_ADDR_NONNULL = 0x20000000, // GT_INDEX_ADDR -- this array must be null checked + GTF_INX_MUST_NULLCHECK = 0x20000000, // GT_INDEX_ADDR -- this array must be null checked GTF_IND_TGT_NOT_HEAP = 0x80000000, // GT_IND -- the target is not on the heap GTF_IND_VOLATILE = 0x40000000, // GT_IND -- the load or store must use volatile semantics (this is a nop on X86) From 3944c638aa7524fd4734d7833fdfcf56761923fe Mon Sep 17 00:00:00 2001 From: petris Date: Tue, 26 Jul 2022 17:11:48 +0200 Subject: [PATCH 15/31] Revert morph changes --- src/coreclr/jit/gentree.h | 8 +------- src/coreclr/jit/importer.cpp | 8 +++++++- src/coreclr/jit/morph.cpp | 33 +-------------------------------- 3 files changed, 9 insertions(+), 40 deletions(-) diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 603d2bf8ae0a14..f27c18eb84ec29 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -498,7 +498,6 @@ enum GenTreeFlags : unsigned int GTF_INX_RNGCHK = 0x80000000, // GT_INDEX_ADDR -- this array address should be range-checked GTF_INX_ADDR_NONNULL = 0x40000000, // GT_INDEX_ADDR -- this array address is not null - GTF_INX_MUST_NULLCHECK = 0x20000000, // GT_INDEX_ADDR -- this array must be null checked GTF_IND_TGT_NOT_HEAP = 0x80000000, // GT_IND -- the target is not on the heap GTF_IND_VOLATILE = 0x40000000, // GT_IND -- the load or store must use volatile semantics (this is a nop on X86) @@ -6563,14 +6562,9 @@ struct GenTreeIndexAddr : public GenTreeOp return (gtFlags & GTF_INX_RNGCHK) != 0; } - bool IsNullChecked() const - { - return (gtFlags & GTF_INX_MUST_NULLCHECK) != 0; - } - bool IsNotNull() const { - return IsBoundsChecked() || IsNullChecked() || ((gtFlags & GTF_INX_ADDR_NONNULL) != 0); + return IsBoundsChecked() || ((gtFlags & GTF_INX_ADDR_NONNULL) != 0); } }; diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 826842ae29d72a..37fdbaddb7e691 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -3888,10 +3888,16 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, CorInfoType jitType = info.compCompHnd->asCorInfoType(elemHnd); var_types elemType = JITtype2varType(jitType); + GenTree* arrayClone; + array = impCloneExpr(array, &arrayClone, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, + nullptr DEBUGARG("MemoryMarshal.GetArrayDataReference array")); + + impAppendTree(gtNewNullCheck(arrayClone, compCurBB), (unsigned)CHECK_SPILL_ALL, impCurStmtDI); + GenTree* index = gtNewIconNode(0, TYP_I_IMPL); GenTreeIndexAddr* indexAddr = gtNewArrayIndexAddr(array, index, elemType, elemHnd); indexAddr->gtFlags &= ~GTF_INX_RNGCHK; - indexAddr->gtFlags |= GTF_INX_MUST_NULLCHECK; + indexAddr->gtFlags |= GTF_INX_ADDR_NONNULL; retNode = indexAddr; break; } diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 4225e211b81d84..14403a8c0f7963 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -4342,7 +4342,6 @@ GenTree* Compiler::fgMorphIndexAddr(GenTreeIndexAddr* indexAddr) GenTree* arrRefDefn = nullptr; // non-NULL if we need to allocate a temp for the arrRef expression GenTree* indexDefn = nullptr; // non-NULL if we need to allocate a temp for the index expression GenTreeBoundsChk* boundsCheck = nullptr; - GenTree* nullCheck = nullptr; // If we're doing range checking, introduce a GT_BOUNDS_CHECK node for the address. if (indexAddr->IsBoundsChecked()) @@ -4423,32 +4422,6 @@ GenTree* Compiler::fgMorphIndexAddr(GenTreeIndexAddr* indexAddr) arrRef = arrRef2; index = index2; } - // bounds checking implies null checking, this is only for no bounds check accesses - else if (indexAddr->IsNullChecked()) - { - GenTree* arrRef2 = nullptr; // The second copy will be used in array address expression - - if (((arrRef->gtFlags & (GTF_ASG | GTF_CALL | GTF_GLOB_REF)) != 0) || - gtComplexityExceeds(&arrRef, MAX_ARR_COMPLEXITY) || arrRef->OperIs(GT_FIELD, GT_LCL_FLD) || - (arrRef->OperIs(GT_LCL_VAR) && lvaIsLocalImplicitlyAccessedByRef(arrRef->AsLclVar()->GetLclNum()))) - { - unsigned arrRefTmpNum = lvaGrabTemp(true DEBUGARG("arr expr")); - arrRefDefn = gtNewTempAssign(arrRefTmpNum, arrRef); - arrRef = gtNewLclvNode(arrRefTmpNum, arrRef->TypeGet()); - arrRef2 = gtNewLclvNode(arrRefTmpNum, arrRef->TypeGet()); - } - else - { - arrRef2 = gtCloneExpr(arrRef); - noway_assert(arrRef2 != nullptr); - } - - nullCheck = gtNewNullCheck(arrRef, compCurBB); - - // Now we'll switch to using the second copy for arrRef - // to compute the address expression - arrRef = arrRef2; - } // Create the "addr" which is "*(arrRef + ((index * elemSize) + elemOffs))" GenTree* addr; @@ -4544,7 +4517,7 @@ GenTree* Compiler::fgMorphIndexAddr(GenTreeIndexAddr* indexAddr) GenTree* tree = addr; - // Prepend the bounds/null check and the assignment trees that were created (if any). + // Prepend the bounds check and the assignment trees that were created (if any). if (boundsCheck != nullptr) { // This is changing a value dependency (INDEX_ADDR node) into a flow @@ -4557,10 +4530,6 @@ GenTree* Compiler::fgMorphIndexAddr(GenTreeIndexAddr* indexAddr) tree = gtNewOperNode(GT_COMMA, tree->TypeGet(), boundsCheck, tree); fgSetRngChkTarget(boundsCheck); } - else if (nullCheck != nullptr) - { - tree = gtNewOperNode(GT_COMMA, tree->TypeGet(), nullCheck, tree); - } if (indexDefn != nullptr) { From 2f65de6941f8ae60aa3f5277a4639799a9514a01 Mon Sep 17 00:00:00 2001 From: petris Date: Tue, 26 Jul 2022 17:53:23 +0200 Subject: [PATCH 16/31] Try a hack to see if the diffs are better --- src/coreclr/jit/importer.cpp | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 37fdbaddb7e691..c6ce606156db59 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -3883,10 +3883,7 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, { assert(sig->numArgs == 1); - GenTree* array = impPopStack().val; - CORINFO_CLASS_HANDLE elemHnd = sig->sigInst.methInst[0]; - CorInfoType jitType = info.compCompHnd->asCorInfoType(elemHnd); - var_types elemType = JITtype2varType(jitType); + GenTree* array = impPopStack().val; GenTree* arrayClone; array = impCloneExpr(array, &arrayClone, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, @@ -3894,11 +3891,8 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, impAppendTree(gtNewNullCheck(arrayClone, compCurBB), (unsigned)CHECK_SPILL_ALL, impCurStmtDI); - GenTree* index = gtNewIconNode(0, TYP_I_IMPL); - GenTreeIndexAddr* indexAddr = gtNewArrayIndexAddr(array, index, elemType, elemHnd); - indexAddr->gtFlags &= ~GTF_INX_RNGCHK; - indexAddr->gtFlags |= GTF_INX_ADDR_NONNULL; - retNode = indexAddr; + GenTree* offset = gtNewIconNode(OFFSETOF__CORINFO_Array__data, TYP_I_IMPL); + retNode = gtNewOperNode(GT_ADD, TYP_BYREF, array, offset); break; } From eb7614f644727169dd536000a89ac8856ee3a809 Mon Sep 17 00:00:00 2001 From: petris Date: Wed, 27 Jul 2022 00:38:17 +0200 Subject: [PATCH 17/31] Revert "Try a hack to see if the diffs are better" This reverts commit 7af6e18a18457fa5f3eab4e73a5801ce9a9b2a16. --- src/coreclr/jit/importer.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index c6ce606156db59..37fdbaddb7e691 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -3883,7 +3883,10 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, { assert(sig->numArgs == 1); - GenTree* array = impPopStack().val; + GenTree* array = impPopStack().val; + CORINFO_CLASS_HANDLE elemHnd = sig->sigInst.methInst[0]; + CorInfoType jitType = info.compCompHnd->asCorInfoType(elemHnd); + var_types elemType = JITtype2varType(jitType); GenTree* arrayClone; array = impCloneExpr(array, &arrayClone, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, @@ -3891,8 +3894,11 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, impAppendTree(gtNewNullCheck(arrayClone, compCurBB), (unsigned)CHECK_SPILL_ALL, impCurStmtDI); - GenTree* offset = gtNewIconNode(OFFSETOF__CORINFO_Array__data, TYP_I_IMPL); - retNode = gtNewOperNode(GT_ADD, TYP_BYREF, array, offset); + GenTree* index = gtNewIconNode(0, TYP_I_IMPL); + GenTreeIndexAddr* indexAddr = gtNewArrayIndexAddr(array, index, elemType, elemHnd); + indexAddr->gtFlags &= ~GTF_INX_RNGCHK; + indexAddr->gtFlags |= GTF_INX_ADDR_NONNULL; + retNode = indexAddr; break; } From 083b88fae76e71f054ba3b76e7176fdf124c5eab Mon Sep 17 00:00:00 2001 From: petris Date: Wed, 27 Jul 2022 00:46:49 +0200 Subject: [PATCH 18/31] Add more tests --- src/coreclr/jit/importer.cpp | 4 +- .../MemoryMarshalGetArrayDataReference.cs | 78 ++++++++++++++++++- 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 37fdbaddb7e691..46c36c964b860c 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -3892,10 +3892,10 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, array = impCloneExpr(array, &arrayClone, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, nullptr DEBUGARG("MemoryMarshal.GetArrayDataReference array")); - impAppendTree(gtNewNullCheck(arrayClone, compCurBB), (unsigned)CHECK_SPILL_ALL, impCurStmtDI); + impAppendTree(gtNewNullCheck(array, compCurBB), (unsigned)CHECK_SPILL_ALL, impCurStmtDI); GenTree* index = gtNewIconNode(0, TYP_I_IMPL); - GenTreeIndexAddr* indexAddr = gtNewArrayIndexAddr(array, index, elemType, elemHnd); + GenTreeIndexAddr* indexAddr = gtNewArrayIndexAddr(arrayClone, index, elemType, elemHnd); indexAddr->gtFlags &= ~GTF_INX_RNGCHK; indexAddr->gtFlags |= GTF_INX_ADDR_NONNULL; retNode = indexAddr; diff --git a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs index 33922addedbaa7..7a3f1bf5a19a6f 100644 --- a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs +++ b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs @@ -89,6 +89,30 @@ unsafe static int Main(string[] args) ThrowsNRE(() => { _ = ref ptrByte(NoInline(null)); }); ThrowsNRE(() => { _ = ref ptrString(NoInline(null)); }); + ThrowsNRE(() => ref MemoryMarshal.GetArrayDataReference(null)); + ThrowsNRE(() => ref MemoryMarshal.GetArrayDataReference(null)); + ThrowsNRE(() => ref MemoryMarshal.GetArrayDataReference(null)); + ThrowsNRE(() => ref MemoryMarshal.GetArrayDataReference>(null)); + ThrowsNRE(() => ref MemoryMarshal.GetArrayDataReference(null)); + ThrowsNRE(() => ref MemoryMarshal.GetArrayDataReference(null)); + ThrowsNRE(() => ref MemoryMarshal.GetArrayDataReference>(null)); + ThrowsNRE(() => ref MemoryMarshal.GetArrayDataReference>(null)); + + ThrowsNRE(() => ref ptrByte(null)); + ThrowsNRE(() => ref ptrString(null)); + + ThrowsNRE(() => ref MemoryMarshal.GetArrayDataReference(NoInline(null))); + ThrowsNRE(() => ref MemoryMarshal.GetArrayDataReference(NoInline(null))); + ThrowsNRE(() => ref MemoryMarshal.GetArrayDataReference(NoInline(null))); + ThrowsNRE(() => ref MemoryMarshal.GetArrayDataReference(NoInline[]>(null))); + ThrowsNRE(() => ref MemoryMarshal.GetArrayDataReference(NoInline(null))); + ThrowsNRE(() => ref MemoryMarshal.GetArrayDataReference(NoInline(null))); + ThrowsNRE(() => ref MemoryMarshal.GetArrayDataReference(NoInline[]>(null))); + ThrowsNRE(() => ref MemoryMarshal.GetArrayDataReference(NoInline[]>(null))); + + ThrowsNRE(() => ref ptrByte(NoInline(null))); + ThrowsNRE(() => ref ptrString(NoInline(null))); + // from https://github.com/dotnet/runtime/issues/58312#issuecomment-993491291 [MethodImpl(MethodImplOptions.NoInlining)] static int Problem1(StructWithByte[] a) @@ -103,7 +127,19 @@ static int Problem1(StructWithByte[] a) Equals(Problem1(new StructWithByte[] { new StructWithByte { Byte = 1 } }), 2); [MethodImpl(MethodImplOptions.NoInlining)] - static int Problem2(byte[] a) + static int Problem2(StructWithByte[] a) + { + Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(a), 1).Byte = 1; + + a[1].Byte = 2; + + return Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(a), 1).Byte; + } + + Equals(Problem2(new StructWithByte[] { new StructWithByte { Byte = 1 }, new StructWithByte { Byte = 1 } }), 2); + + [MethodImpl(MethodImplOptions.NoInlining)] + static int Problem3(byte[] a) { if (MemoryMarshal.GetArrayDataReference(a) == 1) { @@ -117,7 +153,24 @@ static int Problem2(byte[] a) return 0; } - Equals(Problem2(new byte[] { 1 }), 0); + Equals(Problem3(new byte[] { 1 }), 0); + + [MethodImpl(MethodImplOptions.NoInlining)] + static int Problem4(byte[] a) + { + if (Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(a), 1) == 1) + { + a[1] = 2; + if (Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(a), 1) == 1) + { + return -1; + } + } + + return 0; + } + + Equals(Problem4(new byte[] { 1, 1 }), 0); return 100 + _errors; } @@ -171,6 +224,25 @@ static void ThrowsNRE(Action action, [CallerLineNumber] int line = 0, [CallerFil _errors++; } + [MethodImpl(MethodImplOptions.NoInlining)] + static void ThrowsNRE(RefFunction function, [CallerLineNumber] int line = 0, [CallerFilePath] string file = "") + { + try + { + _ = function(); + } + catch (NullReferenceException) + { + return; + } + catch (Exception exc) + { + Console.WriteLine($"{file}:L{line} {exc}"); + } + Console.WriteLine($"Line {line}: test failed (expected: NullReferenceException)"); + _errors++; + } + public struct GenericStruct { public T field; @@ -185,5 +257,7 @@ struct StructWithByte { public byte Byte; } + + delegate ref T RefFunction(); } } From d36e4287b935f0fca836d07806e978ff02232e9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Petryka?= <35800402+MichalPetryka@users.noreply.github.com> Date: Wed, 27 Jul 2022 01:23:10 +0200 Subject: [PATCH 19/31] Future-proof against delegate inlining --- .../JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs index 7a3f1bf5a19a6f..280e079b7224a6 100644 --- a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs +++ b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs @@ -229,7 +229,9 @@ static void ThrowsNRE(RefFunction function, [CallerLineNumber] int line = { try { - _ = function(); + [MethodImpl(MethodImplOptions.NoInlining)] + static void Use(ref T t) { } + Use(ref function()); } catch (NullReferenceException) { From 2249ab809b1a16b92ad54596ad1d9d0439d62be8 Mon Sep 17 00:00:00 2001 From: petris Date: Fri, 16 Dec 2022 17:57:27 +0100 Subject: [PATCH 20/31] Redo changes, add test --- src/coreclr/jit/importercalls.cpp | 33 +++++++++++++++++++ .../MemoryMarshalGetArrayDataReference.cs | 31 +++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 211420188476ca..d46457470d6b9d 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -2601,6 +2601,29 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, break; } + case NI_System_Runtime_InteropService_MemoryMarshal_GetArrayDataReference: + { + assert(sig->numArgs == 1); + + GenTree* array = impPopStack().val; + CORINFO_CLASS_HANDLE elemHnd = sig->sigInst.methInst[0]; + CorInfoType jitType = info.compCompHnd->asCorInfoType(elemHnd); + var_types elemType = JITtype2varType(jitType); + + GenTree* arrayClone; + array = impCloneExpr(array, &arrayClone, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, + nullptr DEBUGARG("MemoryMarshal.GetArrayDataReference array")); + + impAppendTree(gtNewNullCheck(array, compCurBB), (unsigned)CHECK_SPILL_ALL, impCurStmtDI); + + GenTree* index = gtNewIconNode(0, TYP_I_IMPL); + GenTreeIndexAddr* indexAddr = gtNewArrayIndexAddr(arrayClone, index, elemType, elemHnd); + indexAddr->gtFlags &= ~GTF_INX_RNGCHK; + indexAddr->gtFlags |= GTF_INX_ADDR_NONNULL; + retNode = indexAddr; + break; + } + case NI_Internal_Runtime_MethodTable_Of: case NI_System_Activator_AllocatorOf: case NI_System_Activator_DefaultConstructorOf: @@ -7514,6 +7537,16 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) } } } + else if (strcmp(namespaceName, "System.Runtime.InteropServices") == 0) + { + if (strcmp(className, "MemoryMarshal") == 0) + { + if (strcmp(methodName, "GetArrayDataReference") == 0) + { + result = NI_System_Runtime_InteropService_MemoryMarshal_GetArrayDataReference; + } + } + } else if (strncmp(namespaceName, "System.Runtime.Intrinsics", 25) == 0) { // We go down this path even when FEATURE_HW_INTRINSICS isn't enabled diff --git a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs index 280e079b7224a6..34db736c283b5b 100644 --- a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs +++ b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs @@ -172,9 +172,40 @@ static int Problem4(byte[] a) Equals(Problem4(new byte[] { 1, 1 }), 0); + try + { + int[] inputArray = CreateArray(17); + + Create(out Vector v, inputArray, inputArray.Length); + } + catch + { + _errors++; + } + return 100 + _errors; } + public static void Create(out Vector result, int[] values, int index) + { + // We explicitly don't check for `null` because historically this has thrown `NullReferenceException` for perf reasons + + if ((index < 0) || ((values.Length - index) < Vector.Count)) + { + ThrowArgumentOutOfRangeException(); + } + + result = Unsafe.ReadUnaligned>(ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(values), index))); + } + + public static void ThrowArgumentOutOfRangeException() + { + throw new ArgumentOutOfRangeException(); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static int[] CreateArray(int size) => new int[size]; + [MethodImpl(MethodImplOptions.NoInlining)] static void Equals(T left, T right, [CallerLineNumber] int line = 0, [CallerFilePath] string file = "") { From be3ad65d7eb435f37a7a6c08373627ec79ad76b2 Mon Sep 17 00:00:00 2001 From: petris Date: Fri, 16 Dec 2022 17:59:53 +0100 Subject: [PATCH 21/31] Revert merge issue --- src/coreclr/jit/importer.cpp | 27271 +++++++++++---------------------- 1 file changed, 9066 insertions(+), 18205 deletions(-) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 46c36c964b860c..eda9ce179b6397 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -28,36 +28,6 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX } \ } while (0) -#define VerifyOrReturn(cond, msg) \ - do \ - { \ - if (!(cond)) \ - { \ - verRaiseVerifyExceptionIfNeeded(INDEBUG(msg) DEBUGARG(__FILE__) DEBUGARG(__LINE__)); \ - return; \ - } \ - } while (0) - -#define VerifyOrReturnSpeculative(cond, msg, speculative) \ - do \ - { \ - if (speculative) \ - { \ - if (!(cond)) \ - { \ - return false; \ - } \ - } \ - else \ - { \ - if (!(cond)) \ - { \ - verRaiseVerifyExceptionIfNeeded(INDEBUG(msg) DEBUGARG(__FILE__) DEBUGARG(__LINE__)); \ - return false; \ - } \ - } \ - } while (0) - /*****************************************************************************/ void Compiler::impInit() @@ -106,7 +76,7 @@ void Compiler::impPushOnStack(GenTree* tree, typeInfo ti) } } -inline void Compiler::impPushNullObjRefOnStack() +void Compiler::impPushNullObjRefOnStack() { impPushOnStack(gtNewIconNode(0, TYP_REF), typeInfo(TI_NULL)); } @@ -114,11 +84,11 @@ inline void Compiler::impPushNullObjRefOnStack() // This method gets called when we run into unverifiable code // (and we are verifying the method) -inline void Compiler::verRaiseVerifyExceptionIfNeeded(INDEBUG(const char* msg) DEBUGARG(const char* file) - DEBUGARG(unsigned line)) +void Compiler::verRaiseVerifyExceptionIfNeeded(INDEBUG(const char* msg) DEBUGARG(const char* file) + DEBUGARG(unsigned line)) { #ifdef DEBUG - const char* tail = strrchr(file, '\\'); + const char* tail = strrchr(file, DIRECTORY_SEPARATOR_CHAR_A); if (tail) { file = tail + 1; @@ -141,8 +111,8 @@ inline void Compiler::verRaiseVerifyExceptionIfNeeded(INDEBUG(const char* msg) D } } -inline void DECLSPEC_NORETURN Compiler::verRaiseVerifyException(INDEBUG(const char* msg) DEBUGARG(const char* file) - DEBUGARG(unsigned line)) +void DECLSPEC_NORETURN Compiler::verRaiseVerifyException(INDEBUG(const char* msg) DEBUGARG(const char* file) + DEBUGARG(unsigned line)) { JITLOG((LL_ERROR, "Verification failure: %s:%d : %s, while compiling %s opcode %s, IL offset %x\n", file, line, msg, info.compFullName, impCurOpcName, impCurOpcOffs)); @@ -366,7 +336,7 @@ void Compiler::impRestoreStackState(SavedStack* savePtr) //------------------------------------------------------------------------ // impBeginTreeList: Get the tree list started for a new basic block. // -inline void Compiler::impBeginTreeList() +void Compiler::impBeginTreeList() { assert(impStmtList == nullptr && impLastStmt == nullptr); } @@ -378,7 +348,7 @@ inline void Compiler::impBeginTreeList() * directly only for handling CEE_LEAVEs out of finally-protected try's. */ -inline void Compiler::impEndTreeList(BasicBlock* block, Statement* firstStmt, Statement* lastStmt) +void Compiler::impEndTreeList(BasicBlock* block, Statement* firstStmt, Statement* lastStmt) { /* Make the list circular, so that we can easily walk it backwards */ @@ -394,7 +364,7 @@ inline void Compiler::impEndTreeList(BasicBlock* block, Statement* firstStmt, St block->bbFlags |= BBF_IMPORTED; } -inline void Compiler::impEndTreeList(BasicBlock* block) +void Compiler::impEndTreeList(BasicBlock* block) { if (impStmtList == nullptr) { @@ -425,18 +395,18 @@ inline void Compiler::impEndTreeList(BasicBlock* block) * that this has only limited value as we can only check [0..chkLevel). */ -inline void Compiler::impAppendStmtCheck(Statement* stmt, unsigned chkLevel) +void Compiler::impAppendStmtCheck(Statement* stmt, unsigned chkLevel) { #ifndef DEBUG return; #else - if (chkLevel == (unsigned)CHECK_SPILL_ALL) + if (chkLevel == CHECK_SPILL_ALL) { chkLevel = verCurrentState.esStackDepth; } - if (verCurrentState.esStackDepth == 0 || chkLevel == 0 || chkLevel == (unsigned)CHECK_SPILL_NONE) + if (verCurrentState.esStackDepth == 0 || chkLevel == 0 || chkLevel == CHECK_SPILL_NONE) { return; } @@ -453,25 +423,23 @@ inline void Compiler::impAppendStmtCheck(Statement* stmt, unsigned chkLevel) } } - if (tree->gtOper == GT_ASG) + if (tree->OperIs(GT_ASG)) { // For an assignment to a local variable, all references of that // variable have to be spilled. If it is aliased, all calls and // indirect accesses have to be spilled - if (tree->AsOp()->gtOp1->gtOper == GT_LCL_VAR) + if (tree->AsOp()->gtOp1->OperIsLocal()) { unsigned lclNum = tree->AsOp()->gtOp1->AsLclVarCommon()->GetLclNum(); for (unsigned level = 0; level < chkLevel; level++) { - assert(!gtHasRef(verCurrentState.esStack[level].val, lclNum)); - assert(!lvaTable[lclNum].IsAddressExposed() || - (verCurrentState.esStack[level].val->gtFlags & GTF_SIDE_EFFECT) == 0); + GenTree* stkTree = verCurrentState.esStack[level].val; + assert(!gtHasRef(stkTree, lclNum) || impIsInvariant(stkTree)); + assert(!lvaTable[lclNum].IsAddressExposed() || ((stkTree->gtFlags & GTF_SIDE_EFFECT) == 0)); } } - // If the access may be to global memory, all side effects have to be spilled. - else if (tree->AsOp()->gtOp1->gtFlags & GTF_GLOB_REF) { for (unsigned level = 0; level < chkLevel; level++) @@ -490,7 +458,7 @@ inline void Compiler::impAppendStmtCheck(Statement* stmt, unsigned chkLevel) // Arguments: // stmt - The statement to add. // chkLevel - [0..chkLevel) is the portion of the stack which we will check -// for interference with stmt and spill if needed. +// for interference with stmt and spilled if needed. // checkConsumedDebugInfo - Whether to check for consumption of impCurStmtDI. impCurStmtDI // marks the debug info of the current boundary and is set when we // start importing IL at that boundary. If this parameter is true, @@ -500,70 +468,88 @@ inline void Compiler::impAppendStmtCheck(Statement* stmt, unsigned chkLevel) // void Compiler::impAppendStmt(Statement* stmt, unsigned chkLevel, bool checkConsumedDebugInfo) { - if (chkLevel == (unsigned)CHECK_SPILL_ALL) + if (chkLevel == CHECK_SPILL_ALL) { chkLevel = verCurrentState.esStackDepth; } - if ((chkLevel != 0) && (chkLevel != (unsigned)CHECK_SPILL_NONE)) + if ((chkLevel != 0) && (chkLevel != CHECK_SPILL_NONE)) { assert(chkLevel <= verCurrentState.esStackDepth); - /* If the statement being appended has any side-effects, check the stack - to see if anything needs to be spilled to preserve correct ordering. */ - + // If the statement being appended has any side-effects, check the stack to see if anything + // needs to be spilled to preserve correct ordering. + // GenTree* expr = stmt->GetRootNode(); GenTreeFlags flags = expr->gtFlags & GTF_GLOB_EFFECT; - // Assignment to (unaliased) locals don't count as a side-effect as - // we handle them specially using impSpillLclRefs(). Temp locals should - // be fine too. - - if ((expr->gtOper == GT_ASG) && (expr->AsOp()->gtOp1->gtOper == GT_LCL_VAR) && - ((expr->AsOp()->gtOp1->gtFlags & GTF_GLOB_REF) == 0) && !gtHasLocalsWithAddrOp(expr->AsOp()->gtOp2)) + // Assignments to unaliased locals require special handling. Here, we look for trees that + // can modify them and spill the references. In doing so, we make two assumptions: + // + // 1. All locals which can be modified indirectly are marked as address-exposed or with + // "lvHasLdAddrOp" -- we will rely on "impSpillSideEffects(spillGlobEffects: true)" + // below to spill them. + // 2. Trees that assign to unaliased locals are always top-level (this avoids having to + // walk down the tree here), and are a subset of what is recognized here. + // + // If any of the above are violated (say for some temps), the relevant code must spill + // things manually. + // + LclVarDsc* dstVarDsc = nullptr; + if (expr->OperIs(GT_ASG) && expr->AsOp()->gtOp1->OperIsLocal()) { - GenTreeFlags op2Flags = expr->AsOp()->gtOp2->gtFlags & GTF_GLOB_EFFECT; - assert(flags == (op2Flags | GTF_ASG)); - flags = op2Flags; + dstVarDsc = lvaGetDesc(expr->AsOp()->gtOp1->AsLclVarCommon()); } - - if (flags != 0) + else if (expr->OperIs(GT_CALL, GT_RET_EXPR)) // The special case of calls with return buffers. { - bool spillGlobEffects = false; + GenTree* call = expr->OperIs(GT_RET_EXPR) ? expr->AsRetExpr()->gtInlineCandidate : expr; - if ((flags & GTF_CALL) != 0) - { - // If there is a call, we have to spill global refs - spillGlobEffects = true; - } - else if (!expr->OperIs(GT_ASG)) + if (call->TypeIs(TYP_VOID) && call->AsCall()->TreatAsShouldHaveRetBufArg(this)) { - if ((flags & GTF_ASG) != 0) + GenTree* retBuf; + if (call->AsCall()->ShouldHaveRetBufArg()) { - // The expression is not an assignment node but it has an assignment side effect, it - // must be an atomic op, HW intrinsic or some other kind of node that stores to memory. - // Since we don't know what it assigns to, we need to spill global refs. - spillGlobEffects = true; + assert(call->AsCall()->gtArgs.HasRetBuffer()); + retBuf = call->AsCall()->gtArgs.GetRetBufferArg()->GetNode(); } - } - else - { - GenTree* lhs = expr->gtGetOp1(); - GenTree* rhs = expr->gtGetOp2(); + else + { + assert(!call->AsCall()->gtArgs.HasThisPointer()); + retBuf = call->AsCall()->gtArgs.GetArgByIndex(0)->GetNode(); + } + + assert(retBuf->TypeIs(TYP_I_IMPL, TYP_BYREF)); - if (((rhs->gtFlags | lhs->gtFlags) & GTF_ASG) != 0) + GenTreeLclVarCommon* lclNode = retBuf->IsLocalAddrExpr(); + if (lclNode != nullptr) { - // Either side of the assignment node has an assignment side effect. - // Since we don't know what it assigns to, we need to spill global refs. - spillGlobEffects = true; + dstVarDsc = lvaGetDesc(lclNode); } - else if ((lhs->gtFlags & GTF_GLOB_REF) != 0) + } + } + + if ((dstVarDsc != nullptr) && !dstVarDsc->IsAddressExposed() && !dstVarDsc->lvHasLdAddrOp) + { + impSpillLclRefs(lvaGetLclNum(dstVarDsc), chkLevel); + + if (expr->OperIs(GT_ASG)) + { + // For assignments, limit the checking to what the RHS could modify/interfere with. + GenTree* rhs = expr->AsOp()->gtOp2; + flags = rhs->gtFlags & GTF_GLOB_EFFECT; + + // We don't mark indirections off of "aliased" locals with GLOB_REF, but they must still be + // considered as such in the interference checking. + if (((flags & GTF_GLOB_REF) == 0) && !impIsAddressInLocal(rhs) && gtHasLocalsWithAddrOp(rhs)) { - spillGlobEffects = true; + flags |= GTF_GLOB_REF; } } + } - impSpillSideEffects(spillGlobEffects, chkLevel DEBUGARG("impAppendStmt")); + if (flags != 0) + { + impSpillSideEffects((flags & (GTF_ASG | GTF_CALL)) != 0, chkLevel DEBUGARG("impAppendStmt")); } else { @@ -610,7 +596,7 @@ void Compiler::impAppendStmt(Statement* stmt, unsigned chkLevel, bool checkConsu // Arguments: // stmt - the statement to add. // -inline void Compiler::impAppendStmt(Statement* stmt) +void Compiler::impAppendStmt(Statement* stmt) { if (impStmtList == nullptr) { @@ -655,7 +641,7 @@ Statement* Compiler::impExtractLastStmt() // stmt - a statement to insert; // stmtBefore - an insertion point to insert "stmt" before. // -inline void Compiler::impInsertStmtBefore(Statement* stmt, Statement* stmtBefore) +void Compiler::impInsertStmtBefore(Statement* stmt, Statement* stmtBefore) { assert(stmt != nullptr); assert(stmtBefore != nullptr); @@ -793,7 +779,7 @@ void Compiler::impAssignTempGen(unsigned tmpNum, // calls that may not actually be required - e.g. if we only access a field of a struct. GenTree* dst = gtNewLclvNode(tmpNum, varType); - asg = impAssignStruct(dst, val, structType, curLevel, pAfterStmt, di, block); + asg = impAssignStruct(dst, val, curLevel, pAfterStmt, di, block); } else { @@ -815,138 +801,6 @@ void Compiler::impAssignTempGen(unsigned tmpNum, } } -//------------------------------------------------------------------------ -// impPopCallArgs: -// Pop the given number of values from the stack and return a list node with -// their values. -// -// Parameters: -// sig - Signature used to figure out classes the runtime must load, and -// also to record exact receiving argument types that may be needed for ABI -// purposes later. -// call - The call to pop arguments into. -// -void Compiler::impPopCallArgs(CORINFO_SIG_INFO* sig, GenTreeCall* call) -{ - assert(call->gtArgs.IsEmpty()); - - if (impStackHeight() < sig->numArgs) - { - BADCODE("not enough arguments for call"); - } - - struct SigParamInfo - { - CorInfoType CorType; - CORINFO_CLASS_HANDLE ClassHandle; - }; - - SigParamInfo inlineParams[16]; - SigParamInfo* params = sig->numArgs <= 16 ? inlineParams : new (this, CMK_CallArgs) SigParamInfo[sig->numArgs]; - - // We will iterate and pop the args in reverse order as we sometimes need - // to spill some args. However, we need signature information and the - // JIT-EE interface only allows us to iterate the signature forwards. We - // will collect the needed information here and at the same time notify the - // EE that the signature types need to be loaded. - CORINFO_ARG_LIST_HANDLE sigArg = sig->args; - for (unsigned i = 0; i < sig->numArgs; i++) - { - params[i].CorType = strip(info.compCompHnd->getArgType(sig, sigArg, ¶ms[i].ClassHandle)); - - if (params[i].CorType != CORINFO_TYPE_CLASS && params[i].CorType != CORINFO_TYPE_BYREF && - params[i].CorType != CORINFO_TYPE_PTR && params[i].CorType != CORINFO_TYPE_VAR) - { - CORINFO_CLASS_HANDLE argRealClass = info.compCompHnd->getArgClass(sig, sigArg); - if (argRealClass != nullptr) - { - // Make sure that all valuetypes (including enums) that we push are loaded. - // This is to guarantee that if a GC is triggered from the prestub of this methods, - // all valuetypes in the method signature are already loaded. - // We need to be able to find the size of the valuetypes, but we cannot - // do a class-load from within GC. - info.compCompHnd->classMustBeLoadedBeforeCodeIsRun(argRealClass); - } - } - - sigArg = info.compCompHnd->getArgNext(sigArg); - } - - if ((sig->retTypeSigClass != nullptr) && (sig->retType != CORINFO_TYPE_CLASS) && - (sig->retType != CORINFO_TYPE_BYREF) && (sig->retType != CORINFO_TYPE_PTR) && - (sig->retType != CORINFO_TYPE_VAR)) - { - // Make sure that all valuetypes (including enums) that we push are loaded. - // This is to guarantee that if a GC is triggerred from the prestub of this methods, - // all valuetypes in the method signature are already loaded. - // We need to be able to find the size of the valuetypes, but we cannot - // do a class-load from within GC. - info.compCompHnd->classMustBeLoadedBeforeCodeIsRun(sig->retTypeSigClass); - } - - // Now create the arguments in reverse. - for (unsigned i = sig->numArgs; i > 0; i--) - { - StackEntry se = impPopStack(); - typeInfo ti = se.seTypeInfo; - GenTree* argNode = se.val; - - var_types jitSigType = JITtype2varType(params[i - 1].CorType); - CORINFO_CLASS_HANDLE classHnd = params[i - 1].ClassHandle; - - if (!impCheckImplicitArgumentCoercion(jitSigType, argNode->TypeGet())) - { - BADCODE("the call argument has a type that can't be implicitly converted to the signature type"); - } - - if (varTypeIsStruct(argNode)) - { - // Morph trees that aren't already OBJs or MKREFANY to be OBJs - assert(ti.IsType(TI_STRUCT)); - - JITDUMP("Calling impNormStructVal on:\n"); - DISPTREE(argNode); - - argNode = impNormStructVal(argNode, classHnd, CHECK_SPILL_ALL); - // For SIMD types the normalization can normalize TYP_STRUCT to - // e.g. TYP_SIMD16 which we keep (along with the class handle) in - // the CallArgs. - jitSigType = argNode->TypeGet(); - - JITDUMP("resulting tree:\n"); - DISPTREE(argNode); - } - else - { - // insert implied casts (from float to double or double to float) - if ((jitSigType == TYP_DOUBLE) && argNode->TypeIs(TYP_FLOAT)) - { - argNode = gtNewCastNode(TYP_DOUBLE, argNode, false, TYP_DOUBLE); - } - else if ((jitSigType == TYP_FLOAT) && argNode->TypeIs(TYP_DOUBLE)) - { - argNode = gtNewCastNode(TYP_FLOAT, argNode, false, TYP_FLOAT); - } - - // insert any widening or narrowing casts for backwards compatibility - argNode = impImplicitIorI4Cast(argNode, jitSigType); - } - - NewCallArg arg; - if (varTypeIsStruct(jitSigType)) - { - arg = NewCallArg::Struct(argNode, jitSigType, classHnd); - } - else - { - arg = NewCallArg::Primitive(argNode, jitSigType); - } - - call->gtArgs.PushFront(this, arg); - call->gtFlags |= argNode->gtFlags & GTF_GLOB_EFFECT; - } -} - static bool TypeIs(var_types type1, var_types type2) { return type1 == type2; @@ -1052,31 +906,15 @@ bool Compiler::impCheckImplicitArgumentCoercion(var_types sigType, var_types nod return false; } -/***************************************************************************** - * - * Pop the given number of values from the stack in reverse order (STDCALL/CDECL etc.) - * The first "skipReverseCount" items are not reversed. - */ - -void Compiler::impPopReverseCallArgs(CORINFO_SIG_INFO* sig, GenTreeCall* call, unsigned skipReverseCount) -{ - assert(skipReverseCount <= sig->numArgs); - - impPopCallArgs(sig, call); - - call->gtArgs.Reverse(skipReverseCount, sig->numArgs - skipReverseCount); -} - //------------------------------------------------------------------------ // impAssignStruct: Create a struct assignment // // Arguments: // dest - the destination of the assignment // src - the value to be assigned -// structHnd - handle representing the struct type // curLevel - stack level for which a spill may be being done // pAfterStmt - statement to insert any additional statements after -// ilOffset - il offset for new statements +// di - debug info for new statements // block - block to insert any additional statements in // // Return Value: @@ -1084,141 +922,35 @@ void Compiler::impPopReverseCallArgs(CORINFO_SIG_INFO* sig, GenTreeCall* call, u // // Notes: // Temp assignments may be appended to impStmtList if spilling is necessary. - -GenTree* Compiler::impAssignStruct(GenTree* dest, - GenTree* src, - CORINFO_CLASS_HANDLE structHnd, - unsigned curLevel, - Statement** pAfterStmt, /* = nullptr */ - const DebugInfo& di, /* = DebugInfo() */ - BasicBlock* block /* = nullptr */ +// +GenTree* Compiler::impAssignStruct(GenTree* dest, + GenTree* src, + unsigned curLevel, + Statement** pAfterStmt, /* = nullptr */ + const DebugInfo& di, /* = DebugInfo() */ + BasicBlock* block /* = nullptr */ ) { - assert(varTypeIsStruct(dest)); - - DebugInfo usedDI = di; - if (!usedDI.IsValid()) - { - usedDI = impCurStmtDI; - } - - while (dest->gtOper == GT_COMMA) - { - // Second thing is the struct. - assert(varTypeIsStruct(dest->AsOp()->gtOp2)); - - // Append all the op1 of GT_COMMA trees before we evaluate op2 of the GT_COMMA tree. - if (pAfterStmt) - { - Statement* newStmt = gtNewStmt(dest->AsOp()->gtOp1, usedDI); - fgInsertStmtAfter(block, *pAfterStmt, newStmt); - *pAfterStmt = newStmt; - } - else - { - impAppendTree(dest->AsOp()->gtOp1, curLevel, usedDI); // do the side effect - } - - // set dest to the second thing - dest = dest->AsOp()->gtOp2; - } - - assert(dest->gtOper == GT_LCL_VAR || dest->gtOper == GT_RETURN || dest->gtOper == GT_FIELD || - dest->gtOper == GT_IND || dest->gtOper == GT_OBJ); - - // Return a NOP if this is a self-assignment. - if (dest->OperGet() == GT_LCL_VAR && src->OperGet() == GT_LCL_VAR && - src->AsLclVarCommon()->GetLclNum() == dest->AsLclVarCommon()->GetLclNum()) - { - return gtNewNothingNode(); - } + assert(varTypeIsStruct(dest) && (dest->OperIsLocal() || dest->OperIsIndir() || dest->OperIs(GT_FIELD))); - // TODO-1stClassStructs: Avoid creating an address if it is not needed, - // or re-creating a Blk node if it is. - GenTree* destAddr; - - if (dest->gtOper == GT_IND || dest->OperIsBlk()) - { - destAddr = dest->AsOp()->gtOp1; - } - else + assert(dest->TypeGet() == src->TypeGet()); + if (dest->TypeIs(TYP_STRUCT)) { - destAddr = gtNewOperNode(GT_ADDR, TYP_BYREF, dest); + assert(ClassLayout::AreCompatible(dest->GetLayout(this), src->GetLayout(this))); } - return (impAssignStructPtr(destAddr, src, structHnd, curLevel, pAfterStmt, usedDI, block)); -} - -//------------------------------------------------------------------------ -// impAssignStructPtr: Assign (copy) the structure from 'src' to 'destAddr'. -// -// Arguments: -// destAddr - address of the destination of the assignment -// src - source of the assignment -// structHnd - handle representing the struct type -// curLevel - stack level for which a spill may be being done -// pAfterStmt - statement to insert any additional statements after -// di - debug info for new statements -// block - block to insert any additional statements in -// -// Return Value: -// The tree that should be appended to the statement list that represents the assignment. -// -// Notes: -// Temp assignments may be appended to impStmtList if spilling is necessary. - -GenTree* Compiler::impAssignStructPtr(GenTree* destAddr, - GenTree* src, - CORINFO_CLASS_HANDLE structHnd, - unsigned curLevel, - Statement** pAfterStmt, /* = NULL */ - const DebugInfo& di, /* = DebugInfo() */ - BasicBlock* block /* = NULL */ - ) -{ - GenTree* dest = nullptr; - GenTreeFlags destFlags = GTF_EMPTY; - DebugInfo usedDI = di; if (!usedDI.IsValid()) { usedDI = impCurStmtDI; } -#ifdef DEBUG -#ifdef FEATURE_HW_INTRINSICS - if (src->OperIs(GT_HWINTRINSIC)) - { - const GenTreeHWIntrinsic* intrinsic = src->AsHWIntrinsic(); - - if (HWIntrinsicInfo::IsMultiReg(intrinsic->GetHWIntrinsicId())) - { - assert(src->TypeGet() == TYP_STRUCT); - } - else - { - assert(varTypeIsSIMD(src)); - } - } - else -#endif // FEATURE_HW_INTRINSICS - { - assert(src->OperIs(GT_LCL_VAR, GT_LCL_FLD, GT_FIELD, GT_IND, GT_OBJ, GT_CALL, GT_MKREFANY, GT_RET_EXPR, - GT_COMMA, GT_CNS_VEC) || - ((src->TypeGet() != TYP_STRUCT) && src->OperIsSIMD())); - } -#endif // DEBUG - - var_types asgType = src->TypeGet(); - - if (src->gtOper == GT_CALL) + if (src->IsCall()) { GenTreeCall* srcCall = src->AsCall(); if (srcCall->TreatAsShouldHaveRetBufArg(this)) { - // Case of call returning a struct via hidden retbuf arg - CLANG_FORMAT_COMMENT_ANCHOR; - + // Case of call returning a struct via hidden retbuf arg. // Some calls have an "out buffer" that is not actually a ret buff // in the ABI sense. We take the path here for those but it should // not be marked as the ret buff arg since it always follow the @@ -1226,7 +958,8 @@ GenTree* Compiler::impAssignStructPtr(GenTree* destAddr, WellKnownArg wellKnownArgType = srcCall->ShouldHaveRetBufArg() ? WellKnownArg::RetBuffer : WellKnownArg::None; - NewCallArg newArg = NewCallArg::Primitive(destAddr).WellKnown(wellKnownArgType); + GenTree* destAddr = gtNewOperNode(GT_ADDR, TYP_BYREF, dest); + NewCallArg newArg = NewCallArg::Primitive(destAddr).WellKnown(wellKnownArgType); #if !defined(TARGET_ARM) // Unmanaged instance methods on Windows or Unix X86 need the retbuf arg after the first (this) parameter @@ -1280,7 +1013,7 @@ GenTree* Compiler::impAssignStructPtr(GenTree* destAddr, } else { - srcCall->gtArgs.InsertAfter(this, &*srcCall->gtArgs.Args().begin(), newArg); + srcCall->gtArgs.InsertAfter(this, srcCall->gtArgs.Args().begin().GetArg(), newArg); } #endif } @@ -1310,66 +1043,24 @@ GenTree* Compiler::impAssignStructPtr(GenTree* destAddr, // return the morphed call node return src; } - else - { - // Case of call returning a struct in one or more registers. - - var_types returnType = (var_types)srcCall->gtReturnType; - - // First we try to change this to "LclVar/LclFld = call" - // - if ((destAddr->gtOper == GT_ADDR) && (destAddr->AsOp()->gtOp1->gtOper == GT_LCL_VAR)) - { - // If it is a multi-reg struct return, don't change the oper to GT_LCL_FLD. - // That is, the IR will be of the form lclVar = call for multi-reg return - // - GenTreeLclVar* lcl = destAddr->AsOp()->gtOp1->AsLclVar(); - unsigned lclNum = lcl->GetLclNum(); - LclVarDsc* varDsc = lvaGetDesc(lclNum); - if (src->AsCall()->HasMultiRegRetVal()) - { - // Mark the struct LclVar as used in a MultiReg return context - // which currently makes it non promotable. - // TODO-1stClassStructs: Eliminate this pessimization when we can more generally - // handle multireg returns. - lcl->gtFlags |= GTF_DONT_CSE; - varDsc->lvIsMultiRegRet = true; - } - - dest = lcl; -#if defined(TARGET_ARM) - // TODO-Cleanup: This should have been taken care of in the above HasMultiRegRetVal() case, - // but that method has not been updadted to include ARM. - impMarkLclDstNotPromotable(lclNum, src, structHnd); - lcl->gtFlags |= GTF_DONT_CSE; -#elif defined(UNIX_AMD64_ABI) - // Not allowed for FEATURE_CORCLR which is the only SKU available for System V OSs. - assert(!src->AsCall()->IsVarargs() && "varargs not allowed for System V OSs."); - - // Make the struct non promotable. The eightbytes could contain multiple fields. - // TODO-1stClassStructs: Eliminate this pessimization when we can more generally - // handle multireg returns. - // TODO-Cleanup: Why is this needed here? This seems that it will set this even for - // non-multireg returns. - lcl->gtFlags |= GTF_DONT_CSE; - varDsc->lvIsMultiRegRet = true; -#endif - } - else // we don't have a GT_ADDR of a GT_LCL_VAR - { - asgType = returnType; - } +#ifdef UNIX_AMD64_ABI + if (dest->OperIs(GT_LCL_VAR)) + { + // TODO-Cleanup: delete this quirk. + lvaGetDesc(dest->AsLclVar())->lvIsMultiRegRet = true; } +#endif // UNIX_AMD64_ABI } - else if (src->gtOper == GT_RET_EXPR) + else if (src->OperIs(GT_RET_EXPR)) { - noway_assert(src->AsRetExpr()->gtInlineCandidate->OperIs(GT_CALL)); - GenTreeCall* call = src->AsRetExpr()->gtInlineCandidate->AsCall(); + assert(src->AsRetExpr()->gtInlineCandidate->OperIs(GT_CALL)); + GenTreeCall* call = src->AsRetExpr()->gtInlineCandidate; if (call->ShouldHaveRetBufArg()) { // insert the return value buffer into the argument list as first byref parameter after 'this' + GenTree* destAddr = gtNewOperNode(GT_ADDR, TYP_BYREF, dest); call->gtArgs.InsertAfterThisOrFirst(this, NewCallArg::Primitive(destAddr).WellKnown(WellKnownArg::RetBuffer)); @@ -1381,26 +1072,15 @@ GenTree* Compiler::impAssignStructPtr(GenTree* destAddr, // So now we just return an empty node (pruning the GT_RET_EXPR) return src; } - else - { - // Case of inline method returning a struct in one or more registers. - // We won't need a return buffer - asgType = src->gtType; - } - } - else if (src->OperIsBlk()) - { - asgType = impNormStructType(structHnd); - assert(ClassLayout::AreCompatible(src->AsBlk()->GetLayout(), typGetObjLayout(structHnd))); } - else if (src->gtOper == GT_MKREFANY) + else if (src->OperIs(GT_MKREFANY)) { - // Since we are assigning the result of a GT_MKREFANY, - // "destAddr" must point to a refany. - + // Since we are assigning the result of a GT_MKREFANY, "destAddr" must point to a refany. + // TODO-CQ: we can do this without address-exposing the local on the LHS. + GenTree* destAddr = gtNewOperNode(GT_ADDR, TYP_BYREF, dest); GenTree* destAddrClone; - destAddr = - impCloneExpr(destAddr, &destAddrClone, structHnd, curLevel, pAfterStmt DEBUGARG("MKREFANY assignment")); + destAddr = impCloneExpr(destAddr, &destAddrClone, NO_CLASS_HANDLE, curLevel, + pAfterStmt DEBUGARG("MKREFANY assignment")); assert(OFFSETOF__CORINFO_TypedReference__dataPtr == 0); assert(destAddr->gtType == TYP_I_IMPL || destAddr->gtType == TYP_BYREF); @@ -1427,10 +1107,8 @@ GenTree* Compiler::impAssignStructPtr(GenTree* destAddr, // return the assign of the type value, to be appended return gtNewAssignNode(typeSlot, src->AsOp()->gtOp2); } - else if (src->gtOper == GT_COMMA) + else if (src->OperIs(GT_COMMA)) { - // The second thing is the struct or its address. - assert(varTypeIsStruct(src->AsOp()->gtOp2) || src->AsOp()->gtOp2->gtType == TYP_BYREF); if (pAfterStmt) { // Insert op1 after '*pAfterStmt' @@ -1448,138 +1126,86 @@ GenTree* Compiler::impAssignStructPtr(GenTree* destAddr, // In this case we have neither been given a statement to insert after, nor are we // in the importer where we can append the side effect. // Instead, we're going to sink the assignment below the COMMA. - src->AsOp()->gtOp2 = - impAssignStructPtr(destAddr, src->AsOp()->gtOp2, structHnd, curLevel, pAfterStmt, usedDI, block); + src->AsOp()->gtOp2 = impAssignStruct(dest, src->AsOp()->gtOp2, curLevel, pAfterStmt, usedDI, block); src->AddAllEffectsFlags(src->AsOp()->gtOp2); return src; } // Evaluate the second thing using recursion. - return impAssignStructPtr(destAddr, src->AsOp()->gtOp2, structHnd, curLevel, pAfterStmt, usedDI, block); - } - else if (src->IsLocal()) - { - asgType = src->TypeGet(); - } - else if (asgType == TYP_STRUCT) - { - // It should already have the appropriate type. - assert(asgType == impNormStructType(structHnd)); - } - if ((dest == nullptr) && (destAddr->OperGet() == GT_ADDR)) - { - GenTree* destNode = destAddr->gtGetOp1(); - // If the actual destination is a local, or a block node, - // don't insert an OBJ(ADDR) if it already has the right type. - if (destNode->OperIs(GT_LCL_VAR) || destNode->OperIsBlk()) - { - var_types destType = destNode->TypeGet(); - // If one or both types are TYP_STRUCT (one may not yet be normalized), they are compatible - // iff their handles are the same. - // Otherwise, they are compatible if their types are the same. - bool typesAreCompatible = - ((destType == TYP_STRUCT) || (asgType == TYP_STRUCT)) - ? ((gtGetStructHandleIfPresent(destNode) == structHnd) && varTypeIsStruct(asgType)) - : (destType == asgType); - if (typesAreCompatible) - { - dest = destNode; - if (destType != TYP_STRUCT) - { - // Use a normalized type if available. We know from above that they're equivalent. - asgType = destType; - } - } - } - } - - if (dest == nullptr) - { - if (asgType == TYP_STRUCT) - { - dest = gtNewObjNode(structHnd, destAddr); - gtSetObjGcInfo(dest->AsObj()); - // Although an obj as a call argument was always assumed to be a globRef - // (which is itself overly conservative), that is not true of the operands - // of a block assignment. - dest->gtFlags &= ~GTF_GLOB_REF; - dest->gtFlags |= (destAddr->gtFlags & GTF_GLOB_REF); - } - else - { - dest = gtNewOperNode(GT_IND, asgType, destAddr); - } + return impAssignStruct(dest, src->AsOp()->gtOp2, curLevel, pAfterStmt, usedDI, block); } - if (dest->OperIs(GT_LCL_VAR) && - (src->IsMultiRegNode() || - (src->OperIs(GT_RET_EXPR) && src->AsRetExpr()->gtInlineCandidate->AsCall()->HasMultiRegRetVal()))) + if (dest->OperIs(GT_LCL_VAR) && src->IsMultiRegNode()) { - if (lvaEnregMultiRegVars && varTypeIsStruct(dest)) - { - dest->AsLclVar()->SetMultiReg(); - } - if (src->OperIs(GT_CALL)) - { - lvaGetDesc(dest->AsLclVar())->lvIsMultiRegRet = true; - } + lvaGetDesc(dest->AsLclVar())->lvIsMultiRegRet = true; } - dest->gtFlags |= destFlags; - destFlags = dest->gtFlags; + // Return a store node, to be appended. + GenTree* storeNode = gtNewBlkOpNode(dest, src); - // return an assignment node, to be appended - GenTree* asgNode = gtNewAssignNode(dest, src); - gtBlockOpInit(asgNode, dest, src, false); - - // TODO-1stClassStructs: Clean up the settings of GTF_DONT_CSE on the lhs - // of assignments. - if ((destFlags & GTF_DONT_CSE) == 0) - { - dest->gtFlags &= ~(GTF_DONT_CSE); - } - return asgNode; + return storeNode; } -/***************************************************************************** - Given a struct value, and the class handle for that structure, return - the expression for the address for that structure value. - - willDeref - does the caller guarantee to dereference the pointer. -*/ - -GenTree* Compiler::impGetStructAddr(GenTree* structVal, - CORINFO_CLASS_HANDLE structHnd, - unsigned curLevel, - bool willDeref) -{ - assert(varTypeIsStruct(structVal) || eeIsValueClass(structHnd)); - - var_types type = structVal->TypeGet(); - +//------------------------------------------------------------------------ +// impAssignStructPtr: Assign (copy) the structure from 'src' to 'destAddr'. +// +// Arguments: +// destAddr - address of the destination of the assignment +// src - source of the assignment +// structHnd - handle representing the struct type +// curLevel - stack level for which a spill may be being done +// pAfterStmt - statement to insert any additional statements after +// di - debug info for new statements +// block - block to insert any additional statements in +// +// Return Value: +// The tree that should be appended to the statement list that represents the assignment. +// +// Notes: +// Temp assignments may be appended to impStmtList if spilling is necessary. +// +GenTree* Compiler::impAssignStructPtr(GenTree* destAddr, + GenTree* src, + CORINFO_CLASS_HANDLE structHnd, + unsigned curLevel, + Statement** pAfterStmt, /* = NULL */ + const DebugInfo& di, /* = DebugInfo() */ + BasicBlock* block /* = NULL */ + ) +{ + GenTree* dst = gtNewStructVal(typGetObjLayout(structHnd), destAddr); + return impAssignStruct(dst, src, curLevel, pAfterStmt, di, block); +} + +/***************************************************************************** + Given a struct value, and the class handle for that structure, return + the expression for the address for that structure value. + + willDeref - does the caller guarantee to dereference the pointer. +*/ + +GenTree* Compiler::impGetStructAddr(GenTree* structVal, + CORINFO_CLASS_HANDLE structHnd, + unsigned curLevel, + bool willDeref) +{ + assert(varTypeIsStruct(structVal) || eeIsValueClass(structHnd)); + + var_types type = structVal->TypeGet(); genTreeOps oper = structVal->gtOper; - if (oper == GT_OBJ && willDeref) - { - assert(structVal->AsObj()->GetLayout()->GetClassHandle() == structHnd); - return (structVal->AsObj()->Addr()); - } - else if (oper == GT_CALL || oper == GT_RET_EXPR || oper == GT_OBJ || oper == GT_MKREFANY || - structVal->OperIsSimdOrHWintrinsic() || structVal->IsCnsVec()) + if (oper == GT_CALL || oper == GT_RET_EXPR || (oper == GT_OBJ && !willDeref) || oper == GT_MKREFANY || + structVal->OperIsSimdOrHWintrinsic() || structVal->IsCnsVec()) { unsigned tmpNum = lvaGrabTemp(true DEBUGARG("struct address for call/obj")); impAssignTempGen(tmpNum, structVal, structHnd, curLevel); - // The 'return value' is now the temp itself - - type = genActualType(lvaTable[tmpNum].TypeGet()); - GenTree* temp = gtNewLclvNode(tmpNum, type); - temp = gtNewOperNode(GT_ADDR, TYP_BYREF, temp); - return temp; + // The 'return value' is now address of the temp itself. + return gtNewLclVarAddrNode(tmpNum, TYP_BYREF); } - else if (oper == GT_COMMA) + if (oper == GT_COMMA) { assert(structVal->AsOp()->gtOp2->gtType == type); // Second thing is the struct @@ -1612,7 +1238,7 @@ GenTree* Compiler::impGetStructAddr(GenTree* structVal, return (structVal); } - return (gtNewOperNode(GT_ADDR, TYP_BYREF, structVal)); + return gtNewOperNode(GT_ADDR, TYP_BYREF, structVal); } //------------------------------------------------------------------------ @@ -1695,31 +1321,15 @@ GenTree* Compiler::impNormStructVal(GenTree* structVal, CORINFO_CLASS_HANDLE str { structType = impNormStructType(structHnd); } - bool alreadyNormalized = false; - GenTreeLclVarCommon* structLcl = nullptr; + GenTreeLclVarCommon* structLcl = nullptr; - genTreeOps oper = structVal->OperGet(); - switch (oper) + switch (structVal->OperGet()) { - // GT_MKREFANY is supported directly by args morphing. - case GT_MKREFANY: - alreadyNormalized = true; - break; - case GT_CALL: case GT_RET_EXPR: makeTemp = true; break; - case GT_FIELD: - // Wrap it in a GT_OBJ, if needed. - structVal->gtType = structType; - if (structType == TYP_STRUCT) - { - structVal = gtNewObjNode(structHnd, gtNewOperNode(GT_ADDR, TYP_BYREF, structVal)); - } - break; - case GT_LCL_VAR: case GT_LCL_FLD: structLcl = structVal->AsLclVarCommon(); @@ -1727,35 +1337,21 @@ GenTree* Compiler::impNormStructVal(GenTree* structVal, CORINFO_CLASS_HANDLE str structVal = gtNewObjNode(structHnd, gtNewOperNode(GT_ADDR, TYP_BYREF, structVal)); FALLTHROUGH; + case GT_IND: case GT_OBJ: case GT_BLK: - // These should already have the appropriate type. - assert(structVal->gtType == structType); - alreadyNormalized = true; - break; - - case GT_IND: - assert(structVal->gtType == structType); - structVal = gtNewObjNode(structHnd, structVal->gtGetOp1()); - alreadyNormalized = true; - break; - + case GT_FIELD: case GT_CNS_VEC: - assert(varTypeIsSIMD(structVal) && (structVal->gtType == structType)); - break; - #ifdef FEATURE_SIMD case GT_SIMD: - assert(varTypeIsSIMD(structVal) && (structVal->gtType == structType)); - break; -#endif // FEATURE_SIMD +#endif #ifdef FEATURE_HW_INTRINSICS case GT_HWINTRINSIC: - assert(structVal->gtType == structType); - assert(varTypeIsSIMD(structVal) || - HWIntrinsicInfo::IsMultiReg(structVal->AsHWIntrinsic()->GetHWIntrinsicId())); - break; #endif + case GT_MKREFANY: + // These should already have the appropriate type. + assert(structVal->TypeGet() == structType); + break; case GT_COMMA: { @@ -1776,40 +1372,32 @@ GenTree* Compiler::impNormStructVal(GenTree* structVal, CORINFO_CLASS_HANDLE str } while (blockNode->OperGet() == GT_COMMA); } - if (blockNode->OperGet() == GT_FIELD) - { - // If we have a GT_FIELD then wrap it in a GT_OBJ. - blockNode = gtNewObjNode(structHnd, gtNewOperNode(GT_ADDR, TYP_BYREF, blockNode)); - } - #ifdef FEATURE_SIMD if (blockNode->OperIsSimdOrHWintrinsic() || blockNode->IsCnsVec()) { parent->AsOp()->gtOp2 = impNormStructVal(blockNode, structHnd, curLevel); - alreadyNormalized = true; } else #endif { - noway_assert(blockNode->OperIsBlk()); + noway_assert(blockNode->OperIsBlk() || blockNode->OperIs(GT_FIELD)); // Sink the GT_COMMA below the blockNode addr. - // That is GT_COMMA(op1, op2=blockNode) is tranformed into + // That is GT_COMMA(op1, op2=blockNode) is transformed into // blockNode(GT_COMMA(TYP_BYREF, op1, op2's op1)). // // In case of a chained GT_COMMA case, we sink the last // GT_COMMA below the blockNode addr. GenTree* blockNodeAddr = blockNode->AsOp()->gtOp1; - assert(blockNodeAddr->gtType == TYP_BYREF); + assert(blockNodeAddr->TypeIs(TYP_BYREF, TYP_I_IMPL)); GenTree* commaNode = parent; - commaNode->gtType = TYP_BYREF; + commaNode->gtType = blockNodeAddr->gtType; commaNode->AsOp()->gtOp2 = blockNodeAddr; blockNode->AsOp()->gtOp1 = commaNode; if (parent == structVal) { structVal = blockNode; } - alreadyNormalized = true; } } break; @@ -1818,22 +1406,19 @@ GenTree* Compiler::impNormStructVal(GenTree* structVal, CORINFO_CLASS_HANDLE str noway_assert(!"Unexpected node in impNormStructVal()"); break; } - structVal->gtType = structType; - if (!alreadyNormalized) + if (makeTemp) { - if (makeTemp) - { - unsigned tmpNum = lvaGrabTemp(true DEBUGARG("struct address for call/obj")); + unsigned tmpNum = lvaGrabTemp(true DEBUGARG("struct address for call/obj")); - impAssignTempGen(tmpNum, structVal, structHnd, curLevel); + impAssignTempGen(tmpNum, structVal, structHnd, curLevel); - // The structVal is now the temp itself + // The structVal is now the temp itself - structLcl = gtNewLclvNode(tmpNum, structType)->AsLclVarCommon(); - structVal = structLcl; - } - if ((structType == TYP_STRUCT) && !structVal->OperIsBlk()) + structLcl = gtNewLclvNode(tmpNum, structType)->AsLclVarCommon(); + structVal = structLcl; + + if (structType == TYP_STRUCT) { // Wrap it in a GT_OBJ structVal = gtNewObjNode(structHnd, gtNewOperNode(GT_ADDR, TYP_BYREF, structVal)); @@ -2205,7 +1790,7 @@ GenTree* Compiler::impRuntimeLookupToTree(CORINFO_RESOLVED_TOKEN* pResolvedToken if (pRuntimeLookup->testForNull) { - slotPtrTree = impCloneExpr(ctxTree, &ctxTree, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, + slotPtrTree = impCloneExpr(ctxTree, &ctxTree, NO_CLASS_HANDLE, CHECK_SPILL_ALL, nullptr DEBUGARG("impRuntimeLookup slot")); } @@ -2217,7 +1802,7 @@ GenTree* Compiler::impRuntimeLookupToTree(CORINFO_RESOLVED_TOKEN* pResolvedToken { if ((i == 1 && pRuntimeLookup->indirectFirstOffset) || (i == 2 && pRuntimeLookup->indirectSecondOffset)) { - indOffTree = impCloneExpr(slotPtrTree, &slotPtrTree, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, + indOffTree = impCloneExpr(slotPtrTree, &slotPtrTree, NO_CLASS_HANDLE, CHECK_SPILL_ALL, nullptr DEBUGARG("impRuntimeLookup indirectOffset")); } @@ -2244,7 +1829,7 @@ GenTree* Compiler::impRuntimeLookupToTree(CORINFO_RESOLVED_TOKEN* pResolvedToken { if (isLastIndirectionWithSizeCheck) { - lastIndOfTree = impCloneExpr(slotPtrTree, &slotPtrTree, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, + lastIndOfTree = impCloneExpr(slotPtrTree, &slotPtrTree, NO_CLASS_HANDLE, CHECK_SPILL_ALL, nullptr DEBUGARG("impRuntimeLookup indirectOffset")); } @@ -2272,7 +1857,7 @@ GenTree* Compiler::impRuntimeLookupToTree(CORINFO_RESOLVED_TOKEN* pResolvedToken impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("bubbling QMark0")); unsigned slotLclNum = lvaGrabTemp(true DEBUGARG("impRuntimeLookup test")); - impAssignTempGen(slotLclNum, slotPtrTree, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, nullptr, impCurStmtDI); + impAssignTempGen(slotLclNum, slotPtrTree, NO_CLASS_HANDLE, CHECK_SPILL_ALL, nullptr, impCurStmtDI); GenTree* slot = gtNewLclvNode(slotLclNum, TYP_I_IMPL); // downcast the pointer to a TYP_INT on 64-bit targets @@ -2292,7 +1877,7 @@ GenTree* Compiler::impRuntimeLookupToTree(CORINFO_RESOLVED_TOKEN* pResolvedToken GenTree* asg = gtNewAssignNode(slot, indir); GenTreeColon* colon = new (this, GT_COLON) GenTreeColon(TYP_VOID, gtNewNothingNode(), asg); GenTreeQmark* qmark = gtNewQmarkNode(TYP_VOID, relop, colon); - impAppendTree(qmark, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); + impAppendTree(qmark, CHECK_SPILL_NONE, impCurStmtDI); return gtNewLclvNode(slotLclNum, TYP_I_IMPL); } @@ -2353,18 +1938,10 @@ GenTree* Compiler::impRuntimeLookupToTree(CORINFO_RESOLVED_TOKEN* pResolvedToken unsigned tmp = lvaGrabTemp(true DEBUGARG("spilling Runtime Lookup tree")); - impAssignTempGen(tmp, result, (unsigned)CHECK_SPILL_NONE); + impAssignTempGen(tmp, result, CHECK_SPILL_NONE); return gtNewLclvNode(tmp, TYP_I_IMPL); } -/****************************************************************************** - * Spills the stack at verCurrentState.esStack[level] and replaces it with a temp. - * If tnum!=BAD_VAR_NUM, the temp var used to replace the tree is tnum, - * else, grab a new temp. - * For structs (which can be pushed on the stack using obj, etc), - * special handling is needed - */ - struct RecursiveGuard { public: @@ -2498,9 +2075,9 @@ void Compiler::impSpillStackEnsure(bool spillLeaves) * On return the stack is guaranteed to be empty. */ -inline void Compiler::impEvalSideEffects() +void Compiler::impEvalSideEffects() { - impSpillSideEffects(false, (unsigned)CHECK_SPILL_ALL DEBUGARG("impEvalSideEffects")); + impSpillSideEffects(false, CHECK_SPILL_ALL DEBUGARG("impEvalSideEffects")); verCurrentState.esStackDepth = 0; } @@ -2520,7 +2097,7 @@ void Compiler::impSpillSideEffect(bool spillGlobEffects, unsigned i DEBUGARG(con if ((tree->gtFlags & spillFlags) != 0 || (spillGlobEffects && // Only consider the following when spillGlobEffects == true - !impIsAddressInLocal(tree) && // No need to spill the GT_ADDR node on a local. + !impIsAddressInLocal(tree) && // No need to spill the LCL_ADDR nodes. gtHasLocalsWithAddrOp(tree))) // Spill if we still see GT_LCL_VAR that contains lvHasLdAddrOp or // lvAddrTaken flag. { @@ -2535,16 +2112,16 @@ void Compiler::impSpillSideEffect(bool spillGlobEffects, unsigned i DEBUGARG(con * [0..chkLevel) is the portion of the stack which will be checked and spilled. */ -inline void Compiler::impSpillSideEffects(bool spillGlobEffects, unsigned chkLevel DEBUGARG(const char* reason)) +void Compiler::impSpillSideEffects(bool spillGlobEffects, unsigned chkLevel DEBUGARG(const char* reason)) { - assert(chkLevel != (unsigned)CHECK_SPILL_NONE); + assert(chkLevel != CHECK_SPILL_NONE); /* Before we make any appends to the tree list we must spill the * "special" side effects (GTF_ORDER_SIDEEFF on a GT_CATCH_ARG) */ impSpillSpecialSideEff(); - if (chkLevel == (unsigned)CHECK_SPILL_ALL) + if (chkLevel == CHECK_SPILL_ALL) { chkLevel = verCurrentState.esStackDepth; } @@ -2563,7 +2140,7 @@ inline void Compiler::impSpillSideEffects(bool spillGlobEffects, unsigned chkLev * those trees to temps and replace them on the stack with refs to their temps. */ -inline void Compiler::impSpillSpecialSideEff() +void Compiler::impSpillSpecialSideEff() { // Only exception objects need to be carefully handled @@ -2583,21 +2160,27 @@ inline void Compiler::impSpillSpecialSideEff() } } -/***************************************************************************** - * - * If the stack contains any trees with references to local #lclNum, assign - * those trees to temps and replace their place on the stack with refs to - * their temps. - */ - -void Compiler::impSpillLclRefs(unsigned lclNum) +//------------------------------------------------------------------------ +// impSpillLclRefs: Spill all trees referencing the given local. +// +// Arguments: +// lclNum - The local's number +// chkLevel - Height (exclusive) of the portion of the stack to check +// +void Compiler::impSpillLclRefs(unsigned lclNum, unsigned chkLevel) { - /* Before we make any appends to the tree list we must spill the - * "special" side effects (GTF_ORDER_SIDEEFF) - GT_CATCH_ARG */ - + // Before we make any appends to the tree list we must spill the + // "special" side effects (GTF_ORDER_SIDEEFF) - GT_CATCH_ARG. impSpillSpecialSideEff(); - for (unsigned level = 0; level < verCurrentState.esStackDepth; level++) + if (chkLevel == CHECK_SPILL_ALL) + { + chkLevel = verCurrentState.esStackDepth; + } + + assert(chkLevel <= verCurrentState.esStackDepth); + + for (unsigned level = 0; level < chkLevel; level++) { GenTree* tree = verCurrentState.esStack[level].val; @@ -2668,7 +2251,7 @@ BasicBlock* Compiler::impPushCatchArgOnStack(BasicBlock* hndBlk, CORINFO_CLASS_H #if defined(JIT32_GCENCODER) const bool forceInsertNewBlock = isSingleBlockFilter || compStressCompile(STRESS_CATCH_ARG, 5); #else - const bool forceInsertNewBlock = compStressCompile(STRESS_CATCH_ARG, 5); + const bool forceInsertNewBlock = compStressCompile(STRESS_CATCH_ARG, 5); #endif // defined(JIT32_GCENCODER) /* Spill GT_CATCH_ARG to a temp if there are jumps to the beginning of the handler */ @@ -2796,7 +2379,7 @@ DebugInfo Compiler::impCreateDIWithCurrentStackInfo(IL_OFFSET offs, bool isCall) // statements to report debug info for to the EE: for other statements, they // will have no debug information attached. // -inline void Compiler::impCurStmtOffsSet(IL_OFFSET offs) +void Compiler::impCurStmtOffsSet(IL_OFFSET offs) { if (offs == BAD_IL_OFFSET) { @@ -2869,7 +2452,7 @@ void Compiler::impNoteBranchOffs() { if (opts.compDbgCode) { - impAppendTree(gtNewNothingNode(), (unsigned)CHECK_SPILL_NONE, impCurStmtDI); + impAppendTree(gtNewNothingNode(), CHECK_SPILL_NONE, impCurStmtDI); } } @@ -2974,7 +2557,7 @@ bool Compiler::impOpcodeIsCallOpcode(OPCODE opcode) /*****************************************************************************/ -static inline bool impOpcodeIsCallSiteBoundary(OPCODE opcode) +static bool impOpcodeIsCallSiteBoundary(OPCODE opcode) { switch (opcode) { @@ -3108,19956 +2691,11234 @@ GenTree* Compiler::impImplicitR4orR8Cast(GenTree* tree, var_types dstTyp) return tree; } -//------------------------------------------------------------------------ -// impInitializeArrayIntrinsic: Attempts to replace a call to InitializeArray -// with a GT_COPYBLK node. -// -// Arguments: -// sig - The InitializeArray signature. -// -// Return Value: -// A pointer to the newly created GT_COPYBLK node if the replacement succeeds or -// nullptr otherwise. -// -// Notes: -// The function recognizes the following IL pattern: -// ldc or a list of ldc / -// newarr or newobj -// dup -// ldtoken -// call InitializeArray -// The lower bounds need not be constant except when the array rank is 1. -// The function recognizes all kinds of arrays thus enabling a small runtime -// such as NativeAOT to skip providing an implementation for InitializeArray. - -GenTree* Compiler::impInitializeArrayIntrinsic(CORINFO_SIG_INFO* sig) +GenTree* Compiler::impTypeIsAssignable(GenTree* typeTo, GenTree* typeFrom) { - assert(sig->numArgs == 2); - - GenTree* fieldTokenNode = impStackTop(0).val; - GenTree* arrayLocalNode = impStackTop(1).val; - + // Optimize patterns like: // - // Verify that the field token is known and valid. Note that it's also - // possible for the token to come from reflection, in which case we cannot do - // the optimization and must therefore revert to calling the helper. You can - // see an example of this in bvt\DynIL\initarray2.exe (in Main). + // typeof(TTo).IsAssignableFrom(typeof(TTFrom)) + // valueTypeVar.GetType().IsAssignableFrom(typeof(TTFrom)) + // typeof(TTFrom).IsAssignableTo(typeof(TTo)) + // typeof(TTFrom).IsAssignableTo(valueTypeVar.GetType()) // + // to true/false - // Check to see if the ldtoken helper call is what we see here. - if (fieldTokenNode->gtOper != GT_CALL || (fieldTokenNode->AsCall()->gtCallType != CT_HELPER) || - (fieldTokenNode->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_FIELDDESC_TO_STUBRUNTIMEFIELD))) + // make sure both arguments are `typeof()` + CORINFO_CLASS_HANDLE hClassTo = NO_CLASS_HANDLE; + CORINFO_CLASS_HANDLE hClassFrom = NO_CLASS_HANDLE; + if (gtIsTypeof(typeTo, &hClassTo) && gtIsTypeof(typeFrom, &hClassFrom)) { - return nullptr; - } + TypeCompareState castResult = info.compCompHnd->compareTypesForCast(hClassFrom, hClassTo); + if (castResult == TypeCompareState::May) + { + // requires runtime check + // e.g. __Canon, COMObjects, Nullable + return nullptr; + } - // Strip helper call away - fieldTokenNode = fieldTokenNode->AsCall()->gtArgs.GetArgByIndex(0)->GetEarlyNode(); + GenTreeIntCon* retNode = gtNewIconNode((castResult == TypeCompareState::Must) ? 1 : 0); + impPopStack(); // drop both CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE calls + impPopStack(); - if (fieldTokenNode->gtOper == GT_IND) - { - fieldTokenNode = fieldTokenNode->AsOp()->gtOp1; + return retNode; } - // Check for constant - if (fieldTokenNode->gtOper != GT_CNS_INT) - { - return nullptr; - } + return nullptr; +} - CORINFO_FIELD_HANDLE fieldToken = (CORINFO_FIELD_HANDLE)fieldTokenNode->AsIntCon()->gtCompileTimeHandle; - if (!fieldTokenNode->IsIconHandle(GTF_ICON_FIELD_HDL) || (fieldToken == nullptr)) - { - return nullptr; - } +/***************************************************************************** + * 'logMsg' is true if a log message needs to be logged. false if the caller has + * already logged it (presumably in a more detailed fashion than done here) + */ - // - // We need to get the number of elements in the array and the size of each element. - // We verify that the newarr statement is exactly what we expect it to be. - // If it's not then we just return NULL and we don't optimize this call - // +void Compiler::verConvertBBToThrowVerificationException(BasicBlock* block DEBUGARG(bool logMsg)) +{ + block->bbJumpKind = BBJ_THROW; + block->bbFlags |= BBF_FAILED_VERIFICATION; + block->bbFlags &= ~BBF_IMPORTED; - // It is possible the we don't have any statements in the block yet. - if (impLastStmt == nullptr) + impCurStmtOffsSet(block->bbCodeOffs); + + // Clear the statement list as it exists so far; we're only going to have a verification exception. + impStmtList = impLastStmt = nullptr; + +#ifdef DEBUG + if (logMsg) { - return nullptr; + JITLOG((LL_ERROR, "Verification failure: while compiling %s near IL offset %x..%xh \n", info.compFullName, + block->bbCodeOffs, block->bbCodeOffsEnd)); + if (verbose) + { + printf("\n\nVerification failure: %s near IL %xh \n", info.compFullName, block->bbCodeOffs); + } } - // - // We start by looking at the last statement, making sure it's an assignment, and - // that the target of the assignment is the array passed to InitializeArray. - // - GenTree* arrayAssignment = impLastStmt->GetRootNode(); - if ((arrayAssignment->gtOper != GT_ASG) || (arrayAssignment->AsOp()->gtOp1->gtOper != GT_LCL_VAR) || - (arrayLocalNode->gtOper != GT_LCL_VAR) || (arrayAssignment->AsOp()->gtOp1->AsLclVarCommon()->GetLclNum() != - arrayLocalNode->AsLclVarCommon()->GetLclNum())) + if (JitConfig.DebugBreakOnVerificationFailure()) { - return nullptr; + DebugBreak(); } +#endif - // - // Make sure that the object being assigned is a helper call. - // + impBeginTreeList(); - GenTree* newArrayCall = arrayAssignment->AsOp()->gtOp2; - if ((newArrayCall->gtOper != GT_CALL) || (newArrayCall->AsCall()->gtCallType != CT_HELPER)) + // if the stack is non-empty evaluate all the side-effects + if (verCurrentState.esStackDepth > 0) { - return nullptr; + impEvalSideEffects(); } + assert(verCurrentState.esStackDepth == 0); - // - // Verify that it is one of the new array helpers. - // - - bool isMDArray = false; + GenTree* op1 = gtNewHelperCallNode(CORINFO_HELP_VERIFICATION, TYP_VOID, gtNewIconNode(block->bbCodeOffs)); + // verCurrentState.esStackDepth = 0; + impAppendTree(op1, CHECK_SPILL_NONE, impCurStmtDI); - if (newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEWARR_1_DIRECT) && - newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEWARR_1_OBJ) && - newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEWARR_1_VC) && - newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEWARR_1_ALIGN8) -#ifdef FEATURE_READYTORUN - && newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_READYTORUN_NEWARR_1) -#endif - ) - { - if (newArrayCall->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_NEW_MDARR)) - { - return nullptr; - } + // The inliner is not able to handle methods that require throw block, so + // make sure this methods never gets inlined. + info.compCompHnd->setMethodAttribs(info.compMethodHnd, CORINFO_FLG_BAD_INLINEE); +} - isMDArray = true; - } +/***************************************************************************** + * + */ +void Compiler::verHandleVerificationFailure(BasicBlock* block DEBUGARG(bool logMsg)) +{ + verResetCurrentState(block, &verCurrentState); + verConvertBBToThrowVerificationException(block DEBUGARG(logMsg)); - CORINFO_CLASS_HANDLE arrayClsHnd = (CORINFO_CLASS_HANDLE)newArrayCall->AsCall()->compileTimeHelperArgumentHandle; +#ifdef DEBUG + impNoteLastILoffs(); // Remember at which BC offset the tree was finished +#endif // DEBUG +} - // - // Make sure we found a compile time handle to the array - // +typeInfo Compiler::verMakeTypeInfoForLocal(unsigned lclNum) +{ + LclVarDsc* varDsc = lvaGetDesc(lclNum); - if (!arrayClsHnd) + if ((varDsc->TypeGet() == TYP_BLK) || (varDsc->TypeGet() == TYP_LCLBLK)) { - return nullptr; + return typeInfo(); } - - unsigned rank = 0; - S_UINT32 numElements; - - if (isMDArray) + if (varDsc->TypeGet() == TYP_BYREF) { - rank = info.compCompHnd->getArrayRank(arrayClsHnd); + // Pretend all byrefs are pointing to bytes. + return typeInfo(TI_BYTE).MakeByRef(); + } + if (varTypeIsStruct(varDsc)) + { + return typeInfo(TI_STRUCT, varDsc->GetStructHnd()); + } - if (rank == 0) - { - return nullptr; - } + return typeInfo(varDsc->TypeGet()); +} - assert(newArrayCall->AsCall()->gtArgs.CountArgs() == 3); - GenTree* numArgsArg = newArrayCall->AsCall()->gtArgs.GetArgByIndex(1)->GetNode(); - GenTree* argsArg = newArrayCall->AsCall()->gtArgs.GetArgByIndex(2)->GetNode(); +typeInfo Compiler::verMakeTypeInfo(CorInfoType ciType, CORINFO_CLASS_HANDLE clsHnd) +{ + assert(ciType < CORINFO_TYPE_COUNT); - // - // The number of arguments should be a constant between 1 and 64. The rank can't be 0 - // so at least one length must be present and the rank can't exceed 32 so there can - // be at most 64 arguments - 32 lengths and 32 lower bounds. - // + typeInfo tiResult; + switch (ciType) + { + case CORINFO_TYPE_STRING: + case CORINFO_TYPE_CLASS: + tiResult = verMakeTypeInfo(clsHnd); + if (!tiResult.IsType(TI_REF)) + { // type must be consistent with element type + return typeInfo(); + } + break; - if ((!numArgsArg->IsCnsIntOrI()) || (numArgsArg->AsIntCon()->IconValue() < 1) || - (numArgsArg->AsIntCon()->IconValue() > 64)) - { - return nullptr; - } + case CORINFO_TYPE_VALUECLASS: + case CORINFO_TYPE_REFANY: + tiResult = verMakeTypeInfo(clsHnd); + // type must be constant with element type; + if (!tiResult.IsValueClass()) + { + return typeInfo(); + } + break; + case CORINFO_TYPE_VAR: + return verMakeTypeInfo(clsHnd); - unsigned numArgs = static_cast(numArgsArg->AsIntCon()->IconValue()); - bool lowerBoundsSpecified; + case CORINFO_TYPE_PTR: // for now, pointers are treated as an error + case CORINFO_TYPE_VOID: + return typeInfo(); + break; - if (numArgs == rank * 2) + case CORINFO_TYPE_BYREF: { - lowerBoundsSpecified = true; + CORINFO_CLASS_HANDLE childClassHandle; + CorInfoType childType = info.compCompHnd->getChildType(clsHnd, &childClassHandle); + return ByRef(verMakeTypeInfo(childType, childClassHandle)); } - else if (numArgs == rank) - { - lowerBoundsSpecified = false; - - // - // If the rank is 1 and a lower bound isn't specified then the runtime creates - // a SDArray. Note that even if a lower bound is specified it can be 0 and then - // we get a SDArray as well, see the for loop below. - // + break; - if (rank == 1) - { - isMDArray = false; - } - } - else - { - return nullptr; - } - - // - // The rank is known to be at least 1 so we can start with numElements being 1 - // to avoid the need to special case the first dimension. - // - - numElements = S_UINT32(1); - - struct Match - { - static bool IsArgsFieldInit(GenTree* tree, unsigned index, unsigned lvaNewObjArrayArgs) - { - return (tree->OperGet() == GT_ASG) && IsArgsFieldIndir(tree->gtGetOp1(), index, lvaNewObjArrayArgs) && - IsArgsAddr(tree->gtGetOp1()->gtGetOp1()->gtGetOp1(), lvaNewObjArrayArgs); - } - - static bool IsArgsFieldIndir(GenTree* tree, unsigned index, unsigned lvaNewObjArrayArgs) - { - return (tree->OperGet() == GT_IND) && (tree->gtGetOp1()->OperGet() == GT_ADD) && - (tree->gtGetOp1()->gtGetOp2()->IsIntegralConst(sizeof(INT32) * index)) && - IsArgsAddr(tree->gtGetOp1()->gtGetOp1(), lvaNewObjArrayArgs); - } - - static bool IsArgsAddr(GenTree* tree, unsigned lvaNewObjArrayArgs) - { - return (tree->OperGet() == GT_ADDR) && (tree->gtGetOp1()->OperGet() == GT_LCL_VAR) && - (tree->gtGetOp1()->AsLclVar()->GetLclNum() == lvaNewObjArrayArgs); + default: + if (clsHnd) + { // If we have more precise information, use it + return typeInfo(TI_STRUCT, clsHnd); } - - static bool IsComma(GenTree* tree) + else { - return (tree != nullptr) && (tree->OperGet() == GT_COMMA); + return typeInfo(JITtype2tiType(ciType)); } - }; - - unsigned argIndex = 0; - GenTree* comma; - - for (comma = argsArg; Match::IsComma(comma); comma = comma->gtGetOp2()) - { - if (lowerBoundsSpecified) - { - // - // In general lower bounds can be ignored because they're not needed to - // calculate the total number of elements. But for single dimensional arrays - // we need to know if the lower bound is 0 because in this case the runtime - // creates a SDArray and this affects the way the array data offset is calculated. - // - - if (rank == 1) - { - GenTree* lowerBoundAssign = comma->gtGetOp1(); - assert(Match::IsArgsFieldInit(lowerBoundAssign, argIndex, lvaNewObjArrayArgs)); - GenTree* lowerBoundNode = lowerBoundAssign->gtGetOp2(); + } + return tiResult; +} - if (lowerBoundNode->IsIntegralConst(0)) - { - isMDArray = false; - } - } +/******************************************************************************/ - comma = comma->gtGetOp2(); - argIndex++; - } +typeInfo Compiler::verMakeTypeInfo(CORINFO_CLASS_HANDLE clsHnd) +{ + if (clsHnd == NO_CLASS_HANDLE) + { + return typeInfo(); + } - GenTree* lengthNodeAssign = comma->gtGetOp1(); - assert(Match::IsArgsFieldInit(lengthNodeAssign, argIndex, lvaNewObjArrayArgs)); - GenTree* lengthNode = lengthNodeAssign->gtGetOp2(); + // Byrefs should only occur in method and local signatures, which are accessed + // using ICorClassInfo and ICorClassInfo.getChildType. + // So findClass() and getClassAttribs() should not be called for byrefs - if (!lengthNode->IsCnsIntOrI()) - { - return nullptr; - } + if (JITtype2varType(info.compCompHnd->asCorInfoType(clsHnd)) == TYP_BYREF) + { + assert(!"Did findClass() return a Byref?"); + return typeInfo(); + } - numElements *= S_SIZE_T(lengthNode->AsIntCon()->IconValue()); - argIndex++; - } + unsigned attribs = info.compCompHnd->getClassAttribs(clsHnd); - assert((comma != nullptr) && Match::IsArgsAddr(comma, lvaNewObjArrayArgs)); + if (attribs & CORINFO_FLG_VALUECLASS) + { + CorInfoType t = info.compCompHnd->getTypeForPrimitiveValueClass(clsHnd); - if (argIndex != numArgs) + // Meta-data validation should ensure that CORINF_TYPE_BYREF should + // not occur here, so we may want to change this to an assert instead. + if (t == CORINFO_TYPE_VOID || t == CORINFO_TYPE_BYREF || t == CORINFO_TYPE_PTR) { - return nullptr; + return typeInfo(); } - } - else - { - // - // Make sure there are exactly two arguments: the array class and - // the number of elements. - // - - GenTree* arrayLengthNode; -#ifdef FEATURE_READYTORUN - if (newArrayCall->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_READYTORUN_NEWARR_1)) + if (t != CORINFO_TYPE_UNDEF) { - // Array length is 1st argument for readytorun helper - arrayLengthNode = newArrayCall->AsCall()->gtArgs.GetArgByIndex(0)->GetNode(); + return (typeInfo(JITtype2tiType(t))); } else -#endif - { - // Array length is 2nd argument for regular helper - arrayLengthNode = newArrayCall->AsCall()->gtArgs.GetArgByIndex(1)->GetNode(); - } - - // - // This optimization is only valid for a constant array size. - // - if (arrayLengthNode->gtOper != GT_CNS_INT) { - return nullptr; + return (typeInfo(TI_STRUCT, clsHnd)); } + } + else + { + return (typeInfo(TI_REF, clsHnd)); + } +} - numElements = S_SIZE_T(arrayLengthNode->AsIntCon()->gtIconVal); +/***************************************************************************** + */ +typeInfo Compiler::verParseArgSigToTypeInfo(CORINFO_SIG_INFO* sig, CORINFO_ARG_LIST_HANDLE args) +{ + CORINFO_CLASS_HANDLE classHandle; + CorInfoType ciType = strip(info.compCompHnd->getArgType(sig, args, &classHandle)); - if (!info.compCompHnd->isSDArray(arrayClsHnd)) + var_types type = JITtype2varType(ciType); + if (varTypeIsGC(type)) + { + // For efficiency, getArgType only returns something in classHandle for + // value types. For other types that have addition type info, you + // have to call back explicitly + classHandle = info.compCompHnd->getArgClass(sig, args); + if (!classHandle) { - return nullptr; + NO_WAY("Could not figure out Class specified in argument or local signature"); } } - CORINFO_CLASS_HANDLE elemClsHnd; - var_types elementType = JITtype2varType(info.compCompHnd->getChildType(arrayClsHnd, &elemClsHnd)); - - // - // Note that genTypeSize will return zero for non primitive types, which is exactly - // what we want (size will then be 0, and we will catch this in the conditional below). - // Note that we don't expect this to fail for valid binaries, so we assert in the - // non-verification case (the verification case should not assert but rather correctly - // handle bad binaries). This assert is not guarding any specific invariant, but rather - // saying that we don't expect this to happen, and if it is hit, we need to investigate - // why. - // - - S_UINT32 elemSize(genTypeSize(elementType)); - S_UINT32 size = elemSize * S_UINT32(numElements); + return verMakeTypeInfo(ciType, classHandle); +} - if (size.IsOverflow()) +bool Compiler::verIsByRefLike(const typeInfo& ti) +{ + if (ti.IsByRef()) { - return nullptr; + return true; } - - if ((size.Value() == 0) || (varTypeIsGC(elementType))) + if (!ti.IsType(TI_STRUCT)) { - return nullptr; + return false; } + return info.compCompHnd->getClassAttribs(ti.GetClassHandleForValueClass()) & CORINFO_FLG_BYREF_LIKE; +} - void* initData = info.compCompHnd->getArrayInitializationData(fieldToken, size.Value()); - if (!initData) - { - return nullptr; - } +/***************************************************************************** + * + * Check if a TailCall is legal. + */ - // - // At this point we are ready to commit to implementing the InitializeArray - // intrinsic using a struct assignment. Pop the arguments from the stack and - // return the struct assignment node. - // +bool Compiler::verCheckTailCallConstraint(OPCODE opcode, + CORINFO_RESOLVED_TOKEN* pResolvedToken, + CORINFO_RESOLVED_TOKEN* pConstrainedResolvedToken) +{ + DWORD mflags; + CORINFO_SIG_INFO sig; + unsigned int popCount = 0; // we can't pop the stack since impImportCall needs it, so + // this counter is used to keep track of how many items have been + // virtually popped - impPopStack(); - impPopStack(); + CORINFO_METHOD_HANDLE methodHnd = nullptr; + CORINFO_CLASS_HANDLE methodClassHnd = nullptr; + unsigned methodClassFlgs = 0; - const unsigned blkSize = size.Value(); - unsigned dataOffset; + assert(impOpcodeIsCallOpcode(opcode)); - if (isMDArray) + if (compIsForInlining()) { - dataOffset = eeGetMDArrayDataOffset(rank); + return false; } - else + + // For calli, check that this is not a virtual method. + if (opcode == CEE_CALLI) { - dataOffset = eeGetArrayDataOffset(); + /* Get the call sig */ + eeGetSig(pResolvedToken->token, pResolvedToken->tokenScope, pResolvedToken->tokenContext, &sig); + + // We don't know the target method, so we have to infer the flags, or + // assume the worst-case. + mflags = (sig.callConv & CORINFO_CALLCONV_HASTHIS) ? 0 : CORINFO_FLG_STATIC; } + else + { + methodHnd = pResolvedToken->hMethod; - GenTree* dstAddr = gtNewOperNode(GT_ADD, TYP_BYREF, arrayLocalNode, gtNewIconNode(dataOffset, TYP_I_IMPL)); - GenTree* dst = new (this, GT_BLK) GenTreeBlk(GT_BLK, TYP_STRUCT, dstAddr, typGetBlkLayout(blkSize)); - GenTree* src = gtNewIndOfIconHandleNode(TYP_STRUCT, (size_t)initData, GTF_ICON_CONST_PTR, true); + mflags = info.compCompHnd->getMethodAttribs(methodHnd); -#ifdef DEBUG - src->gtGetOp1()->AsIntCon()->gtTargetHandle = THT_InitializeArrayIntrinsics; -#endif + // In generic code we pair the method handle with its owning class to get the exact method signature. + methodClassHnd = pResolvedToken->hClass; + assert(methodClassHnd != NO_CLASS_HANDLE); - return gtNewBlkOpNode(dst, // dst - src, // src - false, // volatile - true); // copyBlock -} + eeGetMethodSig(methodHnd, &sig, methodClassHnd); -GenTree* Compiler::impCreateSpanIntrinsic(CORINFO_SIG_INFO* sig) -{ - assert(sig->numArgs == 1); - assert(sig->sigInst.methInstCount == 1); + // opcode specific check + methodClassFlgs = info.compCompHnd->getClassAttribs(methodClassHnd); + } - GenTree* fieldTokenNode = impStackTop(0).val; + // We must have got the methodClassHnd if opcode is not CEE_CALLI + assert((methodHnd != nullptr && methodClassHnd != NO_CLASS_HANDLE) || opcode == CEE_CALLI); - // - // Verify that the field token is known and valid. Note that it's also - // possible for the token to come from reflection, in which case we cannot do - // the optimization and must therefore revert to calling the helper. You can - // see an example of this in bvt\DynIL\initarray2.exe (in Main). - // + if ((sig.callConv & CORINFO_CALLCONV_MASK) == CORINFO_CALLCONV_VARARG) + { + eeGetCallSiteSig(pResolvedToken->token, pResolvedToken->tokenScope, pResolvedToken->tokenContext, &sig); + } - // Check to see if the ldtoken helper call is what we see here. - if (fieldTokenNode->gtOper != GT_CALL || (fieldTokenNode->AsCall()->gtCallType != CT_HELPER) || - (fieldTokenNode->AsCall()->gtCallMethHnd != eeFindHelper(CORINFO_HELP_FIELDDESC_TO_STUBRUNTIMEFIELD))) + // Check compatibility of the arguments. + unsigned int argCount = sig.numArgs; + CORINFO_ARG_LIST_HANDLE args; + args = sig.args; + while (argCount--) { - return nullptr; + typeInfo tiDeclared = verParseArgSigToTypeInfo(&sig, args).NormaliseForStack(); + + // Check that the argument is not a byref for tailcalls. + if (verIsByRefLike(tiDeclared)) + { + return false; + } + + // For unsafe code, we might have parameters containing pointer to the stack location. + // Disallow the tailcall for this kind. + CORINFO_CLASS_HANDLE classHandle; + CorInfoType ciType = strip(info.compCompHnd->getArgType(&sig, args, &classHandle)); + if (ciType == CORINFO_TYPE_PTR) + { + return false; + } + + args = info.compCompHnd->getArgNext(args); } - // Strip helper call away - fieldTokenNode = fieldTokenNode->AsCall()->gtArgs.GetArgByIndex(0)->GetNode(); - if (fieldTokenNode->gtOper == GT_IND) + // Update popCount. + popCount += sig.numArgs; + + // Check for 'this' which is on non-static methods, not called via NEWOBJ + if (!(mflags & CORINFO_FLG_STATIC)) { - fieldTokenNode = fieldTokenNode->AsOp()->gtOp1; + // Always update the popCount. This is crucial for the stack calculation to be correct. + typeInfo tiThis = impStackTop(popCount).seTypeInfo; + popCount++; + + if (opcode == CEE_CALLI) + { + // For CALLI, we don't know the methodClassHnd. Therefore, let's check the "this" object + // on the stack. + if (tiThis.IsValueClass()) + { + tiThis.MakeByRef(); + } + + if (verIsByRefLike(tiThis)) + { + return false; + } + } + else + { + // Check type compatibility of the this argument + typeInfo tiDeclaredThis = verMakeTypeInfo(methodClassHnd); + if (tiDeclaredThis.IsValueClass()) + { + tiDeclaredThis.MakeByRef(); + } + + if (verIsByRefLike(tiDeclaredThis)) + { + return false; + } + } } - // Check for constant - if (fieldTokenNode->gtOper != GT_CNS_INT) + // Tail calls on constrained calls should be illegal too: + // when instantiated at a value type, a constrained call may pass the address of a stack allocated value + if (pConstrainedResolvedToken != nullptr) { - return nullptr; + return false; } - CORINFO_FIELD_HANDLE fieldToken = (CORINFO_FIELD_HANDLE)fieldTokenNode->AsIntCon()->gtCompileTimeHandle; - if (!fieldTokenNode->IsIconHandle(GTF_ICON_FIELD_HDL) || (fieldToken == nullptr)) + // Get the exact view of the signature for an array method + if (sig.retType != CORINFO_TYPE_VOID) { - return nullptr; + if (methodClassFlgs & CORINFO_FLG_ARRAY) + { + assert(opcode != CEE_CALLI); + eeGetCallSiteSig(pResolvedToken->token, pResolvedToken->tokenScope, pResolvedToken->tokenContext, &sig); + } } - CORINFO_CLASS_HANDLE fieldOwnerHnd = info.compCompHnd->getFieldClass(fieldToken); + var_types calleeRetType = genActualType(JITtype2varType(sig.retType)); + var_types callerRetType = genActualType(JITtype2varType(info.compMethodInfo->args.retType)); - CORINFO_CLASS_HANDLE fieldClsHnd; - var_types fieldElementType = - JITtype2varType(info.compCompHnd->getFieldType(fieldToken, &fieldClsHnd, fieldOwnerHnd)); - unsigned totalFieldSize; + // Normalize TYP_FLOAT to TYP_DOUBLE (it is ok to return one as the other and vice versa). + calleeRetType = (calleeRetType == TYP_FLOAT) ? TYP_DOUBLE : calleeRetType; + callerRetType = (callerRetType == TYP_FLOAT) ? TYP_DOUBLE : callerRetType; - // Most static initialization data fields are of some structure, but it is possible for them to be of various - // primitive types as well - if (fieldElementType == var_types::TYP_STRUCT) + // Make sure the types match. + if (calleeRetType != callerRetType) { - totalFieldSize = info.compCompHnd->getClassSize(fieldClsHnd); + return false; } - else + else if ((callerRetType == TYP_STRUCT) && (sig.retTypeClass != info.compMethodInfo->args.retTypeClass)) { - totalFieldSize = genTypeSize(fieldElementType); + return false; } - // Limit to primitive or enum type - see ArrayNative::GetSpanDataFrom() - CORINFO_CLASS_HANDLE targetElemHnd = sig->sigInst.methInst[0]; - if (info.compCompHnd->getTypeForPrimitiveValueClass(targetElemHnd) == CORINFO_TYPE_UNDEF) + // For tailcall, stack must be empty. + if (verCurrentState.esStackDepth != popCount) { - return nullptr; + return false; } - const unsigned targetElemSize = info.compCompHnd->getClassSize(targetElemHnd); - assert(targetElemSize != 0); + return true; // Yes, tailcall is legal +} - const unsigned count = totalFieldSize / targetElemSize; - if (count == 0) +GenTree* Compiler::impImportLdvirtftn(GenTree* thisPtr, + CORINFO_RESOLVED_TOKEN* pResolvedToken, + CORINFO_CALL_INFO* pCallInfo) +{ + if ((pCallInfo->methodFlags & CORINFO_FLG_EnC) && !(pCallInfo->classFlags & CORINFO_FLG_INTERFACE)) { - return nullptr; + NO_WAY("Virtual call to a function added via EnC is not supported"); } - void* data = info.compCompHnd->getArrayInitializationData(fieldToken, totalFieldSize); - if (!data) + // NativeAOT generic virtual method + if ((pCallInfo->sig.sigInst.methInstCount != 0) && IsTargetAbi(CORINFO_NATIVEAOT_ABI)) { - return nullptr; + GenTree* runtimeMethodHandle = + impLookupToTree(pResolvedToken, &pCallInfo->codePointerLookup, GTF_ICON_METHOD_HDL, pCallInfo->hMethod); + return gtNewHelperCallNode(CORINFO_HELP_GVMLOOKUP_FOR_SLOT, TYP_I_IMPL, thisPtr, runtimeMethodHandle); } - // - // Ready to commit to the work - // - - impPopStack(); - - // Turn count and pointer value into constants. - GenTree* lengthValue = gtNewIconNode(count, TYP_INT); - GenTree* pointerValue = gtNewIconHandleNode((size_t)data, GTF_ICON_CONST_PTR); - - // Construct ReadOnlySpan to return. - CORINFO_CLASS_HANDLE spanHnd = sig->retTypeClass; - unsigned spanTempNum = lvaGrabTemp(true DEBUGARG("ReadOnlySpan for CreateSpan")); - lvaSetStruct(spanTempNum, spanHnd, false); - - GenTreeLclFld* pointerField = gtNewLclFldNode(spanTempNum, TYP_BYREF, 0); - GenTree* pointerFieldAsg = gtNewAssignNode(pointerField, pointerValue); - - GenTreeLclFld* lengthField = gtNewLclFldNode(spanTempNum, TYP_INT, TARGET_POINTER_SIZE); - GenTree* lengthFieldAsg = gtNewAssignNode(lengthField, lengthValue); - - // Now append a few statements the initialize the span - impAppendTree(lengthFieldAsg, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); - impAppendTree(pointerFieldAsg, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); - - // And finally create a tree that points at the span. - return impCreateLocalNode(spanTempNum DEBUGARG(0)); -} - -//------------------------------------------------------------------------ -// impIntrinsic: possibly expand intrinsic call into alternate IR sequence -// -// Arguments: -// newobjThis - for constructor calls, the tree for the newly allocated object -// clsHnd - handle for the intrinsic method's class -// method - handle for the intrinsic method -// sig - signature of the intrinsic method -// methodFlags - CORINFO_FLG_XXX flags of the intrinsic method -// memberRef - the token for the intrinsic method -// readonlyCall - true if call has a readonly prefix -// tailCall - true if call is in tail position -// pConstrainedResolvedToken -- resolved token for constrained call, or nullptr -// if call is not constrained -// constraintCallThisTransform -- this transform to apply for a constrained call -// pIntrinsicName [OUT] -- intrinsic name (see enumeration in namedintrinsiclist.h) -// for "traditional" jit intrinsics -// isSpecialIntrinsic [OUT] -- set true if intrinsic expansion is a call -// that is amenable to special downstream optimization opportunities -// -// Returns: -// IR tree to use in place of the call, or nullptr if the jit should treat -// the intrinsic call like a normal call. -// -// pIntrinsicName set to non-illegal value if the call is recognized as a -// traditional jit intrinsic, even if the intrinsic is not expaned. -// -// isSpecial set true if the expansion is subject to special -// optimizations later in the jit processing -// -// Notes: -// On success the IR tree may be a call to a different method or an inline -// sequence. If it is a call, then the intrinsic processing here is responsible -// for handling all the special cases, as upon return to impImportCall -// expanded intrinsics bypass most of the normal call processing. -// -// Intrinsics are generally not recognized in minopts and debug codegen. -// -// However, certain traditional intrinsics are identifed as "must expand" -// if there is no fallback implementation to invoke; these must be handled -// in all codegen modes. -// -// New style intrinsics (where the fallback implementation is in IL) are -// identified as "must expand" if they are invoked from within their -// own method bodies. -// -GenTree* Compiler::impIntrinsic(GenTree* newobjThis, - CORINFO_CLASS_HANDLE clsHnd, - CORINFO_METHOD_HANDLE method, - CORINFO_SIG_INFO* sig, - unsigned methodFlags, - int memberRef, - bool readonlyCall, - bool tailCall, - CORINFO_RESOLVED_TOKEN* pConstrainedResolvedToken, - CORINFO_THIS_TRANSFORM constraintCallThisTransform, - NamedIntrinsic* pIntrinsicName, - bool* isSpecialIntrinsic) -{ - assert((methodFlags & CORINFO_FLG_INTRINSIC) != 0); - - bool mustExpand = false; - bool isSpecial = false; - NamedIntrinsic ni = NI_Illegal; - - if ((methodFlags & CORINFO_FLG_INTRINSIC) != 0) +#ifdef FEATURE_READYTORUN + if (opts.IsReadyToRun()) { - // The recursive non-virtual calls to Jit intrinsics are must-expand by convention. - mustExpand = mustExpand || (gtIsRecursiveCall(method) && !(methodFlags & CORINFO_FLG_VIRTUAL)); - - ni = lookupNamedIntrinsic(method); - - // We specially support the following on all platforms to allow for dead - // code optimization and to more generally support recursive intrinsics. - - if (ni == NI_IsSupported_True) - { - assert(sig->numArgs == 0); - return gtNewIconNode(true); - } - - if (ni == NI_IsSupported_False) - { - assert(sig->numArgs == 0); - return gtNewIconNode(false); - } - - if (ni == NI_Throw_PlatformNotSupportedException) - { - return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand); - } - - if ((ni > NI_SRCS_UNSAFE_START) && (ni < NI_SRCS_UNSAFE_END)) - { - assert(!mustExpand); - return impSRCSUnsafeIntrinsic(ni, clsHnd, method, sig); - } - -#ifdef FEATURE_HW_INTRINSICS - if ((ni > NI_HW_INTRINSIC_START) && (ni < NI_HW_INTRINSIC_END)) + if (!pCallInfo->exactContextNeedsRuntimeLookup) { - GenTree* hwintrinsic = impHWIntrinsic(ni, clsHnd, method, sig, mustExpand); + GenTreeCall* call = gtNewHelperCallNode(CORINFO_HELP_READYTORUN_VIRTUAL_FUNC_PTR, TYP_I_IMPL, thisPtr); - if (mustExpand && (hwintrinsic == nullptr)) - { - return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_NOT_IMPLEMENTED, method, sig, mustExpand); - } + call->setEntryPoint(pCallInfo->codePointerLookup.constLookup); - return hwintrinsic; + return call; } - if ((ni > NI_SIMD_AS_HWINTRINSIC_START) && (ni < NI_SIMD_AS_HWINTRINSIC_END)) + // We need a runtime lookup. NativeAOT has a ReadyToRun helper for that too. + if (IsTargetAbi(CORINFO_NATIVEAOT_ABI)) { - // These intrinsics aren't defined recursively and so they will never be mustExpand - // Instead, they provide software fallbacks that will be executed instead. + GenTree* ctxTree = getRuntimeContextTree(pCallInfo->codePointerLookup.lookupKind.runtimeLookupKind); - assert(!mustExpand); - return impSimdAsHWIntrinsic(ni, clsHnd, method, sig, newobjThis); + return impReadyToRunHelperToTree(pResolvedToken, CORINFO_HELP_READYTORUN_GENERIC_HANDLE, TYP_I_IMPL, + &pCallInfo->codePointerLookup.lookupKind, ctxTree); } -#endif // FEATURE_HW_INTRINSICS } +#endif - *pIntrinsicName = ni; - - if (ni == NI_System_StubHelpers_GetStubContext) - { - // must be done regardless of DbgCode and MinOpts - return gtNewLclvNode(lvaStubArgumentVar, TYP_I_IMPL); + // Get the exact descriptor for the static callsite + GenTree* exactTypeDesc = impParentClassTokenToHandle(pResolvedToken); + if (exactTypeDesc == nullptr) + { // compDonotInline() + return nullptr; } - if (ni == NI_System_StubHelpers_NextCallReturnAddress) - { - // For now we just avoid inlining anything into these methods since - // this intrinsic is only rarely used. We could do this better if we - // wanted to by trying to match which call is the one we need to get - // the return address of. - info.compHasNextCallRetAddr = true; - return new (this, GT_LABEL) GenTree(GT_LABEL, TYP_I_IMPL); + GenTree* exactMethodDesc = impTokenToHandle(pResolvedToken); + if (exactMethodDesc == nullptr) + { // compDonotInline() + return nullptr; } - switch (ni) - { - // CreateSpan must be expanded for NativeAOT - case NI_System_Runtime_CompilerServices_RuntimeHelpers_CreateSpan: - case NI_System_Runtime_CompilerServices_RuntimeHelpers_InitializeArray: - mustExpand |= IsTargetAbi(CORINFO_NATIVEAOT_ABI); - break; - - case NI_Internal_Runtime_MethodTable_Of: - case NI_System_Activator_AllocatorOf: - case NI_System_Activator_DefaultConstructorOf: - case NI_System_EETypePtr_EETypePtrOf: - mustExpand = true; - break; + // Call helper function. This gets the target address of the final destination callsite. - default: - break; - } + return gtNewHelperCallNode(CORINFO_HELP_VIRTUAL_FUNC_PTR, TYP_I_IMPL, thisPtr, exactTypeDesc, exactMethodDesc); +} - GenTree* retNode = nullptr; +//------------------------------------------------------------------------ +// impBoxPatternMatch: match and import common box idioms +// +// Arguments: +// pResolvedToken - resolved token from the box operation +// codeAddr - position in IL stream after the box instruction +// codeEndp - end of IL stream +// opts - dictate pattern matching behavior +// +// Return Value: +// Number of IL bytes matched and imported, -1 otherwise +// +// Notes: +// pResolvedToken is known to be a value type; ref type boxing +// is handled in the CEE_BOX clause. - // Under debug and minopts, only expand what is required. - // NextCallReturnAddress intrinsic returns the return address of the next call. - // If that call is an intrinsic and is expanded, codegen for NextCallReturnAddress will fail. - // To avoid that we conservatively expand only required intrinsics in methods that call - // the NextCallReturnAddress intrinsic. - if (!mustExpand && (opts.OptimizationDisabled() || info.compHasNextCallRetAddr)) +int Compiler::impBoxPatternMatch(CORINFO_RESOLVED_TOKEN* pResolvedToken, + const BYTE* codeAddr, + const BYTE* codeEndp, + BoxPatterns opts) +{ + if (codeAddr >= codeEndp) { - *pIntrinsicName = NI_Illegal; - return retNode; + return -1; } - CorInfoType callJitType = sig->retType; - var_types callType = JITtype2varType(callJitType); - - /* First do the intrinsics which are always smaller than a call */ - - if (ni != NI_Illegal) + switch (codeAddr[0]) { - assert(retNode == nullptr); - switch (ni) - { - case NI_Array_Address: - case NI_Array_Get: - case NI_Array_Set: - retNode = impArrayAccessIntrinsic(clsHnd, sig, memberRef, readonlyCall, ni); - break; - - case NI_System_String_Equals: - { - retNode = impStringEqualsOrStartsWith(/*startsWith:*/ false, sig, methodFlags); - break; - } - - case NI_System_MemoryExtensions_Equals: - case NI_System_MemoryExtensions_SequenceEqual: - { - retNode = impSpanEqualsOrStartsWith(/*startsWith:*/ false, sig, methodFlags); - break; - } - - case NI_System_String_StartsWith: - { - retNode = impStringEqualsOrStartsWith(/*startsWith:*/ true, sig, methodFlags); - break; - } - - case NI_System_MemoryExtensions_StartsWith: - { - retNode = impSpanEqualsOrStartsWith(/*startsWith:*/ true, sig, methodFlags); - break; - } - - case NI_System_MemoryExtensions_AsSpan: - case NI_System_String_op_Implicit: - { - assert(sig->numArgs == 1); - isSpecial = impStackTop().val->OperIs(GT_CNS_STR); - break; - } - - case NI_System_String_get_Chars: - { - GenTree* op2 = impPopStack().val; - GenTree* op1 = impPopStack().val; - GenTree* addr = gtNewIndexAddr(op1, op2, TYP_USHORT, NO_CLASS_HANDLE, OFFSETOF__CORINFO_String__chars, - OFFSETOF__CORINFO_String__stringLen); - retNode = gtNewIndexIndir(addr->AsIndexAddr()); - break; - } - - case NI_System_String_get_Length: + case CEE_UNBOX_ANY: + // box + unbox.any + if (codeAddr + 1 + sizeof(mdToken) <= codeEndp) { - GenTree* op1 = impPopStack().val; - if (op1->OperIs(GT_CNS_STR)) + if (opts == BoxPatterns::MakeInlineObservation) { - // Optimize `ldstr + String::get_Length()` to CNS_INT - // e.g. "Hello".Length => 5 - GenTreeIntCon* iconNode = gtNewStringLiteralLength(op1->AsStrCon()); - if (iconNode != nullptr) - { - retNode = iconNode; - break; - } + compInlineResult->Note(InlineObservation::CALLEE_FOLDABLE_BOX); + return 1 + sizeof(mdToken); } - GenTreeArrLen* arrLen = gtNewArrLen(TYP_INT, op1, OFFSETOF__CORINFO_String__stringLen, compCurBB); - op1 = arrLen; - // Getting the length of a null string should throw - op1->gtFlags |= GTF_EXCEPT; + CORINFO_RESOLVED_TOKEN unboxResolvedToken; - retNode = op1; - break; - } + impResolveToken(codeAddr + 1, &unboxResolvedToken, CORINFO_TOKENKIND_Class); - case NI_System_Runtime_CompilerServices_RuntimeHelpers_CreateSpan: - { - retNode = impCreateSpanIntrinsic(sig); - break; - } + // See if the resolved tokens describe types that are equal. + const TypeCompareState compare = + info.compCompHnd->compareTypesForEquality(unboxResolvedToken.hClass, pResolvedToken->hClass); - case NI_System_Runtime_CompilerServices_RuntimeHelpers_InitializeArray: - { - retNode = impInitializeArrayIntrinsic(sig); - break; - } + bool optimize = false; - case NI_System_Runtime_CompilerServices_RuntimeHelpers_IsKnownConstant: - { - GenTree* op1 = impPopStack().val; - if (op1->OperIsConst()) + // If so, box/unbox.any is a nop. + if (compare == TypeCompareState::Must) { - // op1 is a known constant, replace with 'true'. - retNode = gtNewIconNode(1); - JITDUMP("\nExpanding RuntimeHelpers.IsKnownConstant to true early\n"); - // We can also consider FTN_ADDR and typeof(T) here + optimize = true; } - else + else if (compare == TypeCompareState::MustNot) { - // op1 is not a known constant, we'll do the expansion in morph - retNode = new (this, GT_INTRINSIC) GenTreeIntrinsic(TYP_INT, op1, ni, method); - JITDUMP("\nConverting RuntimeHelpers.IsKnownConstant to:\n"); - DISPTREE(retNode); + // An attempt to catch cases where we mix enums and primitives, e.g.: + // (IntEnum)(object)myInt + // (byte)(object)myByteEnum + // + CorInfoType typ = info.compCompHnd->getTypeForPrimitiveValueClass(unboxResolvedToken.hClass); + if ((typ >= CORINFO_TYPE_BYTE) && (typ <= CORINFO_TYPE_ULONG) && + (info.compCompHnd->getTypeForPrimitiveValueClass(pResolvedToken->hClass) == typ)) + { + optimize = true; + } } - break; - } - - case NI_System_Runtime_InteropService_MemoryMarshal_GetArrayDataReference: - { - assert(sig->numArgs == 1); - GenTree* array = impPopStack().val; - CORINFO_CLASS_HANDLE elemHnd = sig->sigInst.methInst[0]; - CorInfoType jitType = info.compCompHnd->asCorInfoType(elemHnd); - var_types elemType = JITtype2varType(jitType); - - GenTree* arrayClone; - array = impCloneExpr(array, &arrayClone, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, - nullptr DEBUGARG("MemoryMarshal.GetArrayDataReference array")); - - impAppendTree(gtNewNullCheck(array, compCurBB), (unsigned)CHECK_SPILL_ALL, impCurStmtDI); - - GenTree* index = gtNewIconNode(0, TYP_I_IMPL); - GenTreeIndexAddr* indexAddr = gtNewArrayIndexAddr(arrayClone, index, elemType, elemHnd); - indexAddr->gtFlags &= ~GTF_INX_RNGCHK; - indexAddr->gtFlags |= GTF_INX_ADDR_NONNULL; - retNode = indexAddr; - break; - } - - case NI_Internal_Runtime_MethodTable_Of: - case NI_System_Activator_AllocatorOf: - case NI_System_Activator_DefaultConstructorOf: - case NI_System_EETypePtr_EETypePtrOf: - { - assert(IsTargetAbi(CORINFO_NATIVEAOT_ABI)); // Only NativeAOT supports it. - CORINFO_RESOLVED_TOKEN resolvedToken; - resolvedToken.tokenContext = impTokenLookupContextHandle; - resolvedToken.tokenScope = info.compScopeHnd; - resolvedToken.token = memberRef; - resolvedToken.tokenType = CORINFO_TOKENKIND_Method; - - CORINFO_GENERICHANDLE_RESULT embedInfo; - info.compCompHnd->expandRawHandleIntrinsic(&resolvedToken, &embedInfo); - - GenTree* rawHandle = impLookupToTree(&resolvedToken, &embedInfo.lookup, gtTokenToIconFlags(memberRef), - embedInfo.compileTimeHandle); - if (rawHandle == nullptr) + if (optimize) { - return nullptr; + JITDUMP("\n Importing BOX; UNBOX.ANY as NOP\n"); + // Skip the next unbox.any instruction + return 1 + sizeof(mdToken); } - - noway_assert(genTypeSize(rawHandle->TypeGet()) == genTypeSize(TYP_I_IMPL)); - - unsigned rawHandleSlot = lvaGrabTemp(true DEBUGARG("rawHandle")); - impAssignTempGen(rawHandleSlot, rawHandle, clsHnd, (unsigned)CHECK_SPILL_NONE); - - GenTree* lclVar = gtNewLclvNode(rawHandleSlot, TYP_I_IMPL); - GenTree* lclVarAddr = gtNewOperNode(GT_ADDR, TYP_I_IMPL, lclVar); - var_types resultType = JITtype2varType(sig->retType); - retNode = gtNewOperNode(GT_IND, resultType, lclVarAddr); - - break; } + break; - case NI_System_Span_get_Item: - case NI_System_ReadOnlySpan_get_Item: + case CEE_BRTRUE: + case CEE_BRTRUE_S: + case CEE_BRFALSE: + case CEE_BRFALSE_S: + // box + br_true/false + if ((codeAddr + ((codeAddr[0] >= CEE_BRFALSE) ? 5 : 2)) <= codeEndp) { - // Have index, stack pointer-to Span s on the stack. Expand to: - // - // For Span - // Comma - // BoundsCheck(index, s->_length) - // s->_reference + index * sizeof(T) - // - // For ReadOnlySpan -- same expansion, as it now returns a readonly ref - // - // Signature should show one class type parameter, which - // we need to examine. - assert(sig->sigInst.classInstCount == 1); - assert(sig->numArgs == 1); - CORINFO_CLASS_HANDLE spanElemHnd = sig->sigInst.classInst[0]; - const unsigned elemSize = info.compCompHnd->getClassSize(spanElemHnd); - assert(elemSize > 0); - - const bool isReadOnly = (ni == NI_System_ReadOnlySpan_get_Item); - - JITDUMP("\nimpIntrinsic: Expanding %sSpan.get_Item, T=%s, sizeof(T)=%u\n", - isReadOnly ? "ReadOnly" : "", eeGetClassName(spanElemHnd), elemSize); - - GenTree* index = impPopStack().val; - GenTree* ptrToSpan = impPopStack().val; - GenTree* indexClone = nullptr; - GenTree* ptrToSpanClone = nullptr; - assert(genActualType(index) == TYP_INT); - assert(ptrToSpan->TypeGet() == TYP_BYREF); - -#if defined(DEBUG) - if (verbose) + if (opts == BoxPatterns::MakeInlineObservation) { - printf("with ptr-to-span\n"); - gtDispTree(ptrToSpan); - printf("and index\n"); - gtDispTree(index); + compInlineResult->Note(InlineObservation::CALLEE_FOLDABLE_BOX); + return 0; } -#endif // defined(DEBUG) - // We need to use both index and ptr-to-span twice, so clone or spill. - index = impCloneExpr(index, &indexClone, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, - nullptr DEBUGARG("Span.get_Item index")); - ptrToSpan = impCloneExpr(ptrToSpan, &ptrToSpanClone, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, - nullptr DEBUGARG("Span.get_Item ptrToSpan")); - - // Bounds check - CORINFO_FIELD_HANDLE lengthHnd = info.compCompHnd->getFieldInClass(clsHnd, 1); - const unsigned lengthOffset = info.compCompHnd->getFieldOffset(lengthHnd); - GenTree* length = gtNewFieldRef(TYP_INT, lengthHnd, ptrToSpan, lengthOffset); - GenTree* boundsCheck = new (this, GT_BOUNDS_CHECK) GenTreeBoundsChk(index, length, SCK_RNGCHK_FAIL); - - // Element access - index = indexClone; + GenTree* const treeToBox = impStackTop().val; + bool canOptimize = true; + GenTree* treeToNullcheck = nullptr; -#ifdef TARGET_64BIT - if (index->OperGet() == GT_CNS_INT) - { - index->gtType = TYP_I_IMPL; - } - else + // Can the thing being boxed cause a side effect? + if ((treeToBox->gtFlags & GTF_SIDE_EFFECT) != 0) { - index = gtNewCastNode(TYP_I_IMPL, index, true, TYP_I_IMPL); - } -#endif + // Is this a side effect we can replicate cheaply? + if (((treeToBox->gtFlags & GTF_SIDE_EFFECT) == GTF_EXCEPT) && + treeToBox->OperIs(GT_OBJ, GT_BLK, GT_IND)) + { + // If the only side effect comes from the dereference itself, yes. + GenTree* const addr = treeToBox->AsOp()->gtGetOp1(); - if (elemSize != 1) - { - GenTree* sizeofNode = gtNewIconNode(static_cast(elemSize), TYP_I_IMPL); - index = gtNewOperNode(GT_MUL, TYP_I_IMPL, index, sizeofNode); + if ((addr->gtFlags & GTF_SIDE_EFFECT) != 0) + { + canOptimize = false; + } + else if (fgAddrCouldBeNull(addr)) + { + treeToNullcheck = addr; + } + } + else + { + canOptimize = false; + } } - CORINFO_FIELD_HANDLE ptrHnd = info.compCompHnd->getFieldInClass(clsHnd, 0); - const unsigned ptrOffset = info.compCompHnd->getFieldOffset(ptrHnd); - GenTree* data = gtNewFieldRef(TYP_BYREF, ptrHnd, ptrToSpanClone, ptrOffset); - GenTree* result = gtNewOperNode(GT_ADD, TYP_BYREF, data, index); - - // Prepare result - var_types resultType = JITtype2varType(sig->retType); - assert(resultType == result->TypeGet()); - retNode = gtNewOperNode(GT_COMMA, resultType, boundsCheck, result); - - break; - } - - case NI_System_RuntimeTypeHandle_GetValueInternal: - { - GenTree* op1 = impStackTop(0).val; - if (op1->gtOper == GT_CALL && (op1->AsCall()->gtCallType == CT_HELPER) && - gtIsTypeHandleToRuntimeTypeHandleHelper(op1->AsCall())) + if (canOptimize) { - // Old tree - // Helper-RuntimeTypeHandle -> TreeToGetNativeTypeHandle - // - // New tree - // TreeToGetNativeTypeHandle + if ((opts == BoxPatterns::IsByRefLike) || + info.compCompHnd->getBoxHelper(pResolvedToken->hClass) == CORINFO_HELP_BOX) + { + JITDUMP("\n Importing BOX; BR_TRUE/FALSE as %sconstant\n", + treeToNullcheck == nullptr ? "" : "nullcheck+"); + impPopStack(); - // Remove call to helper and return the native TypeHandle pointer that was the parameter - // to that helper. + GenTree* result = gtNewIconNode(1); - op1 = impPopStack().val; - - // Get native TypeHandle argument to old helper - assert(op1->AsCall()->gtArgs.CountArgs() == 1); - op1 = op1->AsCall()->gtArgs.GetArgByIndex(0)->GetNode(); - retNode = op1; - } - // Call the regular function. - break; - } + if (treeToNullcheck != nullptr) + { + GenTree* nullcheck = gtNewNullCheck(treeToNullcheck, compCurBB); + result = gtNewOperNode(GT_COMMA, TYP_INT, nullcheck, result); + } - case NI_System_Type_GetTypeFromHandle: - { - GenTree* op1 = impStackTop(0).val; - CorInfoHelpFunc typeHandleHelper; - if (op1->gtOper == GT_CALL && (op1->AsCall()->gtCallType == CT_HELPER) && - gtIsTypeHandleToRuntimeTypeHandleHelper(op1->AsCall(), &typeHandleHelper)) - { - op1 = impPopStack().val; - // Replace helper with a more specialized helper that returns RuntimeType - if (typeHandleHelper == CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPEHANDLE) - { - typeHandleHelper = CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE; - } - else - { - assert(typeHandleHelper == CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPEHANDLE_MAYBENULL); - typeHandleHelper = CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE_MAYBENULL; + impPushOnStack(result, typeInfo(TI_INT)); + return 0; } - assert(op1->AsCall()->gtArgs.CountArgs() == 1); - op1 = gtNewHelperCallNode(typeHandleHelper, TYP_REF, - op1->AsCall()->gtArgs.GetArgByIndex(0)->GetEarlyNode()); - op1->gtType = TYP_REF; - retNode = op1; - } - break; - } - - case NI_System_Type_op_Equality: - case NI_System_Type_op_Inequality: - { - JITDUMP("Importing Type.op_*Equality intrinsic\n"); - GenTree* op1 = impStackTop(1).val; - GenTree* op2 = impStackTop(0).val; - GenTree* optTree = gtFoldTypeEqualityCall(ni == NI_System_Type_op_Equality, op1, op2); - if (optTree != nullptr) - { - // Success, clean up the evaluation stack. - impPopStack(); - impPopStack(); - - // See if we can optimize even further, to a handle compare. - optTree = gtFoldTypeCompare(optTree); - - // See if we can now fold a handle compare to a constant. - optTree = gtFoldExpr(optTree); - - retNode = optTree; - } - else - { - // Retry optimizing these later - isSpecial = true; } - break; - } - - case NI_System_Enum_HasFlag: - { - GenTree* thisOp = impStackTop(1).val; - GenTree* flagOp = impStackTop(0).val; - GenTree* optTree = gtOptimizeEnumHasFlag(thisOp, flagOp); - - if (optTree != nullptr) - { - // Optimization successful. Pop the stack for real. - impPopStack(); - impPopStack(); - retNode = optTree; - } - else - { - // Retry optimizing this during morph. - isSpecial = true; - } - - break; - } - - case NI_System_Type_IsAssignableFrom: - { - GenTree* typeTo = impStackTop(1).val; - GenTree* typeFrom = impStackTop(0).val; - - retNode = impTypeIsAssignable(typeTo, typeFrom); - break; } + break; - case NI_System_Type_IsAssignableTo: + case CEE_ISINST: + if (codeAddr + 1 + sizeof(mdToken) + 1 <= codeEndp) { - GenTree* typeTo = impStackTop(0).val; - GenTree* typeFrom = impStackTop(1).val; - - retNode = impTypeIsAssignable(typeTo, typeFrom); - break; - } + const BYTE* nextCodeAddr = codeAddr + 1 + sizeof(mdToken); - case NI_System_Type_get_IsValueType: - case NI_System_Type_get_IsByRefLike: - { - // Optimize - // - // call Type.GetTypeFromHandle (which is replaced with CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE) - // call Type.IsXXX - // - // to `true` or `false` - // e.g., `typeof(int).IsValueType` => `true` - // e.g., `typeof(Span).IsByRefLike` => `true` - if (impStackTop().val->IsCall()) + switch (nextCodeAddr[0]) { - GenTreeCall* call = impStackTop().val->AsCall(); - if (call->gtCallMethHnd == eeFindHelper(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE)) - { - assert(call->gtArgs.CountArgs() == 1); - CORINFO_CLASS_HANDLE hClass = - gtGetHelperArgClassHandle(call->gtArgs.GetArgByIndex(0)->GetEarlyNode()); - if (hClass != NO_CLASS_HANDLE) + // box + isinst + br_true/false + case CEE_BRTRUE: + case CEE_BRTRUE_S: + case CEE_BRFALSE: + case CEE_BRFALSE_S: + if ((nextCodeAddr + ((nextCodeAddr[0] >= CEE_BRFALSE) ? 5 : 2)) <= codeEndp) { - switch (ni) + if (opts == BoxPatterns::MakeInlineObservation) { - case NI_System_Type_get_IsValueType: - retNode = gtNewIconNode( - (eeIsValueClass(hClass) && - // pointers are not value types (e.g. typeof(int*).IsValueType is false) - info.compCompHnd->asCorInfoType(hClass) != CORINFO_TYPE_PTR) - ? 1 - : 0); - break; - case NI_System_Type_get_IsByRefLike: - retNode = gtNewIconNode( - (info.compCompHnd->getClassAttribs(hClass) & CORINFO_FLG_BYREF_LIKE) ? 1 : 0); - break; - default: - NO_WAY("Intrinsic not supported in this path."); + compInlineResult->Note(InlineObservation::CALLEE_FOLDABLE_BOX); + return 1 + sizeof(mdToken); } - impPopStack(); // drop CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE call - } - } - } - break; - } - - case NI_System_Threading_Thread_get_ManagedThreadId: - { - if (impStackTop().val->OperIs(GT_RET_EXPR)) - { - GenTreeCall* call = impStackTop().val->AsRetExpr()->gtInlineCandidate->AsCall(); - if (call->gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC) - { - if (lookupNamedIntrinsic(call->gtCallMethHnd) == NI_System_Threading_Thread_get_CurrentThread) - { - // drop get_CurrentThread() call - impPopStack(); - call->ReplaceWith(gtNewNothingNode(), this); - retNode = gtNewHelperCallNode(CORINFO_HELP_GETCURRENTMANAGEDTHREADID, TYP_INT); - } - } - } - break; - } - -#ifdef TARGET_ARM64 - // Intrinsify Interlocked.Or and Interlocked.And only for arm64-v8.1 (and newer) - // TODO-CQ: Implement for XArch (https://github.com/dotnet/runtime/issues/32239). - case NI_System_Threading_Interlocked_Or: - case NI_System_Threading_Interlocked_And: - { - if (compOpportunisticallyDependsOn(InstructionSet_Atomics)) - { - assert(sig->numArgs == 2); - GenTree* op2 = impPopStack().val; - GenTree* op1 = impPopStack().val; - genTreeOps op = (ni == NI_System_Threading_Interlocked_Or) ? GT_XORR : GT_XAND; - retNode = gtNewOperNode(op, genActualType(callType), op1, op2); - retNode->gtFlags |= GTF_GLOB_REF | GTF_ASG; - } - break; - } -#endif // TARGET_ARM64 - -#if defined(TARGET_XARCH) || defined(TARGET_ARM64) - // TODO-ARM-CQ: reenable treating InterlockedCmpXchg32 operation as intrinsic - case NI_System_Threading_Interlocked_CompareExchange: - { - var_types retType = JITtype2varType(sig->retType); - if ((retType == TYP_LONG) && (TARGET_POINTER_SIZE == 4)) - { - break; - } - if ((retType != TYP_INT) && (retType != TYP_LONG)) - { - break; - } - assert(callType != TYP_STRUCT); - assert(sig->numArgs == 3); - - GenTree* op3 = impPopStack().val; // comparand - GenTree* op2 = impPopStack().val; // value - GenTree* op1 = impPopStack().val; // location + if ((impStackTop().val->gtFlags & GTF_SIDE_EFFECT) == 0) + { + CorInfoHelpFunc foldAsHelper; + if (opts == BoxPatterns::IsByRefLike) + { + // Treat ByRefLike types as if they were regular boxing operations + // so they can be elided. + foldAsHelper = CORINFO_HELP_BOX; + } + else + { + foldAsHelper = info.compCompHnd->getBoxHelper(pResolvedToken->hClass); + } - GenTree* node = new (this, GT_CMPXCHG) GenTreeCmpXchg(genActualType(callType), op1, op2, op3); + if (foldAsHelper == CORINFO_HELP_BOX) + { + CORINFO_RESOLVED_TOKEN isInstResolvedToken; - node->AsCmpXchg()->gtOpLocation->gtFlags |= GTF_DONT_CSE; - retNode = node; - break; - } + impResolveToken(codeAddr + 1, &isInstResolvedToken, CORINFO_TOKENKIND_Casting); - case NI_System_Threading_Interlocked_Exchange: - case NI_System_Threading_Interlocked_ExchangeAdd: - { - assert(callType != TYP_STRUCT); - assert(sig->numArgs == 2); + TypeCompareState castResult = + info.compCompHnd->compareTypesForCast(pResolvedToken->hClass, + isInstResolvedToken.hClass); + if (castResult != TypeCompareState::May) + { + JITDUMP("\n Importing BOX; ISINST; BR_TRUE/FALSE as constant\n"); + impPopStack(); - var_types retType = JITtype2varType(sig->retType); - if ((retType == TYP_LONG) && (TARGET_POINTER_SIZE == 4)) - { - break; - } - if ((retType != TYP_INT) && (retType != TYP_LONG)) - { - break; - } + impPushOnStack(gtNewIconNode((castResult == TypeCompareState::Must) ? 1 : 0), + typeInfo(TI_INT)); - GenTree* op2 = impPopStack().val; - GenTree* op1 = impPopStack().val; + // Skip the next isinst instruction + return 1 + sizeof(mdToken); + } + } + else if (foldAsHelper == CORINFO_HELP_BOX_NULLABLE) + { + // For nullable we're going to fold it to "ldfld hasValue + brtrue/brfalse" or + // "ldc.i4.0 + brtrue/brfalse" in case if the underlying type is not castable to + // the target type. + CORINFO_RESOLVED_TOKEN isInstResolvedToken; + impResolveToken(codeAddr + 1, &isInstResolvedToken, CORINFO_TOKENKIND_Casting); - // This creates: - // val - // XAdd - // addr - // field (for example) - // - // In the case where the first argument is the address of a local, we might - // want to make this *not* make the var address-taken -- but atomic instructions - // on a local are probably pretty useless anyway, so we probably don't care. - - op1 = gtNewOperNode(ni == NI_System_Threading_Interlocked_ExchangeAdd ? GT_XADD : GT_XCHG, - genActualType(callType), op1, op2); - op1->gtFlags |= GTF_GLOB_REF | GTF_ASG; - retNode = op1; - break; - } -#endif // defined(TARGET_XARCH) || defined(TARGET_ARM64) + CORINFO_CLASS_HANDLE nullableCls = pResolvedToken->hClass; + CORINFO_CLASS_HANDLE underlyingCls = info.compCompHnd->getTypeForBox(nullableCls); - case NI_System_Threading_Interlocked_MemoryBarrier: - case NI_System_Threading_Interlocked_ReadMemoryBarrier: - { - assert(sig->numArgs == 0); + TypeCompareState castResult = + info.compCompHnd->compareTypesForCast(underlyingCls, + isInstResolvedToken.hClass); - GenTree* op1 = new (this, GT_MEMORYBARRIER) GenTree(GT_MEMORYBARRIER, TYP_VOID); - op1->gtFlags |= GTF_GLOB_REF | GTF_ASG; + if (castResult == TypeCompareState::Must) + { + const CORINFO_FIELD_HANDLE hasValueFldHnd = + info.compCompHnd->getFieldInClass(nullableCls, 0); - // On XARCH `NI_System_Threading_Interlocked_ReadMemoryBarrier` fences need not be emitted. - // However, we still need to capture the effect on reordering. - if (ni == NI_System_Threading_Interlocked_ReadMemoryBarrier) - { - op1->gtFlags |= GTF_MEMORYBARRIER_LOAD; - } + assert(info.compCompHnd->getFieldOffset(hasValueFldHnd) == 0); - retNode = op1; - break; - } + GenTree* objToBox = impPopStack().val; -#ifdef FEATURE_HW_INTRINSICS - case NI_System_Math_FusedMultiplyAdd: - { -#ifdef TARGET_XARCH - if (compExactlyDependsOn(InstructionSet_FMA)) - { - assert(varTypeIsFloating(callType)); - - // We are constructing a chain of intrinsics similar to: - // return FMA.MultiplyAddScalar( - // Vector128.CreateScalarUnsafe(x), - // Vector128.CreateScalarUnsafe(y), - // Vector128.CreateScalarUnsafe(z) - // ).ToScalar(); - - GenTree* op3 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, impPopStack().val, - NI_Vector128_CreateScalarUnsafe, callJitType, 16); - GenTree* op2 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, impPopStack().val, - NI_Vector128_CreateScalarUnsafe, callJitType, 16); - GenTree* op1 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, impPopStack().val, - NI_Vector128_CreateScalarUnsafe, callJitType, 16); - GenTree* res = - gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, op3, NI_FMA_MultiplyAddScalar, callJitType, 16); - - retNode = gtNewSimdHWIntrinsicNode(callType, res, NI_Vector128_ToScalar, callJitType, 16); - break; - } -#elif defined(TARGET_ARM64) - if (compExactlyDependsOn(InstructionSet_AdvSimd)) - { - assert(varTypeIsFloating(callType)); + // Spill struct to get its address (to access hasValue field) + objToBox = impGetStructAddr(objToBox, nullableCls, CHECK_SPILL_ALL, true); - // We are constructing a chain of intrinsics similar to: - // return AdvSimd.FusedMultiplyAddScalar( - // Vector64.Create{ScalarUnsafe}(z), - // Vector64.Create{ScalarUnsafe}(y), - // Vector64.Create{ScalarUnsafe}(x) - // ).ToScalar(); + impPushOnStack(gtNewFieldRef(TYP_BOOL, hasValueFldHnd, objToBox, 0), + typeInfo(TI_INT)); - NamedIntrinsic createVector64 = - (callType == TYP_DOUBLE) ? NI_Vector64_Create : NI_Vector64_CreateScalarUnsafe; + JITDUMP("\n Importing BOX; ISINST; BR_TRUE/FALSE as nullableVT.hasValue\n"); + return 1 + sizeof(mdToken); + } + else if (castResult == TypeCompareState::MustNot) + { + impPopStack(); + impPushOnStack(gtNewIconNode(0), typeInfo(TI_INT)); + JITDUMP("\n Importing BOX; ISINST; BR_TRUE/FALSE as constant (false)\n"); + return 1 + sizeof(mdToken); + } + } + } + } + break; - constexpr unsigned int simdSize = 8; + // box + isinst + unbox.any + case CEE_UNBOX_ANY: + if ((nextCodeAddr + 1 + sizeof(mdToken)) <= codeEndp) + { + if (opts == BoxPatterns::MakeInlineObservation) + { + compInlineResult->Note(InlineObservation::CALLEE_FOLDABLE_BOX); + return 2 + sizeof(mdToken) * 2; + } - GenTree* op3 = - gtNewSimdHWIntrinsicNode(TYP_SIMD8, impPopStack().val, createVector64, callJitType, simdSize); - GenTree* op2 = - gtNewSimdHWIntrinsicNode(TYP_SIMD8, impPopStack().val, createVector64, callJitType, simdSize); - GenTree* op1 = - gtNewSimdHWIntrinsicNode(TYP_SIMD8, impPopStack().val, createVector64, callJitType, simdSize); + // See if the resolved tokens in box, isinst and unbox.any describe types that are equal. + CORINFO_RESOLVED_TOKEN isinstResolvedToken = {}; + impResolveToken(codeAddr + 1, &isinstResolvedToken, CORINFO_TOKENKIND_Class); - // Note that AdvSimd.FusedMultiplyAddScalar(op1,op2,op3) corresponds to op1 + op2 * op3 - // while Math{F}.FusedMultiplyAddScalar(op1,op2,op3) corresponds to op1 * op2 + op3 - retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op3, op2, op1, NI_AdvSimd_FusedMultiplyAddScalar, - callJitType, simdSize); + if (info.compCompHnd->compareTypesForEquality(isinstResolvedToken.hClass, + pResolvedToken->hClass) == + TypeCompareState::Must) + { + CORINFO_RESOLVED_TOKEN unboxResolvedToken = {}; + impResolveToken(nextCodeAddr + 1, &unboxResolvedToken, CORINFO_TOKENKIND_Class); - retNode = gtNewSimdHWIntrinsicNode(callType, retNode, NI_Vector64_ToScalar, callJitType, simdSize); - break; + // If so, box + isinst + unbox.any is a nop. + if (info.compCompHnd->compareTypesForEquality(unboxResolvedToken.hClass, + pResolvedToken->hClass) == + TypeCompareState::Must) + { + JITDUMP("\n Importing BOX; ISINST, UNBOX.ANY as NOP\n"); + return 2 + sizeof(mdToken) * 2; + } + } + } + break; } -#endif - - // TODO-CQ-XArch: Ideally we would create a GT_INTRINSIC node for fma, however, that currently - // requires more extensive changes to valuenum to support methods with 3 operands - - // We want to generate a GT_INTRINSIC node in the case the call can't be treated as - // a target intrinsic so that we can still benefit from CSE and constant folding. - - break; - } -#endif // FEATURE_HW_INTRINSICS - - case NI_System_Math_Abs: - case NI_System_Math_Acos: - case NI_System_Math_Acosh: - case NI_System_Math_Asin: - case NI_System_Math_Asinh: - case NI_System_Math_Atan: - case NI_System_Math_Atanh: - case NI_System_Math_Atan2: - case NI_System_Math_Cbrt: - case NI_System_Math_Ceiling: - case NI_System_Math_Cos: - case NI_System_Math_Cosh: - case NI_System_Math_Exp: - case NI_System_Math_Floor: - case NI_System_Math_FMod: - case NI_System_Math_ILogB: - case NI_System_Math_Log: - case NI_System_Math_Log2: - case NI_System_Math_Log10: - { - retNode = impMathIntrinsic(method, sig, callType, ni, tailCall); - break; } -#if defined(TARGET_ARM64) - // ARM64 has fmax/fmin which are IEEE754:2019 minimum/maximum compatible - // TODO-XARCH-CQ: Enable this for XARCH when one of the arguments is a constant - // so we can then emit maxss/minss and avoid NaN/-0.0 handling - case NI_System_Math_Max: - case NI_System_Math_Min: -#endif - -#if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_XARCH) - case NI_System_Math_Max: - case NI_System_Math_Min: - { - assert(varTypeIsFloating(callType)); - assert(sig->numArgs == 2); - - GenTreeDblCon* cnsNode = nullptr; - GenTree* otherNode = nullptr; - - GenTree* op2 = impStackTop().val; - GenTree* op1 = impStackTop(1).val; - - if (op2->IsCnsFltOrDbl()) - { - cnsNode = op2->AsDblCon(); - otherNode = op1; - } - else if (op1->IsCnsFltOrDbl()) - { - cnsNode = op1->AsDblCon(); - otherNode = op2; - } - - if (cnsNode == nullptr) - { - // no constant node, nothing to do - break; - } - - if (otherNode->IsCnsFltOrDbl()) - { - // both are constant, we can fold this operation completely. Pop both peeked values - - if (ni == NI_System_Math_Max) - { - cnsNode->gtDconVal = - FloatingPointUtils::maximum(cnsNode->gtDconVal, otherNode->AsDblCon()->gtDconVal); - } - else - { - assert(ni == NI_System_Math_Min); - cnsNode->gtDconVal = - FloatingPointUtils::minimum(cnsNode->gtDconVal, otherNode->AsDblCon()->gtDconVal); - } + break; - retNode = cnsNode; + default: + break; + } - impPopStack(); - impPopStack(); - DEBUG_DESTROY_NODE(otherNode); + return -1; +} - break; - } - - // only one is constant, we can fold in specialized scenarios - - if (cnsNode->IsFloatNaN()) - { - impSpillSideEffects(false, (unsigned)CHECK_SPILL_ALL DEBUGARG( - "spill side effects before propagating NaN")); +//------------------------------------------------------------------------ +// impImportAndPushBox: build and import a value-type box +// +// Arguments: +// pResolvedToken - resolved token from the box operation +// +// Return Value: +// None. +// +// Side Effects: +// The value to be boxed is popped from the stack, and a tree for +// the boxed value is pushed. This method may create upstream +// statements, spill side effecting trees, and create new temps. +// +// If importing an inlinee, we may also discover the inline must +// fail. If so there is no new value pushed on the stack. Callers +// should use CompDoNotInline after calling this method to see if +// ongoing importation should be aborted. +// +// Notes: +// Boxing of ref classes results in the same value as the value on +// the top of the stack, so is handled inline in impImportBlockCode +// for the CEE_BOX case. Only value or primitive type boxes make it +// here. +// +// Boxing for nullable types is done via a helper call; boxing +// of other value types is expanded inline or handled via helper +// call, depending on the jit's codegen mode. +// +// When the jit is operating in size and time constrained modes, +// using a helper call here can save jit time and code size. But it +// also may inhibit cleanup optimizations that could have also had a +// even greater benefit effect on code size and jit time. An optimal +// strategy may need to peek ahead and see if it is easy to tell how +// the box is being used. For now, we defer. - // maxsd, maxss, minsd, and minss all return op2 if either is NaN - // we require NaN to be propagated so ensure the known NaN is op2 +void Compiler::impImportAndPushBox(CORINFO_RESOLVED_TOKEN* pResolvedToken) +{ + // Spill any special side effects + impSpillSpecialSideEff(); - impPopStack(); - impPopStack(); - DEBUG_DESTROY_NODE(otherNode); + // Get get the expression to box from the stack. + GenTree* op1 = nullptr; + GenTree* op2 = nullptr; + StackEntry se = impPopStack(); + CORINFO_CLASS_HANDLE operCls = se.seTypeInfo.GetClassHandle(); + GenTree* exprToBox = se.val; - retNode = cnsNode; - break; - } + // Look at what helper we should use. + CorInfoHelpFunc boxHelper = info.compCompHnd->getBoxHelper(pResolvedToken->hClass); - if (ni == NI_System_Math_Max) - { - // maxsd, maxss return op2 if both inputs are 0 of either sign - // we require +0 to be greater than -0, so we can't handle if - // the known constant is +0. This is because if the unknown value - // is -0, we'd need the cns to be op2. But if the unknown value - // is NaN, we'd need the cns to be op1 instead. + // Determine what expansion to prefer. + // + // In size/time/debuggable constrained modes, the helper call + // expansion for box is generally smaller and is preferred, unless + // the value to box is a struct that comes from a call. In that + // case the call can construct its return value directly into the + // box payload, saving possibly some up-front zeroing. + // + // Currently primitive type boxes always get inline expanded. We may + // want to do the same for small structs if they don't come from + // calls and don't have GC pointers, since explicitly copying such + // structs is cheap. + JITDUMP("\nCompiler::impImportAndPushBox -- handling BOX(value class) via"); + bool canExpandInline = (boxHelper == CORINFO_HELP_BOX); + bool optForSize = !exprToBox->IsCall() && (operCls != nullptr) && opts.OptimizationDisabled(); + bool expandInline = canExpandInline && !optForSize; - if (cnsNode->IsFloatPositiveZero()) - { - break; - } + if (expandInline) + { + JITDUMP(" inline allocate/copy sequence\n"); - // Given the checks, op1 can safely be the cns and op2 the other node + // we are doing 'normal' boxing. This means that we can inline the box operation + // Box(expr) gets morphed into + // temp = new(clsHnd) + // cpobj(temp+4, expr, clsHnd) + // push temp + // The code paths differ slightly below for structs and primitives because + // "cpobj" differs in these cases. In one case you get + // impAssignStructPtr(temp+4, expr, clsHnd) + // and the other you get + // *(temp+4) = expr - ni = (callType == TYP_DOUBLE) ? NI_SSE2_Max : NI_SSE_Max; + if (opts.OptimizationDisabled()) + { + // For minopts/debug code, try and minimize the total number + // of box temps by reusing an existing temp when possible. + if (impBoxTempInUse || impBoxTemp == BAD_VAR_NUM) + { + impBoxTemp = lvaGrabTemp(true DEBUGARG("Reusable Box Helper")); + } + } + else + { + // When optimizing, use a new temp for each box operation + // since we then know the exact class of the box temp. + impBoxTemp = lvaGrabTemp(true DEBUGARG("Single-def Box Helper")); + lvaTable[impBoxTemp].lvType = TYP_REF; + lvaTable[impBoxTemp].lvSingleDef = 1; + JITDUMP("Marking V%02u as a single def local\n", impBoxTemp); + const bool isExact = true; + lvaSetClass(impBoxTemp, pResolvedToken->hClass, isExact); + } - // one is constant and we know its something we can handle, so pop both peeked values + // needs to stay in use until this box expression is appended + // some other node. We approximate this by keeping it alive until + // the opcode stack becomes empty + impBoxTempInUse = true; - op1 = cnsNode; - op2 = otherNode; - } - else - { - assert(ni == NI_System_Math_Min); + // Remember the current last statement in case we need to move + // a range of statements to ensure the box temp is initialized + // before it's used. + // + Statement* const cursor = impLastStmt; - // minsd, minss return op2 if both inputs are 0 of either sign - // we require -0 to be lesser than +0, so we can't handle if - // the known constant is -0. This is because if the unknown value - // is +0, we'd need the cns to be op2. But if the unknown value - // is NaN, we'd need the cns to be op1 instead. + const bool useParent = false; + op1 = gtNewAllocObjNode(pResolvedToken, useParent); + if (op1 == nullptr) + { + // If we fail to create the newobj node, we must be inlining + // and have run across a type we can't describe. + // + assert(compDonotInline()); + return; + } - if (cnsNode->IsFloatNegativeZero()) - { - break; - } + // Remember that this basic block contains 'new' of an object, + // and so does this method + // + compCurBB->bbFlags |= BBF_HAS_NEWOBJ; + optMethodFlags |= OMF_HAS_NEWOBJ; - // Given the checks, op1 can safely be the cns and op2 the other node + // Assign the boxed object to the box temp. + // + GenTree* asg = gtNewTempAssign(impBoxTemp, op1); + Statement* asgStmt = impAppendTree(asg, CHECK_SPILL_NONE, impCurStmtDI); - ni = (callType == TYP_DOUBLE) ? NI_SSE2_Min : NI_SSE_Min; + // If the exprToBox is a call that returns its value via a ret buf arg, + // move the assignment statement(s) before the call (which must be a top level tree). + // + // We do this because impAssignStructPtr (invoked below) will + // back-substitute into a call when it sees a GT_RET_EXPR and the call + // has a hidden buffer pointer, So we need to reorder things to avoid + // creating out-of-sequence IR. + // + if (varTypeIsStruct(exprToBox) && exprToBox->OperIs(GT_RET_EXPR)) + { + GenTreeCall* const call = exprToBox->AsRetExpr()->gtInlineCandidate->AsCall(); - // one is constant and we know its something we can handle, so pop both peeked values + if (call->ShouldHaveRetBufArg()) + { + JITDUMP("Must insert newobj stmts for box before call [%06u]\n", dspTreeID(call)); - op1 = cnsNode; - op2 = otherNode; - } + // Walk back through the statements in this block, looking for the one + // that has this call as the root node. + // + // Because gtNewTempAssign (above) may have added statements that + // feed into the actual assignment we need to move this set of added + // statements as a group. + // + // Note boxed allocations are side-effect free (no com or finalizer) so + // our only worries here are (correctness) not overlapping the box temp + // lifetime and (perf) stretching the temp lifetime across the inlinee + // body. + // + // Since this is an inline candidate, we must be optimizing, and so we have + // a unique box temp per call. So no worries about overlap. + // + assert(!opts.OptimizationDisabled()); - assert(op1->IsCnsFltOrDbl() && !op2->IsCnsFltOrDbl()); + // Lifetime stretching could addressed with some extra cleverness--sinking + // the allocation back down to just before the copy, once we figure out + // where the copy is. We defer for now. + // + Statement* insertBeforeStmt = cursor; + noway_assert(insertBeforeStmt != nullptr); - impPopStack(); - impPopStack(); + while (true) + { + if (insertBeforeStmt->GetRootNode() == call) + { + break; + } - GenTreeVecCon* vecCon = gtNewVconNode(TYP_SIMD16, callJitType); + // If we've searched all the statements in the block and failed to + // find the call, then something's wrong. + // + noway_assert(insertBeforeStmt != impStmtList); - if (callJitType == CORINFO_TYPE_FLOAT) - { - vecCon->gtSimd16Val.f32[0] = (float)op1->AsDblCon()->gtDconVal; - } - else - { - vecCon->gtSimd16Val.f64[0] = op1->AsDblCon()->gtDconVal; + insertBeforeStmt = insertBeforeStmt->GetPrevStmt(); } - op1 = vecCon; - op2 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, NI_Vector128_CreateScalarUnsafe, callJitType, 16); + // Found the call. Move the statements comprising the assignment. + // + JITDUMP("Moving " FMT_STMT "..." FMT_STMT " before " FMT_STMT "\n", cursor->GetNextStmt()->GetID(), + asgStmt->GetID(), insertBeforeStmt->GetID()); + assert(asgStmt == impLastStmt); + do + { + Statement* movingStmt = impExtractLastStmt(); + impInsertStmtBefore(movingStmt, insertBeforeStmt); + insertBeforeStmt = movingStmt; + } while (impLastStmt != cursor); + } + } - retNode = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, op2, ni, callJitType, 16); - retNode = gtNewSimdHWIntrinsicNode(callType, retNode, NI_Vector128_ToScalar, callJitType, 16); + // Create a pointer to the box payload in op1. + // + op1 = gtNewLclvNode(impBoxTemp, TYP_REF); + op2 = gtNewIconNode(TARGET_POINTER_SIZE, TYP_I_IMPL); + op1 = gtNewOperNode(GT_ADD, TYP_BYREF, op1, op2); - break; + // Copy from the exprToBox to the box payload. + // + if (varTypeIsStruct(exprToBox)) + { + assert(info.compCompHnd->getClassSize(pResolvedToken->hClass) == info.compCompHnd->getClassSize(operCls)); + op1 = impAssignStructPtr(op1, exprToBox, operCls, CHECK_SPILL_ALL); + } + else + { + var_types lclTyp = exprToBox->TypeGet(); + if (lclTyp == TYP_BYREF) + { + lclTyp = TYP_I_IMPL; } -#endif - - case NI_System_Math_Pow: - case NI_System_Math_Round: - case NI_System_Math_Sin: - case NI_System_Math_Sinh: - case NI_System_Math_Sqrt: - case NI_System_Math_Tan: - case NI_System_Math_Tanh: - case NI_System_Math_Truncate: + CorInfoType jitType = info.compCompHnd->asCorInfoType(pResolvedToken->hClass); + if (impIsPrimitive(jitType)) { - retNode = impMathIntrinsic(method, sig, callType, ni, tailCall); - break; + lclTyp = JITtype2varType(jitType); } - case NI_System_Array_Clone: - case NI_System_Collections_Generic_Comparer_get_Default: - case NI_System_Collections_Generic_EqualityComparer_get_Default: - case NI_System_Object_MemberwiseClone: - case NI_System_Threading_Thread_get_CurrentThread: + var_types srcTyp = exprToBox->TypeGet(); + var_types dstTyp = lclTyp; + + // We allow float <-> double mismatches and implicit truncation for small types. + assert((genActualType(srcTyp) == genActualType(dstTyp)) || + (varTypeIsFloating(srcTyp) == varTypeIsFloating(dstTyp))); + + // Note regarding small types. + // We are going to store to the box here via an indirection, so the cast added below is + // redundant, since the store has an implicit truncation semantic. The reason we still + // add this cast is so that the code which deals with GT_BOX optimizations does not have + // to account for this implicit truncation (e. g. understand that BOX(0xFF + 1) is + // actually BOX(0) or deal with signedness mismatch and other GT_CAST complexities). + if (srcTyp != dstTyp) { - // Flag for later handling. - isSpecial = true; - break; + exprToBox = gtNewCastNode(genActualType(dstTyp), exprToBox, false, dstTyp); } - case NI_System_Object_GetType: - { - JITDUMP("\n impIntrinsic: call to Object.GetType\n"); - GenTree* op1 = impStackTop(0).val; + op1 = gtNewAssignNode(gtNewOperNode(GT_IND, dstTyp, op1), exprToBox); + } - // If we're calling GetType on a boxed value, just get the type directly. - if (op1->IsBoxedValue()) - { - JITDUMP("Attempting to optimize box(...).getType() to direct type construction\n"); + // Spill eval stack to flush out any pending side effects. + impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("impImportAndPushBox")); - // Try and clean up the box. Obtain the handle we - // were going to pass to the newobj. - GenTree* boxTypeHandle = gtTryRemoveBoxUpstreamEffects(op1, BR_REMOVE_AND_NARROW_WANT_TYPE_HANDLE); + // Set up this copy as a second assignment. + Statement* copyStmt = impAppendTree(op1, CHECK_SPILL_NONE, impCurStmtDI); - if (boxTypeHandle != nullptr) - { - // Note we don't need to play the TYP_STRUCT games here like - // do for LDTOKEN since the return value of this operator is Type, - // not RuntimeTypeHandle. - impPopStack(); - GenTree* runtimeType = - gtNewHelperCallNode(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE, TYP_REF, boxTypeHandle); - retNode = runtimeType; - } - } + op1 = gtNewLclvNode(impBoxTemp, TYP_REF); - // If we have a constrained callvirt with a "box this" transform - // we know we have a value class and hence an exact type. - // - // If so, instead of boxing and then extracting the type, just - // construct the type directly. - if ((retNode == nullptr) && (pConstrainedResolvedToken != nullptr) && - (constraintCallThisTransform == CORINFO_BOX_THIS)) - { - // Ensure this is one of the simple box cases (in particular, rule out nullables). - const CorInfoHelpFunc boxHelper = info.compCompHnd->getBoxHelper(pConstrainedResolvedToken->hClass); - const bool isSafeToOptimize = (boxHelper == CORINFO_HELP_BOX); + // Record that this is a "box" node and keep track of the matching parts. + op1 = new (this, GT_BOX) GenTreeBox(TYP_REF, op1, asgStmt, copyStmt); - if (isSafeToOptimize) - { - JITDUMP("Optimizing constrained box-this obj.getType() to direct type construction\n"); - impPopStack(); - GenTree* typeHandleOp = - impTokenToHandle(pConstrainedResolvedToken, nullptr, true /* mustRestoreHandle */); - if (typeHandleOp == nullptr) - { - assert(compDonotInline()); - return nullptr; - } - GenTree* runtimeType = - gtNewHelperCallNode(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE, TYP_REF, typeHandleOp); - retNode = runtimeType; - } - } + // If it is a value class, mark the "box" node. We can use this information + // to optimise several cases: + // "box(x) == null" --> false + // "(box(x)).CallAnInterfaceMethod(...)" --> "(&x).CallAValueTypeMethod" + // "(box(x)).CallAnObjectMethod(...)" --> "(&x).CallAValueTypeMethod" -#ifdef DEBUG - if (retNode != nullptr) - { - JITDUMP("Optimized result for call to GetType is\n"); - if (verbose) - { - gtDispTree(retNode); - } - } -#endif + op1->gtFlags |= GTF_BOX_VALUE; + assert(op1->IsBoxedValue()); + assert(asg->gtOper == GT_ASG); + } + else + { + // Don't optimize, just call the helper and be done with it. + JITDUMP(" helper call because: %s\n", canExpandInline ? "optimizing for size" : "nullable"); + assert(operCls != nullptr); - // Else expand as an intrinsic, unless the call is constrained, - // in which case we defer expansion to allow impImportCall do the - // special constraint processing. - if ((retNode == nullptr) && (pConstrainedResolvedToken == nullptr)) - { - JITDUMP("Expanding as special intrinsic\n"); - impPopStack(); - op1 = new (this, GT_INTRINSIC) GenTreeIntrinsic(genActualType(callType), op1, ni, method); + // Ensure that the value class is restored + op2 = impTokenToHandle(pResolvedToken, nullptr, true /* mustRestoreHandle */); + if (op2 == nullptr) + { + // We must be backing out of an inline. + assert(compDonotInline()); + return; + } - // Set the CALL flag to indicate that the operator is implemented by a call. - // Set also the EXCEPTION flag because the native implementation of - // NI_System_Object_GetType intrinsic can throw NullReferenceException. - op1->gtFlags |= (GTF_CALL | GTF_EXCEPT); - retNode = op1; - // Might be further optimizable, so arrange to leave a mark behind - isSpecial = true; - } + op1 = gtNewHelperCallNode(boxHelper, TYP_REF, op2, impGetStructAddr(exprToBox, operCls, CHECK_SPILL_ALL, true)); + } - if (retNode == nullptr) - { - JITDUMP("Leaving as normal call\n"); - // Might be further optimizable, so arrange to leave a mark behind - isSpecial = true; - } + /* Push the result back on the stack, */ + /* even if clsHnd is a value class we want the TI_REF */ + typeInfo tiRetVal = typeInfo(TI_REF, info.compCompHnd->getTypeForBox(pResolvedToken->hClass)); + impPushOnStack(op1, tiRetVal); +} - break; - } +//------------------------------------------------------------------------ +// impImportNewObjArray: Build and import `new` of multi-dimensional array +// +// Arguments: +// pResolvedToken - The CORINFO_RESOLVED_TOKEN that has been initialized +// by a call to CEEInfo::resolveToken(). +// pCallInfo - The CORINFO_CALL_INFO that has been initialized +// by a call to CEEInfo::getCallInfo(). +// +// Assumptions: +// The multi-dimensional array constructor arguments (array dimensions) are +// pushed on the IL stack on entry to this method. +// +// Notes: +// Multi-dimensional array constructors are imported as calls to a JIT +// helper, not as regular calls. +// +void Compiler::impImportNewObjArray(CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_CALL_INFO* pCallInfo) +{ + GenTree* classHandle = impParentClassTokenToHandle(pResolvedToken); + if (classHandle == nullptr) + { // compDonotInline() + return; + } - case NI_System_Array_GetLength: - case NI_System_Array_GetLowerBound: - case NI_System_Array_GetUpperBound: - { - // System.Array.GetLength(Int32) method: - // public int GetLength(int dimension) - // System.Array.GetLowerBound(Int32) method: - // public int GetLowerBound(int dimension) - // System.Array.GetUpperBound(Int32) method: - // public int GetUpperBound(int dimension) - // - // Only implement these as intrinsics for multi-dimensional arrays. - // Only handle constant dimension arguments. + assert(pCallInfo->sig.numArgs); - GenTree* gtDim = impStackTop().val; - GenTree* gtArr = impStackTop(1).val; + GenTree* node; - if (gtDim->IsIntegralConst()) - { - bool isExact = false; - bool isNonNull = false; - CORINFO_CLASS_HANDLE arrCls = gtGetClassHandle(gtArr, &isExact, &isNonNull); - if (arrCls != NO_CLASS_HANDLE) - { - unsigned rank = info.compCompHnd->getArrayRank(arrCls); - if ((rank > 1) && !info.compCompHnd->isSDArray(arrCls)) - { - // `rank` is guaranteed to be <=32 (see MAX_RANK in vm\array.h). Any constant argument - // is `int` sized. - INT64 dimValue = gtDim->AsIntConCommon()->IntegralValue(); - assert((unsigned int)dimValue == dimValue); - unsigned dim = (unsigned int)dimValue; - if (dim < rank) - { - // This is now known to be a multi-dimension array with a constant dimension - // that is in range; we can expand it as an intrinsic. - - impPopStack().val; // Pop the dim and array object; we already have a pointer to them. - impPopStack().val; - - // Make sure there are no global effects in the array (such as it being a function - // call), so we can mark the generated indirection with GTF_IND_INVARIANT. In the - // GetUpperBound case we need the cloned object, since we refer to the array - // object twice. In the other cases, we don't need to clone. - GenTree* gtArrClone = nullptr; - if (((gtArr->gtFlags & GTF_GLOB_EFFECT) != 0) || (ni == NI_System_Array_GetUpperBound)) - { - gtArr = impCloneExpr(gtArr, >ArrClone, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, - nullptr DEBUGARG("MD intrinsics array")); - } + // Reuse the temp used to pass the array dimensions to avoid bloating + // the stack frame in case there are multiple calls to multi-dim array + // constructors within a single method. + if (lvaNewObjArrayArgs == BAD_VAR_NUM) + { + lvaNewObjArrayArgs = lvaGrabTemp(false DEBUGARG("NewObjArrayArgs")); + lvaTable[lvaNewObjArrayArgs].lvType = TYP_BLK; + lvaTable[lvaNewObjArrayArgs].lvExactSize = 0; + } - switch (ni) - { - case NI_System_Array_GetLength: - { - // Generate *(array + offset-to-length-array + sizeof(int) * dim) - unsigned offs = eeGetMDArrayLengthOffset(rank, dim); - GenTree* gtOffs = gtNewIconNode(offs, TYP_I_IMPL); - GenTree* gtAddr = gtNewOperNode(GT_ADD, TYP_BYREF, gtArr, gtOffs); - retNode = gtNewIndir(TYP_INT, gtAddr); - retNode->gtFlags |= GTF_IND_INVARIANT; - break; - } - case NI_System_Array_GetLowerBound: - { - // Generate *(array + offset-to-bounds-array + sizeof(int) * dim) - unsigned offs = eeGetMDArrayLowerBoundOffset(rank, dim); - GenTree* gtOffs = gtNewIconNode(offs, TYP_I_IMPL); - GenTree* gtAddr = gtNewOperNode(GT_ADD, TYP_BYREF, gtArr, gtOffs); - retNode = gtNewIndir(TYP_INT, gtAddr); - retNode->gtFlags |= GTF_IND_INVARIANT; - break; - } - case NI_System_Array_GetUpperBound: - { - assert(gtArrClone != nullptr); - - // Generate: - // *(array + offset-to-length-array + sizeof(int) * dim) + - // *(array + offset-to-bounds-array + sizeof(int) * dim) - 1 - unsigned offs = eeGetMDArrayLowerBoundOffset(rank, dim); - GenTree* gtOffs = gtNewIconNode(offs, TYP_I_IMPL); - GenTree* gtAddr = gtNewOperNode(GT_ADD, TYP_BYREF, gtArr, gtOffs); - GenTree* gtLowerBound = gtNewIndir(TYP_INT, gtAddr); - gtLowerBound->gtFlags |= GTF_IND_INVARIANT; - - offs = eeGetMDArrayLengthOffset(rank, dim); - gtOffs = gtNewIconNode(offs, TYP_I_IMPL); - gtAddr = gtNewOperNode(GT_ADD, TYP_BYREF, gtArrClone, gtOffs); - GenTree* gtLength = gtNewIndir(TYP_INT, gtAddr); - gtLength->gtFlags |= GTF_IND_INVARIANT; - - GenTree* gtSum = gtNewOperNode(GT_ADD, TYP_INT, gtLowerBound, gtLength); - GenTree* gtOne = gtNewIconNode(1, TYP_INT); - retNode = gtNewOperNode(GT_SUB, TYP_INT, gtSum, gtOne); - break; - } - default: - unreached(); - } - } - } - } - } - break; - } + // Increase size of lvaNewObjArrayArgs to be the largest size needed to hold 'numArgs' integers + // for our call to CORINFO_HELP_NEW_MDARR. + lvaTable[lvaNewObjArrayArgs].lvExactSize = + max(lvaTable[lvaNewObjArrayArgs].lvExactSize, pCallInfo->sig.numArgs * sizeof(INT32)); - case NI_System_Buffers_Binary_BinaryPrimitives_ReverseEndianness: - { - assert(sig->numArgs == 1); + // The side-effects may include allocation of more multi-dimensional arrays. Spill all side-effects + // to ensure that the shared lvaNewObjArrayArgs local variable is only ever used to pass arguments + // to one allocation at a time. + impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("impImportNewObjArray")); - // We expect the return type of the ReverseEndianness routine to match the type of the - // one and only argument to the method. We use a special instruction for 16-bit - // BSWAPs since on x86 processors this is implemented as ROR <16-bit reg>, 8. Additionally, - // we only emit 64-bit BSWAP instructions on 64-bit archs; if we're asked to perform a - // 64-bit byte swap on a 32-bit arch, we'll fall to the default case in the switch block below. + // + // The arguments of the CORINFO_HELP_NEW_MDARR helper are: + // - Array class handle + // - Number of dimension arguments + // - Pointer to block of int32 dimensions: address of lvaNewObjArrayArgs temp. + // - switch (sig->retType) - { - case CorInfoType::CORINFO_TYPE_SHORT: - case CorInfoType::CORINFO_TYPE_USHORT: - retNode = gtNewCastNode(TYP_INT, gtNewOperNode(GT_BSWAP16, TYP_INT, impPopStack().val), false, - callType); - break; + node = gtNewLclVarAddrNode(lvaNewObjArrayArgs); - case CorInfoType::CORINFO_TYPE_INT: - case CorInfoType::CORINFO_TYPE_UINT: -#ifdef TARGET_64BIT - case CorInfoType::CORINFO_TYPE_LONG: - case CorInfoType::CORINFO_TYPE_ULONG: -#endif // TARGET_64BIT - retNode = gtNewOperNode(GT_BSWAP, callType, impPopStack().val); - break; + // Pop dimension arguments from the stack one at a time and store it + // into lvaNewObjArrayArgs temp. + for (int i = pCallInfo->sig.numArgs - 1; i >= 0; i--) + { + GenTree* arg = impImplicitIorI4Cast(impPopStack().val, TYP_INT); + GenTree* dest = gtNewLclFldNode(lvaNewObjArrayArgs, TYP_INT, sizeof(INT32) * i); + node = gtNewOperNode(GT_COMMA, node->TypeGet(), gtNewAssignNode(dest, arg), node); + } - default: - // This default case gets hit on 32-bit archs when a call to a 64-bit overload - // of ReverseEndianness is encountered. In that case we'll let JIT treat this as a standard - // method call, where the implementation decomposes the operation into two 32-bit - // bswap routines. If the input to the 64-bit function is a constant, then we rely - // on inlining + constant folding of 32-bit bswaps to effectively constant fold - // the 64-bit call site. - break; - } + node = + gtNewHelperCallNode(CORINFO_HELP_NEW_MDARR, TYP_REF, classHandle, gtNewIconNode(pCallInfo->sig.numArgs), node); - break; - } + node->AsCall()->compileTimeHelperArgumentHandle = (CORINFO_GENERIC_HANDLE)pResolvedToken->hClass; - // Fold PopCount for constant input - case NI_System_Numerics_BitOperations_PopCount: - { - assert(sig->numArgs == 1); - if (impStackTop().val->IsIntegralConst()) - { - typeInfo argType = verParseArgSigToTypeInfo(sig, sig->args).NormaliseForStack(); - INT64 cns = impPopStack().val->AsIntConCommon()->IntegralValue(); - if (argType.IsType(TI_LONG)) - { - retNode = gtNewIconNode(genCountBits(cns), callType); - } - else - { - assert(argType.IsType(TI_INT)); - retNode = gtNewIconNode(genCountBits(static_cast(cns)), callType); - } - } - break; - } + // Remember that this function contains 'new' of a MD array. + optMethodFlags |= OMF_HAS_MDNEWARRAY; - case NI_System_GC_KeepAlive: - { - retNode = impKeepAliveIntrinsic(impPopStack().val); - break; - } + impPushOnStack(node, typeInfo(TI_REF, pResolvedToken->hClass)); +} - case NI_System_BitConverter_DoubleToInt64Bits: - { - GenTree* op1 = impStackTop().val; - assert(varTypeIsFloating(op1)); +//------------------------------------------------------------------------ +// impInitClass: Build a node to initialize the class before accessing the +// field if necessary +// +// Arguments: +// pResolvedToken - The CORINFO_RESOLVED_TOKEN that has been initialized +// by a call to CEEInfo::resolveToken(). +// +// Return Value: If needed, a pointer to the node that will perform the class +// initializtion. Otherwise, nullptr. +// - if (op1->IsCnsFltOrDbl()) - { - impPopStack(); +GenTree* Compiler::impInitClass(CORINFO_RESOLVED_TOKEN* pResolvedToken) +{ + CorInfoInitClassResult initClassResult = + info.compCompHnd->initClass(pResolvedToken->hField, info.compMethodHnd, impTokenLookupContextHandle); - double f64Cns = op1->AsDblCon()->gtDconVal; - retNode = gtNewLconNode(*reinterpret_cast(&f64Cns)); - } -#if TARGET_64BIT - else - { - // TODO-Cleanup: We should support this on 32-bit but it requires decomposition work - impPopStack(); + if ((initClassResult & CORINFO_INITCLASS_USE_HELPER) == 0) + { + return nullptr; + } + bool runtimeLookup; - if (op1->TypeGet() != TYP_DOUBLE) - { - op1 = gtNewCastNode(TYP_DOUBLE, op1, false, TYP_DOUBLE); - } - retNode = gtNewBitCastNode(TYP_LONG, op1); - } -#endif - break; - } + GenTree* node = impParentClassTokenToHandle(pResolvedToken, &runtimeLookup); - case NI_System_BitConverter_Int32BitsToSingle: - { - GenTree* op1 = impPopStack().val; - assert(varTypeIsInt(op1)); + if (node == nullptr) + { + assert(compDonotInline()); + return nullptr; + } - if (op1->IsIntegralConst()) - { - int32_t i32Cns = (int32_t)op1->AsIntConCommon()->IconValue(); - retNode = gtNewDconNode(*reinterpret_cast(&i32Cns), TYP_FLOAT); - } - else - { - retNode = gtNewBitCastNode(TYP_FLOAT, op1); - } - break; - } + if (runtimeLookup) + { + node = gtNewHelperCallNode(CORINFO_HELP_INITCLASS, TYP_VOID, node); + } + else + { + // Call the shared non gc static helper, as its the fastest + node = fgGetSharedCCtor(pResolvedToken->hClass); + } - case NI_System_BitConverter_Int64BitsToDouble: - { - GenTree* op1 = impStackTop().val; - assert(varTypeIsLong(op1)); + return node; +} - if (op1->IsIntegralConst()) - { - impPopStack(); +//------------------------------------------------------------------------ +// impImportStaticReadOnlyField: Tries to import 'static readonly' field +// as a constant if the host type is statically initialized. +// +// Arguments: +// field - 'static readonly' field +// ownerCls - class handle of the type the given field defined in +// +// Return Value: +// The tree representing the constant value of the statically initialized +// readonly tree. +// +GenTree* Compiler::impImportStaticReadOnlyField(CORINFO_FIELD_HANDLE field, CORINFO_CLASS_HANDLE ownerCls) +{ + if (!opts.OptimizationEnabled()) + { + return nullptr; + } - int64_t i64Cns = op1->AsIntConCommon()->LngValue(); - retNode = gtNewDconNode(*reinterpret_cast(&i64Cns)); - } -#if TARGET_64BIT - else - { - // TODO-Cleanup: We should support this on 32-bit but it requires decomposition work - impPopStack(); + JITDUMP("\nChecking if we can import 'static readonly' as a jit-time constant... ") - retNode = gtNewBitCastNode(TYP_DOUBLE, op1); - } -#endif - break; + CORINFO_CLASS_HANDLE fieldClsHnd; + var_types fieldType = JITtype2varType(info.compCompHnd->getFieldType(field, &fieldClsHnd, ownerCls)); + + const int bufferSize = sizeof(uint64_t); + uint8_t buffer[bufferSize] = {0}; + if (varTypeIsIntegral(fieldType) || varTypeIsFloating(fieldType) || (fieldType == TYP_REF)) + { + assert(bufferSize >= genTypeSize(fieldType)); + if (info.compCompHnd->getReadonlyStaticFieldValue(field, buffer, genTypeSize(fieldType))) + { + GenTree* cnsValue = impImportCnsTreeFromBuffer(buffer, fieldType); + if (cnsValue != nullptr) + { + JITDUMP("... success! The value is:\n"); + DISPTREE(cnsValue); + return cnsValue; } + } + } + else if (fieldType == TYP_STRUCT) + { + unsigned totalSize = info.compCompHnd->getClassSize(fieldClsHnd); + unsigned fieldsCnt = info.compCompHnd->getClassNumInstanceFields(fieldClsHnd); - case NI_System_BitConverter_SingleToInt32Bits: + // For large structs we only want to handle "initialized with zero" case + // e.g. Guid.Empty and decimal.Zero static readonly fields. + if ((totalSize > TARGET_POINTER_SIZE) || (fieldsCnt != 1)) + { + JITDUMP("checking if we can do anything for a large struct ..."); + const int MaxStructSize = 64; + if ((totalSize == 0) || (totalSize > MaxStructSize)) { - GenTree* op1 = impPopStack().val; - assert(varTypeIsFloating(op1)); + // Limit to 64 bytes for better throughput + JITDUMP("struct is larger than 64 bytes - bail out."); + return nullptr; + } - if (op1->IsCnsFltOrDbl()) - { - float f32Cns = (float)op1->AsDblCon()->gtDconVal; - retNode = gtNewIconNode(*reinterpret_cast(&f32Cns)); - } - else + uint8_t buffer[MaxStructSize] = {0}; + if (info.compCompHnd->getReadonlyStaticFieldValue(field, buffer, totalSize)) + { + for (unsigned i = 0; i < totalSize; i++) { - if (op1->TypeGet() != TYP_FLOAT) + if (buffer[i] != 0) { - op1 = gtNewCastNode(TYP_FLOAT, op1, false, TYP_FLOAT); + // Value is not all zeroes - bail out. + // Although, We might eventually support that too. + JITDUMP("value is not all zeros - bail out."); + return nullptr; } - retNode = gtNewBitCastNode(TYP_INT, op1); } - break; - } - default: - break; - } - } + JITDUMP("success! Optimizing to ASG(struct, 0)."); + unsigned structTempNum = lvaGrabTemp(true DEBUGARG("folding static ro fld empty struct")); + lvaSetStruct(structTempNum, fieldClsHnd, false); - if (mustExpand && (retNode == nullptr)) - { - assert(!"Unhandled must expand intrinsic, throwing PlatformNotSupportedException"); - return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED, method, sig, mustExpand); - } - - // Optionally report if this intrinsic is special - // (that is, potentially re-optimizable during morph). - if (isSpecialIntrinsic != nullptr) - { - *isSpecialIntrinsic = isSpecial; - } - - return retNode; -} - -GenTree* Compiler::impSRCSUnsafeIntrinsic(NamedIntrinsic intrinsic, - CORINFO_CLASS_HANDLE clsHnd, - CORINFO_METHOD_HANDLE method, - CORINFO_SIG_INFO* sig) -{ - assert(sig->sigInst.classInstCount == 0); - - switch (intrinsic) - { - case NI_SRCS_UNSAFE_Add: - { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // ldarg.1 - // sizeof !!T - // conv.i - // mul - // add - // ret - - GenTree* op2 = impPopStack().val; - GenTree* op1 = impPopStack().val; - impBashVarAddrsToI(op1, op2); - - op2 = impImplicitIorI4Cast(op2, TYP_I_IMPL); - - unsigned classSize = info.compCompHnd->getClassSize(sig->sigInst.methInst[0]); + // realType is either struct or SIMD + var_types realType = lvaGetRealType(structTempNum); + GenTreeLclVar* structLcl = gtNewLclvNode(structTempNum, realType); + impAppendTree(gtNewBlkOpNode(structLcl, gtNewIconNode(0)), CHECK_SPILL_NONE, impCurStmtDI); - if (classSize != 1) - { - GenTree* size = gtNewIconNode(classSize, TYP_I_IMPL); - op2 = gtNewOperNode(GT_MUL, TYP_I_IMPL, op2, size); + return gtNewLclvNode(structTempNum, realType); } - var_types type = impGetByRefResultType(GT_ADD, /* uns */ false, &op1, &op2); - return gtNewOperNode(GT_ADD, type, op1, op2); + JITDUMP("getReadonlyStaticFieldValue returned false - bail out."); + return nullptr; } - case NI_SRCS_UNSAFE_AddByteOffset: - { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // ldarg.1 - // add - // ret + // Only single-field structs are supported here to avoid potential regressions where + // Metadata-driven struct promotion leads to regressions. - GenTree* op2 = impPopStack().val; - GenTree* op1 = impPopStack().val; - impBashVarAddrsToI(op1, op2); + CORINFO_FIELD_HANDLE innerField = info.compCompHnd->getFieldInClass(fieldClsHnd, 0); + CORINFO_CLASS_HANDLE innerFieldClsHnd; + var_types fieldVarType = + JITtype2varType(info.compCompHnd->getFieldType(innerField, &innerFieldClsHnd, fieldClsHnd)); - var_types type = impGetByRefResultType(GT_ADD, /* uns */ false, &op1, &op2); - return gtNewOperNode(GT_ADD, type, op1, op2); + // Technically, we can support frozen gc refs here and maybe floating point in future + if (!varTypeIsIntegral(fieldVarType)) + { + JITDUMP("struct has non-primitive fields - bail out."); + return nullptr; } - case NI_SRCS_UNSAFE_AreSame: - { - assert(sig->sigInst.methInstCount == 1); + unsigned fldOffset = info.compCompHnd->getFieldOffset(innerField); - // ldarg.0 - // ldarg.1 - // ceq - // ret + if ((fldOffset != 0) || (totalSize != genTypeSize(fieldVarType)) || (totalSize == 0)) + { + // The field is expected to be of the exact size as the struct with 0 offset + JITDUMP("struct has complex layout - bail out."); + return nullptr; + } - GenTree* op2 = impPopStack().val; - GenTree* op1 = impPopStack().val; + const int bufferSize = TARGET_POINTER_SIZE; + uint8_t buffer[bufferSize] = {0}; - GenTree* tmp = gtNewOperNode(GT_EQ, TYP_INT, op1, op2); - return gtFoldExpr(tmp); + if ((totalSize > bufferSize) || !info.compCompHnd->getReadonlyStaticFieldValue(field, buffer, totalSize)) + { + return nullptr; } - case NI_SRCS_UNSAFE_As: - { - assert((sig->sigInst.methInstCount == 1) || (sig->sigInst.methInstCount == 2)); + unsigned structTempNum = lvaGrabTemp(true DEBUGARG("folding static ro fld struct")); + lvaSetStruct(structTempNum, fieldClsHnd, false); - // ldarg.0 - // ret + GenTree* constValTree = impImportCnsTreeFromBuffer(buffer, fieldVarType); + assert(constValTree != nullptr); - return impPopStack().val; - } + GenTreeLclFld* fieldTree = gtNewLclFldNode(structTempNum, fieldVarType, fldOffset); + GenTree* fieldAsgTree = gtNewAssignNode(fieldTree, constValTree); + impAppendTree(fieldAsgTree, CHECK_SPILL_NONE, impCurStmtDI); - case NI_SRCS_UNSAFE_AsPointer: - { - assert(sig->sigInst.methInstCount == 1); + JITDUMP("Folding 'static readonly %s' field to an ASG(LCL, CNS) node\n", eeGetClassName(fieldClsHnd)); - // ldarg.0 - // conv.u - // ret + return impCreateLocalNode(structTempNum DEBUGARG(0)); + } + return nullptr; +} - GenTree* op1 = impPopStack().val; - impBashVarAddrsToI(op1); +//------------------------------------------------------------------------ +// impImportCnsTreeFromBuffer: read value of the given type from the +// given buffer to create a tree representing that constant. +// +// Arguments: +// buffer - array of bytes representing the value +// valueType - type of the value +// +// Return Value: +// The tree representing the constant from the given buffer +// +GenTree* Compiler::impImportCnsTreeFromBuffer(uint8_t* buffer, var_types valueType) +{ + GenTree* tree = nullptr; + switch (valueType) + { +// Use memcpy to read from the buffer and create an Icon/Dcon tree +#define CreateTreeFromBuffer(type, treeFactory) \ + type v##type; \ + memcpy(&v##type, buffer, sizeof(type)); \ + tree = treeFactory(v##type); - return gtNewCastNode(TYP_I_IMPL, op1, /* uns */ false, TYP_I_IMPL); + case TYP_BOOL: + { + CreateTreeFromBuffer(bool, gtNewIconNode); + break; } - - case NI_SRCS_UNSAFE_AsRef: + case TYP_BYTE: { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // ret - - return impPopStack().val; + CreateTreeFromBuffer(int8_t, gtNewIconNode); + break; } - - case NI_SRCS_UNSAFE_ByteOffset: + case TYP_UBYTE: { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.1 - // ldarg.0 - // sub - // ret - - impSpillSideEffect(true, verCurrentState.esStackDepth - - 2 DEBUGARG("Spilling op1 side effects for Unsafe.ByteOffset")); - - GenTree* op2 = impPopStack().val; - GenTree* op1 = impPopStack().val; - impBashVarAddrsToI(op1, op2); - - var_types type = impGetByRefResultType(GT_SUB, /* uns */ false, &op2, &op1); - return gtNewOperNode(GT_SUB, type, op2, op1); + CreateTreeFromBuffer(uint8_t, gtNewIconNode); + break; } - - case NI_SRCS_UNSAFE_Copy: + case TYP_SHORT: { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // ldarg.1 - // ldobj !!T - // stobj !!T - // ret - - return nullptr; + CreateTreeFromBuffer(int16_t, gtNewIconNode); + break; } - - case NI_SRCS_UNSAFE_CopyBlock: + case TYP_USHORT: { - assert(sig->sigInst.methInstCount == 0); - - // ldarg.0 - // ldarg.1 - // ldarg.2 - // cpblk - // ret - - return nullptr; + CreateTreeFromBuffer(uint16_t, gtNewIconNode); + break; } - - case NI_SRCS_UNSAFE_CopyBlockUnaligned: + case TYP_UINT: + case TYP_INT: { - assert(sig->sigInst.methInstCount == 0); - - // ldarg.0 - // ldarg.1 - // ldarg.2 - // unaligned. 0x1 - // cpblk - // ret - - return nullptr; + CreateTreeFromBuffer(int32_t, gtNewIconNode); + break; } - - case NI_SRCS_UNSAFE_InitBlock: + case TYP_LONG: + case TYP_ULONG: { - assert(sig->sigInst.methInstCount == 0); - - // ldarg.0 - // ldarg.1 - // ldarg.2 - // initblk - // ret - - return nullptr; + CreateTreeFromBuffer(int64_t, gtNewLconNode); + break; } - - case NI_SRCS_UNSAFE_InitBlockUnaligned: + case TYP_FLOAT: { - assert(sig->sigInst.methInstCount == 0); - - // ldarg.0 - // ldarg.1 - // ldarg.2 - // unaligned. 0x1 - // initblk - // ret - - return nullptr; + CreateTreeFromBuffer(float, gtNewDconNode); + break; } - - case NI_SRCS_UNSAFE_IsAddressGreaterThan: + case TYP_DOUBLE: { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // ldarg.1 - // cgt.un - // ret - - GenTree* op2 = impPopStack().val; - GenTree* op1 = impPopStack().val; - - GenTree* tmp = gtNewOperNode(GT_GT, TYP_INT, op1, op2); - tmp->gtFlags |= GTF_UNSIGNED; - return gtFoldExpr(tmp); + CreateTreeFromBuffer(double, gtNewDconNode); + break; } - - case NI_SRCS_UNSAFE_IsAddressLessThan: + case TYP_REF: { - assert(sig->sigInst.methInstCount == 1); + size_t ptr; + memcpy(&ptr, buffer, sizeof(ssize_t)); - // ldarg.0 - // ldarg.1 - // clt.un - // ret - - GenTree* op2 = impPopStack().val; - GenTree* op1 = impPopStack().val; - - GenTree* tmp = gtNewOperNode(GT_LT, TYP_INT, op1, op2); - tmp->gtFlags |= GTF_UNSIGNED; - return gtFoldExpr(tmp); + if (ptr == 0) + { + tree = gtNewNull(); + } + else + { + setMethodHasFrozenObjects(); + tree = gtNewIconEmbHndNode((void*)ptr, nullptr, GTF_ICON_OBJ_HDL, nullptr); + tree->gtType = TYP_REF; + INDEBUG(tree->AsIntCon()->gtTargetHandle = ptr); + } + break; } + default: + return nullptr; + } - case NI_SRCS_UNSAFE_IsNullRef: - { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // ldc.i4.0 - // conv.u - // ceq - // ret - - GenTree* op1 = impPopStack().val; - GenTree* cns = gtNewIconNode(0, TYP_BYREF); - GenTree* tmp = gtNewOperNode(GT_EQ, TYP_INT, op1, cns); - return gtFoldExpr(tmp); - } + assert(tree != nullptr); + tree->gtType = genActualType(valueType); + return tree; +} - case NI_SRCS_UNSAFE_NullRef: - { - assert(sig->sigInst.methInstCount == 1); +GenTree* Compiler::impImportStaticFieldAccess(CORINFO_RESOLVED_TOKEN* pResolvedToken, + CORINFO_ACCESS_FLAGS access, + CORINFO_FIELD_INFO* pFieldInfo, + var_types lclTyp) +{ + // Ordinary static fields never overlap. RVA statics, however, can overlap (if they're + // mapped to the same ".data" declaration). That said, such mappings only appear to be + // possible with ILASM, and in ILASM-produced (ILONLY) images, RVA statics are always + // read-only (using "stsfld" on them is UB). In mixed-mode assemblies, RVA statics can + // be mutable, but the only current producer of such images, the C++/CLI compiler, does + // not appear to support mapping different fields to the same address. So we will say + // that "mutable overlapping RVA statics" are UB as well. - // ldc.i4.0 - // conv.u - // ret + // For statics that are not "boxed", the initial address tree will contain the field sequence. + // For those that are, we will attach it later, when adding the indirection for the box, since + // that tree will represent the true address. + bool isBoxedStatic = (pFieldInfo->fieldFlags & CORINFO_FLG_FIELD_STATIC_IN_HEAP) != 0; + bool isSharedStatic = (pFieldInfo->fieldAccessor == CORINFO_FIELD_STATIC_GENERICS_STATIC_HELPER) || + (pFieldInfo->fieldAccessor == CORINFO_FIELD_STATIC_READYTORUN_HELPER); + FieldSeq::FieldKind fieldKind = + isSharedStatic ? FieldSeq::FieldKind::SharedStatic : FieldSeq::FieldKind::SimpleStatic; - return gtNewIconNode(0, TYP_BYREF); - } + bool hasConstAddr = (pFieldInfo->fieldAccessor == CORINFO_FIELD_STATIC_ADDRESS) || + (pFieldInfo->fieldAccessor == CORINFO_FIELD_STATIC_RVA_ADDRESS); - case NI_SRCS_UNSAFE_Read: + FieldSeq* innerFldSeq; + FieldSeq* outerFldSeq; + if (isBoxedStatic) + { + innerFldSeq = nullptr; + outerFldSeq = GetFieldSeqStore()->Create(pResolvedToken->hField, TARGET_POINTER_SIZE, fieldKind); + } + else + { + ssize_t offset; + if (hasConstAddr) { - assert(sig->sigInst.methInstCount == 1); + // Change SimpleStatic to SimpleStaticKnownAddress + assert(fieldKind == FieldSeq::FieldKind::SimpleStatic); + fieldKind = FieldSeq::FieldKind::SimpleStaticKnownAddress; - // ldarg.0 - // ldobj !!T - // ret - - return nullptr; + assert(pFieldInfo->fieldLookup.accessType == IAT_VALUE); + offset = reinterpret_cast(pFieldInfo->fieldLookup.addr); + } + else + { + offset = pFieldInfo->offset; } - case NI_SRCS_UNSAFE_ReadUnaligned: + innerFldSeq = GetFieldSeqStore()->Create(pResolvedToken->hField, offset, fieldKind); + outerFldSeq = nullptr; + } + + bool isStaticReadOnlyInitedRef = false; + GenTree* op1; + switch (pFieldInfo->fieldAccessor) + { + case CORINFO_FIELD_STATIC_GENERICS_STATIC_HELPER: { - assert(sig->sigInst.methInstCount == 1); + assert(!compIsForInlining()); - // ldarg.0 - // unaligned. 0x1 - // ldobj !!T - // ret + // We first call a special helper to get the statics base pointer + op1 = impParentClassTokenToHandle(pResolvedToken); - return nullptr; - } + // compIsForInlining() is false so we should not get NULL here + assert(op1 != nullptr); - case NI_SRCS_UNSAFE_SizeOf: - { - assert(sig->sigInst.methInstCount == 1); + var_types type = TYP_BYREF; - // sizeof !!T - // ret + switch (pFieldInfo->helper) + { + case CORINFO_HELP_GETGENERICS_NONGCTHREADSTATIC_BASE: + type = TYP_I_IMPL; + break; + case CORINFO_HELP_GETGENERICS_GCSTATIC_BASE: + case CORINFO_HELP_GETGENERICS_NONGCSTATIC_BASE: + case CORINFO_HELP_GETGENERICS_GCTHREADSTATIC_BASE: + break; + default: + assert(!"unknown generic statics helper"); + break; + } - unsigned classSize = info.compCompHnd->getClassSize(sig->sigInst.methInst[0]); - return gtNewIconNode(classSize, TYP_INT); + op1 = gtNewHelperCallNode(pFieldInfo->helper, type, op1); + op1 = gtNewOperNode(GT_ADD, type, op1, gtNewIconNode(pFieldInfo->offset, innerFldSeq)); } + break; - case NI_SRCS_UNSAFE_SkipInit: + case CORINFO_FIELD_STATIC_SHARED_STATIC_HELPER: { - assert(sig->sigInst.methInstCount == 1); +#ifdef FEATURE_READYTORUN + if (opts.IsReadyToRun()) + { + GenTreeFlags callFlags = GTF_EMPTY; - // ret + if (info.compCompHnd->getClassAttribs(pResolvedToken->hClass) & CORINFO_FLG_BEFOREFIELDINIT) + { + callFlags |= GTF_CALL_HOISTABLE; + } - GenTree* op1 = impPopStack().val; + op1 = gtNewHelperCallNode(pFieldInfo->helper, TYP_BYREF); + if (pResolvedToken->hClass == info.compClassHnd && m_preferredInitCctor == CORINFO_HELP_UNDEF && + (pFieldInfo->helper == CORINFO_HELP_READYTORUN_GCSTATIC_BASE || + pFieldInfo->helper == CORINFO_HELP_READYTORUN_NONGCSTATIC_BASE)) + { + m_preferredInitCctor = pFieldInfo->helper; + } + op1->gtFlags |= callFlags; - if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0) - { - return gtUnusedValNode(op1); + op1->AsCall()->setEntryPoint(pFieldInfo->fieldLookup); } else +#endif { - return gtNewNothingNode(); + op1 = fgGetStaticsCCtorHelper(pResolvedToken->hClass, pFieldInfo->helper); } + + op1 = gtNewOperNode(GT_ADD, op1->TypeGet(), op1, gtNewIconNode(pFieldInfo->offset, innerFldSeq)); + break; } - case NI_SRCS_UNSAFE_Subtract: + case CORINFO_FIELD_STATIC_READYTORUN_HELPER: { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // ldarg.1 - // sizeof !!T - // conv.i - // mul - // sub - // ret - - GenTree* op2 = impPopStack().val; - GenTree* op1 = impPopStack().val; - impBashVarAddrsToI(op1, op2); +#ifdef FEATURE_READYTORUN + assert(opts.IsReadyToRun()); + assert(!compIsForInlining()); + CORINFO_LOOKUP_KIND kind; + info.compCompHnd->getLocationOfThisType(info.compMethodHnd, &kind); + assert(kind.needsRuntimeLookup); - op2 = impImplicitIorI4Cast(op2, TYP_I_IMPL); + GenTree* ctxTree = getRuntimeContextTree(kind.runtimeLookupKind); - unsigned classSize = info.compCompHnd->getClassSize(sig->sigInst.methInst[0]); + GenTreeFlags callFlags = GTF_EMPTY; - if (classSize != 1) + if (info.compCompHnd->getClassAttribs(pResolvedToken->hClass) & CORINFO_FLG_BEFOREFIELDINIT) { - GenTree* size = gtNewIconNode(classSize, TYP_I_IMPL); - op2 = gtNewOperNode(GT_MUL, TYP_I_IMPL, op2, size); + callFlags |= GTF_CALL_HOISTABLE; } + var_types type = TYP_BYREF; + op1 = gtNewHelperCallNode(CORINFO_HELP_READYTORUN_GENERIC_STATIC_BASE, type, ctxTree); + op1->gtFlags |= callFlags; - var_types type = impGetByRefResultType(GT_SUB, /* uns */ false, &op1, &op2); - return gtNewOperNode(GT_SUB, type, op1, op2); + op1->AsCall()->setEntryPoint(pFieldInfo->fieldLookup); + op1 = gtNewOperNode(GT_ADD, type, op1, gtNewIconNode(pFieldInfo->offset, innerFldSeq)); +#else + unreached(); +#endif // FEATURE_READYTORUN } + break; - case NI_SRCS_UNSAFE_SubtractByteOffset: + default: { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // ldarg.1 - // sub - // ret - - GenTree* op2 = impPopStack().val; - GenTree* op1 = impPopStack().val; - impBashVarAddrsToI(op1, op2); +// TODO-CQ: enable this optimization for 32 bit targets. +#ifdef TARGET_64BIT + if (!isBoxedStatic && (lclTyp == TYP_REF) && ((access & CORINFO_ACCESS_ADDRESS) == 0)) + { + bool isSpeculative = true; + if ((info.compCompHnd->getStaticFieldCurrentClass(pResolvedToken->hField, &isSpeculative) != + NO_CLASS_HANDLE)) + { + isStaticReadOnlyInitedRef = !isSpeculative; + } + } +#endif // TARGET_64BIT - var_types type = impGetByRefResultType(GT_SUB, /* uns */ false, &op1, &op2); - return gtNewOperNode(GT_SUB, type, op1, op2); + assert(hasConstAddr); + assert(pFieldInfo->fieldLookup.addr != nullptr); + assert(pFieldInfo->fieldLookup.accessType == IAT_VALUE); + size_t fldAddr = reinterpret_cast(pFieldInfo->fieldLookup.addr); + GenTreeFlags handleKind; + if (isBoxedStatic) + { + handleKind = GTF_ICON_STATIC_BOX_PTR; + } + else if (isStaticReadOnlyInitedRef) + { + handleKind = GTF_ICON_CONST_PTR; + } + else + { + handleKind = GTF_ICON_STATIC_HDL; + } + op1 = gtNewIconHandleNode(fldAddr, handleKind, innerFldSeq); + INDEBUG(op1->AsIntCon()->gtTargetHandle = reinterpret_cast(pResolvedToken->hField)); + if (pFieldInfo->fieldFlags & CORINFO_FLG_FIELD_INITCLASS) + { + op1->gtFlags |= GTF_ICON_INITCLASS; + } + break; } + } - case NI_SRCS_UNSAFE_Unbox: - { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // unbox !!T - // ret - - return nullptr; - } + if (isBoxedStatic) + { + op1 = gtNewIndir(TYP_REF, op1, GTF_IND_INVARIANT | GTF_IND_NONFAULTING | GTF_IND_NONNULL); + op1 = gtNewOperNode(GT_ADD, TYP_BYREF, op1, gtNewIconNode(TARGET_POINTER_SIZE, outerFldSeq)); + } - case NI_SRCS_UNSAFE_Write: + if (!(access & CORINFO_ACCESS_ADDRESS)) + { + if (varTypeIsStruct(lclTyp)) { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // ldarg.1 - // stobj !!T - // ret - - return nullptr; + // Constructor adds GTF_GLOB_REF. Note that this is *not* GTF_EXCEPT. + op1 = gtNewObjNode(pFieldInfo->structType, op1); } - - case NI_SRCS_UNSAFE_WriteUnaligned: + else { - assert(sig->sigInst.methInstCount == 1); - - // ldarg.0 - // ldarg.1 - // unaligned. 0x01 - // stobj !!T - // ret - - return nullptr; + op1 = gtNewOperNode(GT_IND, lclTyp, op1); + op1->gtFlags |= GTF_GLOB_REF; } - - default: + if (isStaticReadOnlyInitedRef) { - unreached(); + op1->gtFlags |= (GTF_IND_INVARIANT | GTF_IND_NONFAULTING | GTF_IND_NONNULL); } } + + return op1; } -GenTree* Compiler::impTypeIsAssignable(GenTree* typeTo, GenTree* typeFrom) +// In general try to call this before most of the verification work. Most people expect the access +// exceptions before the verification exceptions. If you do this after, that usually doesn't happen. Turns +// out if you can't access something we also think that you're unverifiable for other reasons. +void Compiler::impHandleAccessAllowed(CorInfoIsAccessAllowedResult result, CORINFO_HELPER_DESC* helperCall) { - // Optimize patterns like: - // - // typeof(TTo).IsAssignableFrom(typeof(TTFrom)) - // valueTypeVar.GetType().IsAssignableFrom(typeof(TTFrom)) - // typeof(TTFrom).IsAssignableTo(typeof(TTo)) - // typeof(TTFrom).IsAssignableTo(valueTypeVar.GetType()) - // - // to true/false - - if (typeTo->IsCall() && typeFrom->IsCall()) + if (result != CORINFO_ACCESS_ALLOWED) { - // make sure both arguments are `typeof()` - CORINFO_METHOD_HANDLE hTypeof = eeFindHelper(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE); - if ((typeTo->AsCall()->gtCallMethHnd == hTypeof) && (typeFrom->AsCall()->gtCallMethHnd == hTypeof)) - { - assert((typeTo->AsCall()->gtArgs.CountArgs() == 1) && (typeFrom->AsCall()->gtArgs.CountArgs() == 1)); - CORINFO_CLASS_HANDLE hClassTo = - gtGetHelperArgClassHandle(typeTo->AsCall()->gtArgs.GetArgByIndex(0)->GetEarlyNode()); - CORINFO_CLASS_HANDLE hClassFrom = - gtGetHelperArgClassHandle(typeFrom->AsCall()->gtArgs.GetArgByIndex(0)->GetEarlyNode()); + impHandleAccessAllowedInternal(result, helperCall); + } +} - if (hClassTo == NO_CLASS_HANDLE || hClassFrom == NO_CLASS_HANDLE) +void Compiler::impHandleAccessAllowedInternal(CorInfoIsAccessAllowedResult result, CORINFO_HELPER_DESC* helperCall) +{ + switch (result) + { + case CORINFO_ACCESS_ALLOWED: + break; + case CORINFO_ACCESS_ILLEGAL: + // if we're verifying, then we need to reject the illegal access to ensure that we don't think the + // method is verifiable. Otherwise, delay the exception to runtime. + if (compIsForImportOnly()) { - return nullptr; + info.compCompHnd->ThrowExceptionForHelper(helperCall); } - - TypeCompareState castResult = info.compCompHnd->compareTypesForCast(hClassFrom, hClassTo); - if (castResult == TypeCompareState::May) + else { - // requires runtime check - // e.g. __Canon, COMObjects, Nullable - return nullptr; + impInsertHelperCall(helperCall); } + break; + } +} - GenTreeIntCon* retNode = gtNewIconNode((castResult == TypeCompareState::Must) ? 1 : 0); - impPopStack(); // drop both CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE calls - impPopStack(); +void Compiler::impInsertHelperCall(CORINFO_HELPER_DESC* helperInfo) +{ + assert(helperInfo->helperNum != CORINFO_HELP_UNDEF); - return retNode; + /* TODO-Review: + * Mark as CSE'able, and hoistable. Consider marking hoistable unless you're in the inlinee. + * Also, consider sticking this in the first basic block. + */ + GenTreeCall* callout = gtNewHelperCallNode(helperInfo->helperNum, TYP_VOID); + // Add the arguments + for (unsigned i = helperInfo->numArgs; i > 0; --i) + { + const CORINFO_HELPER_ARG& helperArg = helperInfo->args[i - 1]; + GenTree* currentArg = nullptr; + switch (helperArg.argType) + { + case CORINFO_HELPER_ARG_TYPE_Field: + info.compCompHnd->classMustBeLoadedBeforeCodeIsRun( + info.compCompHnd->getFieldClass(helperArg.fieldHandle)); + currentArg = gtNewIconEmbFldHndNode(helperArg.fieldHandle); + break; + case CORINFO_HELPER_ARG_TYPE_Method: + info.compCompHnd->methodMustBeLoadedBeforeCodeIsRun(helperArg.methodHandle); + currentArg = gtNewIconEmbMethHndNode(helperArg.methodHandle); + break; + case CORINFO_HELPER_ARG_TYPE_Class: + info.compCompHnd->classMustBeLoadedBeforeCodeIsRun(helperArg.classHandle); + currentArg = gtNewIconEmbClsHndNode(helperArg.classHandle); + break; + case CORINFO_HELPER_ARG_TYPE_Module: + currentArg = gtNewIconEmbScpHndNode(helperArg.moduleHandle); + break; + case CORINFO_HELPER_ARG_TYPE_Const: + currentArg = gtNewIconNode(helperArg.constant); + break; + default: + NO_WAY("Illegal helper arg type"); } + callout->gtArgs.PushFront(this, NewCallArg::Primitive(currentArg)); } - return nullptr; + impAppendTree(callout, CHECK_SPILL_NONE, impCurStmtDI); } -GenTree* Compiler::impMathIntrinsic(CORINFO_METHOD_HANDLE method, - CORINFO_SIG_INFO* sig, - var_types callType, - NamedIntrinsic intrinsicName, - bool tailCall) +/******************************************************************************** + * + * Returns true if the current opcode and and the opcodes following it correspond + * to a supported tail call IL pattern. + * + */ +bool Compiler::impIsTailCallILPattern( + bool tailPrefixed, OPCODE curOpcode, const BYTE* codeAddrOfNextOpcode, const BYTE* codeEnd, bool isRecursive) { - GenTree* op1; - GenTree* op2; - - assert(callType != TYP_STRUCT); - assert(IsMathIntrinsic(intrinsicName)); - - op1 = nullptr; + // Bail out if the current opcode is not a call. + if (!impOpcodeIsCallOpcode(curOpcode)) + { + return false; + } -#if !defined(TARGET_X86) - // Intrinsics that are not implemented directly by target instructions will - // be re-materialized as users calls in rationalizer. For prefixed tail calls, - // don't do this optimization, because - // a) For back compatibility reasons on desktop .NET Framework 4.6 / 4.6.1 - // b) It will be non-trivial task or too late to re-materialize a surviving - // tail prefixed GT_INTRINSIC as tail call in rationalizer. - if (!IsIntrinsicImplementedByUserCall(intrinsicName) || !tailCall) -#else - // On x86 RyuJIT, importing intrinsics that are implemented as user calls can cause incorrect calculation - // of the depth of the stack if these intrinsics are used as arguments to another call. This causes bad - // code generation for certain EH constructs. - if (!IsIntrinsicImplementedByUserCall(intrinsicName)) +#if !FEATURE_TAILCALL_OPT_SHARED_RETURN + // If shared ret tail opt is not enabled, we will enable + // it for recursive methods. + if (isRecursive) #endif { - CORINFO_CLASS_HANDLE tmpClass; - CORINFO_ARG_LIST_HANDLE arg; - var_types op1Type; - var_types op2Type; - - switch (sig->numArgs) - { - case 1: - op1 = impPopStack().val; - - arg = sig->args; - op1Type = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg, &tmpClass))); - - if (op1->TypeGet() != genActualType(op1Type)) - { - assert(varTypeIsFloating(op1)); - op1 = gtNewCastNode(callType, op1, false, callType); - } + // we can actually handle if the ret is in a fallthrough block, as long as that is the only part of the + // sequence. Make sure we don't go past the end of the IL however. + codeEnd = min(codeEnd + 1, info.compCode + info.compILCodeSize); + } - op1 = new (this, GT_INTRINSIC) GenTreeIntrinsic(genActualType(callType), op1, intrinsicName, method); - break; + // Bail out if there is no next opcode after call + if (codeAddrOfNextOpcode >= codeEnd) + { + return false; + } - case 2: - op2 = impPopStack().val; - op1 = impPopStack().val; + OPCODE nextOpcode = (OPCODE)getU1LittleEndian(codeAddrOfNextOpcode); - arg = sig->args; - op1Type = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg, &tmpClass))); + return (nextOpcode == CEE_RET); +} - if (op1->TypeGet() != genActualType(op1Type)) - { - assert(varTypeIsFloating(op1)); - op1 = gtNewCastNode(callType, op1, false, callType); - } +/***************************************************************************** + * + * Determine whether the call could be converted to an implicit tail call + * + */ +bool Compiler::impIsImplicitTailCallCandidate( + OPCODE opcode, const BYTE* codeAddrOfNextOpcode, const BYTE* codeEnd, int prefixFlags, bool isRecursive) +{ - arg = info.compCompHnd->getArgNext(arg); - op2Type = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg, &tmpClass))); +#if FEATURE_TAILCALL_OPT + if (!opts.compTailCallOpt) + { + return false; + } - if (op2->TypeGet() != genActualType(op2Type)) - { - assert(varTypeIsFloating(op2)); - op2 = gtNewCastNode(callType, op2, false, callType); - } + if (opts.OptimizationDisabled()) + { + return false; + } - op1 = - new (this, GT_INTRINSIC) GenTreeIntrinsic(genActualType(callType), op1, op2, intrinsicName, method); - break; + // must not be tail prefixed + if (prefixFlags & PREFIX_TAILCALL_EXPLICIT) + { + return false; + } - default: - NO_WAY("Unsupported number of args for Math Intrinsic"); - } +#if !FEATURE_TAILCALL_OPT_SHARED_RETURN + // the block containing call is marked as BBJ_RETURN + // We allow shared ret tail call optimization on recursive calls even under + // !FEATURE_TAILCALL_OPT_SHARED_RETURN. + if (!isRecursive && (compCurBB->bbJumpKind != BBJ_RETURN)) + return false; +#endif // !FEATURE_TAILCALL_OPT_SHARED_RETURN - if (IsIntrinsicImplementedByUserCall(intrinsicName)) - { - op1->gtFlags |= GTF_CALL; - } + // must be call+ret or call+pop+ret + if (!impIsTailCallILPattern(false, opcode, codeAddrOfNextOpcode, codeEnd, isRecursive)) + { + return false; } - return op1; + return true; +#else + return false; +#endif // FEATURE_TAILCALL_OPT } +/***************************************************************************** + For struct return values, re-type the operand in the case where the ABI + does not use a struct return buffer + */ + //------------------------------------------------------------------------ -// lookupNamedIntrinsic: map method to jit named intrinsic value +// impFixupStructReturnType: Adjust a struct value being returned. +// +// In the multi-reg case, we we force IR to be one of the following: +// GT_RETURN(LCL_VAR) or GT_RETURN(CALL). If op is anything other than +// a lclvar or call, it is assigned to a temp, which is then returned. +// In the non-multireg case, the two special helpers with "fake" return +// buffers are handled ("GETFIELDSTRUCT" and "UNBOX_NULLABLE"). // // Arguments: -// method -- method handle for method +// op - the return value // // Return Value: -// Id for the named intrinsic, or Illegal if none. -// -// Notes: -// method should have CORINFO_FLG_INTRINSIC set in its attributes, -// otherwise it is not a named jit intrinsic. +// The (possibly modified) value to return. // -NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) +GenTree* Compiler::impFixupStructReturnType(GenTree* op) { - const char* className = nullptr; - const char* namespaceName = nullptr; - const char* enclosingClassName = nullptr; - const char* methodName = - info.compCompHnd->getMethodNameFromMetadata(method, &className, &namespaceName, &enclosingClassName); - - JITDUMP("Named Intrinsic "); + assert(varTypeIsStruct(info.compRetType)); + assert(info.compRetBuffArg == BAD_VAR_NUM); - if (namespaceName != nullptr) - { - JITDUMP("%s.", namespaceName); - } - if (enclosingClassName != nullptr) - { - JITDUMP("%s.", enclosingClassName); - } - if (className != nullptr) - { - JITDUMP("%s.", className); - } - if (methodName != nullptr) - { - JITDUMP("%s", methodName); - } + JITDUMP("\nimpFixupStructReturnType: retyping\n"); + DISPTREE(op); - if ((namespaceName == nullptr) || (className == nullptr) || (methodName == nullptr)) + if (op->IsCall() && op->AsCall()->TreatAsShouldHaveRetBufArg(this)) { - // Check if we are dealing with an MD array's known runtime method - CorInfoArrayIntrinsic arrayFuncIndex = info.compCompHnd->getArrayIntrinsicID(method); - switch (arrayFuncIndex) - { - case CorInfoArrayIntrinsic::GET: - JITDUMP("ARRAY_FUNC_GET: Recognized\n"); - return NI_Array_Get; - case CorInfoArrayIntrinsic::SET: - JITDUMP("ARRAY_FUNC_SET: Recognized\n"); - return NI_Array_Set; - case CorInfoArrayIntrinsic::ADDRESS: - JITDUMP("ARRAY_FUNC_ADDRESS: Recognized\n"); - return NI_Array_Address; - default: - break; - } + // This must be one of those 'special' helpers that don't really have a return buffer, but instead + // use it as a way to keep the trees cleaner with fewer address-taken temps. Well now we have to + // materialize the return buffer as an address-taken temp. Then we can return the temp. + // + unsigned tmpNum = lvaGrabTemp(true DEBUGARG("pseudo return buffer")); - JITDUMP(": Not recognized, not enough metadata\n"); - return NI_Illegal; - } + // No need to spill anything as we're about to return. + impAssignTempGen(tmpNum, op, info.compMethodInfo->args.retTypeClass, CHECK_SPILL_NONE); - JITDUMP(": "); + op = gtNewLclvNode(tmpNum, info.compRetType); + JITDUMP("\nimpFixupStructReturnType: created a pseudo-return buffer for a special helper\n"); + DISPTREE(op); - NamedIntrinsic result = NI_Illegal; + return op; + } - if (strcmp(namespaceName, "System") == 0) + if (compMethodReturnsMultiRegRetType() || op->IsMultiRegNode()) { - if ((strcmp(className, "Enum") == 0) && (strcmp(methodName, "HasFlag") == 0)) + // We can use any local with multiple registers (it will be forced to memory on mismatch), + // except for implicit byrefs (they may turn into indirections). + if (op->OperIs(GT_LCL_VAR) && !lvaIsImplicitByRefLocal(op->AsLclVar()->GetLclNum())) { - result = NI_System_Enum_HasFlag; + // Note that this is a multi-reg return. + unsigned lclNum = op->AsLclVarCommon()->GetLclNum(); + lvaTable[lclNum].lvIsMultiRegRet = true; + + // TODO-1stClassStructs: Handle constant propagation and CSE-ing of multireg returns. + op->gtFlags |= GTF_DONT_CSE; + + return op; } - else if (strcmp(className, "Activator") == 0) + + // In contrast, we can only use multi-reg calls directly if they have the exact same ABI. + // Calling convention equality is a conservative approximation for that check. + if (op->IsCall() && (op->AsCall()->GetUnmanagedCallConv() == info.compCallConv) +#if defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) + // TODO-Review: this seems unnecessary. Return ABI doesn't change under varargs. + && !op->AsCall()->IsVarargs() +#endif // defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) + ) { - if (strcmp(methodName, "AllocatorOf") == 0) - { - result = NI_System_Activator_AllocatorOf; - } - else if (strcmp(methodName, "DefaultConstructorOf") == 0) - { - result = NI_System_Activator_DefaultConstructorOf; - } + return op; } - else if (strcmp(className, "BitConverter") == 0) + + if (op->IsCall()) { - if (methodName[0] == 'D') - { - if (strcmp(methodName, "DoubleToInt64Bits") == 0) - { - result = NI_System_BitConverter_DoubleToInt64Bits; - } - else if (strcmp(methodName, "DoubleToUInt64Bits") == 0) - { - result = NI_System_BitConverter_DoubleToInt64Bits; - } - } - else if (methodName[0] == 'I') - { - if (strcmp(methodName, "Int32BitsToSingle") == 0) - { - result = NI_System_BitConverter_Int32BitsToSingle; - } - else if (strcmp(methodName, "Int64BitsToDouble") == 0) - { - result = NI_System_BitConverter_Int64BitsToDouble; - } - } - else if (methodName[0] == 'S') - { - if (strcmp(methodName, "SingleToInt32Bits") == 0) - { - result = NI_System_BitConverter_SingleToInt32Bits; - } - else if (strcmp(methodName, "SingleToUInt32Bits") == 0) - { - result = NI_System_BitConverter_SingleToInt32Bits; - } - } - else if (methodName[0] == 'U') - { - if (strcmp(methodName, "UInt32BitsToSingle") == 0) - { - result = NI_System_BitConverter_Int32BitsToSingle; - } - else if (strcmp(methodName, "UInt64BitsToDouble") == 0) - { - result = NI_System_BitConverter_Int64BitsToDouble; - } - } + // We cannot tail call because control needs to return to fixup the calling convention + // for result return. + op->AsCall()->gtCallMoreFlags &= ~GTF_CALL_M_TAILCALL; + op->AsCall()->gtCallMoreFlags &= ~GTF_CALL_M_EXPLICIT_TAILCALL; } - else if (strcmp(className, "Math") == 0 || strcmp(className, "MathF") == 0) - { - if (strcmp(methodName, "Abs") == 0) - { - result = NI_System_Math_Abs; - } - else if (strcmp(methodName, "Acos") == 0) - { - result = NI_System_Math_Acos; - } - else if (strcmp(methodName, "Acosh") == 0) - { - result = NI_System_Math_Acosh; - } - else if (strcmp(methodName, "Asin") == 0) - { - result = NI_System_Math_Asin; - } - else if (strcmp(methodName, "Asinh") == 0) - { - result = NI_System_Math_Asinh; - } - else if (strcmp(methodName, "Atan") == 0) - { - result = NI_System_Math_Atan; - } - else if (strcmp(methodName, "Atanh") == 0) - { - result = NI_System_Math_Atanh; - } - else if (strcmp(methodName, "Atan2") == 0) - { - result = NI_System_Math_Atan2; - } - else if (strcmp(methodName, "Cbrt") == 0) - { - result = NI_System_Math_Cbrt; - } - else if (strcmp(methodName, "Ceiling") == 0) - { - result = NI_System_Math_Ceiling; - } - else if (strcmp(methodName, "Cos") == 0) - { - result = NI_System_Math_Cos; - } - else if (strcmp(methodName, "Cosh") == 0) - { - result = NI_System_Math_Cosh; - } - else if (strcmp(methodName, "Exp") == 0) - { - result = NI_System_Math_Exp; - } - else if (strcmp(methodName, "Floor") == 0) - { - result = NI_System_Math_Floor; - } - else if (strcmp(methodName, "FMod") == 0) - { - result = NI_System_Math_FMod; - } - else if (strcmp(methodName, "FusedMultiplyAdd") == 0) - { - result = NI_System_Math_FusedMultiplyAdd; - } - else if (strcmp(methodName, "ILogB") == 0) - { - result = NI_System_Math_ILogB; - } - else if (strcmp(methodName, "Log") == 0) - { - result = NI_System_Math_Log; - } - else if (strcmp(methodName, "Log2") == 0) - { - result = NI_System_Math_Log2; - } - else if (strcmp(methodName, "Log10") == 0) - { - result = NI_System_Math_Log10; - } - else if (strcmp(methodName, "Max") == 0) - { - result = NI_System_Math_Max; - } - else if (strcmp(methodName, "Min") == 0) - { - result = NI_System_Math_Min; - } - else if (strcmp(methodName, "Pow") == 0) - { - result = NI_System_Math_Pow; - } - else if (strcmp(methodName, "Round") == 0) - { - result = NI_System_Math_Round; - } - else if (strcmp(methodName, "Sin") == 0) - { - result = NI_System_Math_Sin; - } - else if (strcmp(methodName, "Sinh") == 0) - { - result = NI_System_Math_Sinh; - } - else if (strcmp(methodName, "Sqrt") == 0) - { - result = NI_System_Math_Sqrt; - } - else if (strcmp(methodName, "Tan") == 0) - { - result = NI_System_Math_Tan; - } - else if (strcmp(methodName, "Tanh") == 0) - { - result = NI_System_Math_Tanh; - } - else if (strcmp(methodName, "Truncate") == 0) - { - result = NI_System_Math_Truncate; - } - } - else if (strcmp(className, "GC") == 0) - { - if (strcmp(methodName, "KeepAlive") == 0) - { - result = NI_System_GC_KeepAlive; - } - } - else if (strcmp(className, "Array") == 0) - { - if (strcmp(methodName, "Clone") == 0) - { - result = NI_System_Array_Clone; - } - else if (strcmp(methodName, "GetLength") == 0) - { - result = NI_System_Array_GetLength; - } - else if (strcmp(methodName, "GetLowerBound") == 0) - { - result = NI_System_Array_GetLowerBound; - } - else if (strcmp(methodName, "GetUpperBound") == 0) - { - result = NI_System_Array_GetUpperBound; - } - } - else if (strcmp(className, "Object") == 0) - { - if (strcmp(methodName, "MemberwiseClone") == 0) - { - result = NI_System_Object_MemberwiseClone; - } - else if (strcmp(methodName, "GetType") == 0) - { - result = NI_System_Object_GetType; - } - } - else if (strcmp(className, "RuntimeTypeHandle") == 0) - { - if (strcmp(methodName, "GetValueInternal") == 0) - { - result = NI_System_RuntimeTypeHandle_GetValueInternal; - } - } - else if (strcmp(className, "Type") == 0) - { - if (strcmp(methodName, "get_IsValueType") == 0) - { - result = NI_System_Type_get_IsValueType; - } - else if (strcmp(methodName, "get_IsByRefLike") == 0) - { - result = NI_System_Type_get_IsByRefLike; - } - else if (strcmp(methodName, "IsAssignableFrom") == 0) - { - result = NI_System_Type_IsAssignableFrom; - } - else if (strcmp(methodName, "IsAssignableTo") == 0) - { - result = NI_System_Type_IsAssignableTo; - } - else if (strcmp(methodName, "op_Equality") == 0) - { - result = NI_System_Type_op_Equality; - } - else if (strcmp(methodName, "op_Inequality") == 0) - { - result = NI_System_Type_op_Inequality; - } - else if (strcmp(methodName, "GetTypeFromHandle") == 0) - { - result = NI_System_Type_GetTypeFromHandle; - } - } - else if (strcmp(className, "String") == 0) - { - if (strcmp(methodName, "Equals") == 0) - { - result = NI_System_String_Equals; - } - else if (strcmp(methodName, "get_Chars") == 0) - { - result = NI_System_String_get_Chars; - } - else if (strcmp(methodName, "get_Length") == 0) - { - result = NI_System_String_get_Length; - } - else if (strcmp(methodName, "op_Implicit") == 0) - { - result = NI_System_String_op_Implicit; - } - else if (strcmp(methodName, "StartsWith") == 0) - { - result = NI_System_String_StartsWith; - } - } - else if (strcmp(className, "MemoryExtensions") == 0) - { - if (strcmp(methodName, "AsSpan") == 0) - { - result = NI_System_MemoryExtensions_AsSpan; - } - if (strcmp(methodName, "SequenceEqual") == 0) - { - result = NI_System_MemoryExtensions_SequenceEqual; - } - else if (strcmp(methodName, "Equals") == 0) - { - result = NI_System_MemoryExtensions_Equals; - } - else if (strcmp(methodName, "StartsWith") == 0) - { - result = NI_System_MemoryExtensions_StartsWith; - } - } - else if (strcmp(className, "Span`1") == 0) - { - if (strcmp(methodName, "get_Item") == 0) - { - result = NI_System_Span_get_Item; - } - } - else if (strcmp(className, "ReadOnlySpan`1") == 0) - { - if (strcmp(methodName, "get_Item") == 0) - { - result = NI_System_ReadOnlySpan_get_Item; - } - } - else if (strcmp(className, "EETypePtr") == 0) - { - if (strcmp(methodName, "EETypePtrOf") == 0) - { - result = NI_System_EETypePtr_EETypePtrOf; - } - } - } - else if (strcmp(namespaceName, "Internal.Runtime") == 0) - { - if (strcmp(className, "MethodTable") == 0) - { - if (strcmp(methodName, "Of") == 0) - { - result = NI_Internal_Runtime_MethodTable_Of; - } - } - } - else if (strcmp(namespaceName, "System.Threading") == 0) - { - if (strcmp(className, "Thread") == 0) - { - if (strcmp(methodName, "get_CurrentThread") == 0) - { - result = NI_System_Threading_Thread_get_CurrentThread; - } - else if (strcmp(methodName, "get_ManagedThreadId") == 0) - { - result = NI_System_Threading_Thread_get_ManagedThreadId; - } - } - else if (strcmp(className, "Interlocked") == 0) - { -#ifndef TARGET_ARM64 - // TODO-CQ: Implement for XArch (https://github.com/dotnet/runtime/issues/32239). - if (strcmp(methodName, "And") == 0) - { - result = NI_System_Threading_Interlocked_And; - } - else if (strcmp(methodName, "Or") == 0) - { - result = NI_System_Threading_Interlocked_Or; - } -#endif - if (strcmp(methodName, "CompareExchange") == 0) - { - result = NI_System_Threading_Interlocked_CompareExchange; - } - else if (strcmp(methodName, "Exchange") == 0) - { - result = NI_System_Threading_Interlocked_Exchange; - } - else if (strcmp(methodName, "ExchangeAdd") == 0) - { - result = NI_System_Threading_Interlocked_ExchangeAdd; - } - else if (strcmp(methodName, "MemoryBarrier") == 0) - { - result = NI_System_Threading_Interlocked_MemoryBarrier; - } - else if (strcmp(methodName, "ReadMemoryBarrier") == 0) - { - result = NI_System_Threading_Interlocked_ReadMemoryBarrier; - } - } - } -#if defined(TARGET_XARCH) || defined(TARGET_ARM64) - else if (strcmp(namespaceName, "System.Buffers.Binary") == 0) - { - if ((strcmp(className, "BinaryPrimitives") == 0) && (strcmp(methodName, "ReverseEndianness") == 0)) - { - result = NI_System_Buffers_Binary_BinaryPrimitives_ReverseEndianness; - } - } -#endif // defined(TARGET_XARCH) || defined(TARGET_ARM64) - else if (strcmp(namespaceName, "System.Collections.Generic") == 0) - { - if ((strcmp(className, "EqualityComparer`1") == 0) && (strcmp(methodName, "get_Default") == 0)) - { - result = NI_System_Collections_Generic_EqualityComparer_get_Default; - } - else if ((strcmp(className, "Comparer`1") == 0) && (strcmp(methodName, "get_Default") == 0)) - { - result = NI_System_Collections_Generic_Comparer_get_Default; - } - } - else if ((strcmp(namespaceName, "System.Numerics") == 0) && (strcmp(className, "BitOperations") == 0)) - { - if (strcmp(methodName, "PopCount") == 0) - { - result = NI_System_Numerics_BitOperations_PopCount; - } - } -#ifdef FEATURE_HW_INTRINSICS - else if (strcmp(namespaceName, "System.Numerics") == 0) - { - CORINFO_SIG_INFO sig; - info.compCompHnd->getMethodSig(method, &sig); - - int sizeOfVectorT = getSIMDVectorRegisterByteLength(); - - result = SimdAsHWIntrinsicInfo::lookupId(&sig, className, methodName, enclosingClassName, sizeOfVectorT); - } -#endif // FEATURE_HW_INTRINSICS - else if (strcmp(namespaceName, "System.Runtime.CompilerServices") == 0) - { - if (strcmp(className, "Unsafe") == 0) - { - if (strcmp(methodName, "Add") == 0) - { - result = NI_SRCS_UNSAFE_Add; - } - else if (strcmp(methodName, "AddByteOffset") == 0) - { - result = NI_SRCS_UNSAFE_AddByteOffset; - } - else if (strcmp(methodName, "AreSame") == 0) - { - result = NI_SRCS_UNSAFE_AreSame; - } - else if (strcmp(methodName, "As") == 0) - { - result = NI_SRCS_UNSAFE_As; - } - else if (strcmp(methodName, "AsPointer") == 0) - { - result = NI_SRCS_UNSAFE_AsPointer; - } - else if (strcmp(methodName, "AsRef") == 0) - { - result = NI_SRCS_UNSAFE_AsRef; - } - else if (strcmp(methodName, "ByteOffset") == 0) - { - result = NI_SRCS_UNSAFE_ByteOffset; - } - else if (strcmp(methodName, "Copy") == 0) - { - result = NI_SRCS_UNSAFE_Copy; - } - else if (strcmp(methodName, "CopyBlock") == 0) - { - result = NI_SRCS_UNSAFE_CopyBlock; - } - else if (strcmp(methodName, "CopyBlockUnaligned") == 0) - { - result = NI_SRCS_UNSAFE_CopyBlockUnaligned; - } - else if (strcmp(methodName, "InitBlock") == 0) - { - result = NI_SRCS_UNSAFE_InitBlock; - } - else if (strcmp(methodName, "InitBlockUnaligned") == 0) - { - result = NI_SRCS_UNSAFE_InitBlockUnaligned; - } - else if (strcmp(methodName, "IsAddressGreaterThan") == 0) - { - result = NI_SRCS_UNSAFE_IsAddressGreaterThan; - } - else if (strcmp(methodName, "IsAddressLessThan") == 0) - { - result = NI_SRCS_UNSAFE_IsAddressLessThan; - } - else if (strcmp(methodName, "IsNullRef") == 0) - { - result = NI_SRCS_UNSAFE_IsNullRef; - } - else if (strcmp(methodName, "NullRef") == 0) - { - result = NI_SRCS_UNSAFE_NullRef; - } - else if (strcmp(methodName, "Read") == 0) - { - result = NI_SRCS_UNSAFE_Read; - } - else if (strcmp(methodName, "ReadUnaligned") == 0) - { - result = NI_SRCS_UNSAFE_ReadUnaligned; - } - else if (strcmp(methodName, "SizeOf") == 0) - { - result = NI_SRCS_UNSAFE_SizeOf; - } - else if (strcmp(methodName, "SkipInit") == 0) - { - result = NI_SRCS_UNSAFE_SkipInit; - } - else if (strcmp(methodName, "Subtract") == 0) - { - result = NI_SRCS_UNSAFE_Subtract; - } - else if (strcmp(methodName, "SubtractByteOffset") == 0) - { - result = NI_SRCS_UNSAFE_SubtractByteOffset; - } - else if (strcmp(methodName, "Unbox") == 0) - { - result = NI_SRCS_UNSAFE_Unbox; - } - else if (strcmp(methodName, "Write") == 0) - { - result = NI_SRCS_UNSAFE_Write; - } - else if (strcmp(methodName, "WriteUnaligned") == 0) - { - result = NI_SRCS_UNSAFE_WriteUnaligned; - } - } - else if (strcmp(className, "RuntimeHelpers") == 0) - { - if (strcmp(methodName, "CreateSpan") == 0) - { - result = NI_System_Runtime_CompilerServices_RuntimeHelpers_CreateSpan; - } - else if (strcmp(methodName, "InitializeArray") == 0) - { - result = NI_System_Runtime_CompilerServices_RuntimeHelpers_InitializeArray; - } - else if (strcmp(methodName, "IsKnownConstant") == 0) - { - result = NI_System_Runtime_CompilerServices_RuntimeHelpers_IsKnownConstant; - } - } - } - else if (strcmp(namespaceName, "System.Runtime.InteropServices") == 0) - { - if (strcmp(className, "MemoryMarshal") == 0) - { - if (strcmp(methodName, "GetArrayDataReference") == 0) - { - result = NI_System_Runtime_InteropService_MemoryMarshal_GetArrayDataReference; - } - } - } - else if (strncmp(namespaceName, "System.Runtime.Intrinsics", 25) == 0) - { - // We go down this path even when FEATURE_HW_INTRINSICS isn't enabled - // so we can specially handle IsSupported and recursive calls. - - // This is required to appropriately handle the intrinsics on platforms - // which don't support them. On such a platform methods like Vector64.Create - // will be seen as `Intrinsic` and `mustExpand` due to having a code path - // which is recursive. When such a path is hit we expect it to be handled by - // the importer and we fire an assert if it wasn't and in previous versions - // of the JIT would fail fast. This was changed to throw a PNSE instead but - // we still assert as most intrinsics should have been recognized/handled. - - // In order to avoid the assert, we specially handle the IsSupported checks - // (to better allow dead-code optimizations) and we explicitly throw a PNSE - // as we know that is the desired behavior for the HWIntrinsics when not - // supported. For cases like Vector64.Create, this is fine because it will - // be behind a relevant IsSupported check and will never be hit and the - // software fallback will be executed instead. - - CLANG_FORMAT_COMMENT_ANCHOR; - -#ifdef FEATURE_HW_INTRINSICS - namespaceName += 25; - const char* platformNamespaceName; - -#if defined(TARGET_XARCH) - platformNamespaceName = ".X86"; -#elif defined(TARGET_ARM64) - platformNamespaceName = ".Arm"; -#else -#error Unsupported platform -#endif - - if ((namespaceName[0] == '\0') || (strcmp(namespaceName, platformNamespaceName) == 0)) - { - CORINFO_SIG_INFO sig; - info.compCompHnd->getMethodSig(method, &sig); - - result = HWIntrinsicInfo::lookupId(this, &sig, className, methodName, enclosingClassName); - } -#endif // FEATURE_HW_INTRINSICS - - if (result == NI_Illegal) - { - if ((strcmp(methodName, "get_IsSupported") == 0) || (strcmp(methodName, "get_IsHardwareAccelerated") == 0)) - { - // This allows the relevant code paths to be dropped as dead code even - // on platforms where FEATURE_HW_INTRINSICS is not supported. - - result = NI_IsSupported_False; - } - else if (gtIsRecursiveCall(method)) - { - // For the framework itself, any recursive intrinsics will either be - // only supported on a single platform or will be guarded by a relevant - // IsSupported check so the throw PNSE will be valid or dropped. - - result = NI_Throw_PlatformNotSupportedException; - } - } - } - else if (strcmp(namespaceName, "System.StubHelpers") == 0) - { - if (strcmp(className, "StubHelpers") == 0) - { - if (strcmp(methodName, "GetStubContext") == 0) - { - result = NI_System_StubHelpers_GetStubContext; - } - else if (strcmp(methodName, "NextCallReturnAddress") == 0) - { - result = NI_System_StubHelpers_NextCallReturnAddress; - } - } - } - - if (result == NI_Illegal) - { - JITDUMP("Not recognized\n"); - } - else if (result == NI_IsSupported_False) - { - JITDUMP("Unsupported - return false"); - } - else if (result == NI_Throw_PlatformNotSupportedException) - { - JITDUMP("Unsupported - throw PlatformNotSupportedException"); - } - else - { - JITDUMP("Recognized\n"); - } - return result; -} - -//------------------------------------------------------------------------ -// impUnsupportedNamedIntrinsic: Throws an exception for an unsupported named intrinsic -// -// Arguments: -// helper - JIT helper ID for the exception to be thrown -// method - method handle of the intrinsic function. -// sig - signature of the intrinsic call -// mustExpand - true if the intrinsic must return a GenTree*; otherwise, false -// -// Return Value: -// a gtNewMustThrowException if mustExpand is true; otherwise, nullptr -// -GenTree* Compiler::impUnsupportedNamedIntrinsic(unsigned helper, - CORINFO_METHOD_HANDLE method, - CORINFO_SIG_INFO* sig, - bool mustExpand) -{ - // We've hit some error case and may need to return a node for the given error. - // - // When `mustExpand=false`, we are attempting to inline the intrinsic directly into another method. In this - // scenario, we need to return `nullptr` so that a GT_CALL to the intrinsic is emitted instead. This is to - // ensure that everything continues to behave correctly when optimizations are enabled (e.g. things like the - // inliner may expect the node we return to have a certain signature, and the `MustThrowException` node won't - // match that). - // - // When `mustExpand=true`, we are in a GT_CALL to the intrinsic and are attempting to JIT it. This will generally - // be in response to an indirect call (e.g. done via reflection) or in response to an earlier attempt returning - // `nullptr` (under `mustExpand=false`). In that scenario, we are safe to return the `MustThrowException` node. - - if (mustExpand) - { - for (unsigned i = 0; i < sig->numArgs; i++) - { - impPopStack(); - } - - return gtNewMustThrowException(helper, JITtype2varType(sig->retType), sig->retTypeClass); - } - else - { - return nullptr; - } -} - -//------------------------------------------------------------------------ -// impArrayAccessIntrinsic: try to replace a multi-dimensional array intrinsics with IR nodes. -// -// Arguments: -// clsHnd - handle for the intrinsic method's class -// sig - signature of the intrinsic method -// memberRef - the token for the intrinsic method -// readonlyCall - true if call has a readonly prefix -// intrinsicName - the intrinsic to expand: one of NI_Array_Address, NI_Array_Get, NI_Array_Set -// -// Return Value: -// The intrinsic expansion, or nullptr if the expansion was not done (and a function call should be made instead). -// -GenTree* Compiler::impArrayAccessIntrinsic( - CORINFO_CLASS_HANDLE clsHnd, CORINFO_SIG_INFO* sig, int memberRef, bool readonlyCall, NamedIntrinsic intrinsicName) -{ - assert((intrinsicName == NI_Array_Address) || (intrinsicName == NI_Array_Get) || (intrinsicName == NI_Array_Set)); - - // If we are generating SMALL_CODE, we don't want to use intrinsics, as it generates fatter code. - if (compCodeOpt() == SMALL_CODE) - { - JITDUMP("impArrayAccessIntrinsic: rejecting array intrinsic due to SMALL_CODE\n"); - return nullptr; - } - - unsigned rank = (intrinsicName == NI_Array_Set) ? (sig->numArgs - 1) : sig->numArgs; - - // Handle a maximum rank of GT_ARR_MAX_RANK (3). This is an implementation choice (larger ranks are expected - // to be rare) and could be increased. - if (rank > GT_ARR_MAX_RANK) - { - JITDUMP("impArrayAccessIntrinsic: rejecting array intrinsic because rank (%d) > GT_ARR_MAX_RANK (%d)\n", rank, - GT_ARR_MAX_RANK); - return nullptr; - } - - // The rank 1 case is special because it has to handle two array formats. We will simply not do that case. - if (rank <= 1) - { - JITDUMP("impArrayAccessIntrinsic: rejecting array intrinsic because rank (%d) <= 1\n", rank); - return nullptr; - } - - CORINFO_CLASS_HANDLE arrElemClsHnd = nullptr; - var_types elemType = JITtype2varType(info.compCompHnd->getChildType(clsHnd, &arrElemClsHnd)); - - // For the ref case, we will only be able to inline if the types match - // (verifier checks for this, we don't care for the nonverified case and the - // type is final (so we don't need to do the cast)) - if ((intrinsicName != NI_Array_Get) && !readonlyCall && varTypeIsGC(elemType)) - { - // Get the call site signature - CORINFO_SIG_INFO LocalSig; - eeGetCallSiteSig(memberRef, info.compScopeHnd, impTokenLookupContextHandle, &LocalSig); - assert(LocalSig.hasThis()); - - CORINFO_CLASS_HANDLE actualElemClsHnd; - - if (intrinsicName == NI_Array_Set) - { - // Fetch the last argument, the one that indicates the type we are setting. - CORINFO_ARG_LIST_HANDLE argType = LocalSig.args; - for (unsigned r = 0; r < rank; r++) - { - argType = info.compCompHnd->getArgNext(argType); - } - - typeInfo argInfo = verParseArgSigToTypeInfo(&LocalSig, argType); - actualElemClsHnd = argInfo.GetClassHandle(); - } - else - { - assert(intrinsicName == NI_Array_Address); - - // Fetch the return type - typeInfo retInfo = verMakeTypeInfo(LocalSig.retType, LocalSig.retTypeClass); - assert(retInfo.IsByRef()); - actualElemClsHnd = retInfo.GetClassHandle(); - } - - // if it's not final, we can't do the optimization - if (!(info.compCompHnd->getClassAttribs(actualElemClsHnd) & CORINFO_FLG_FINAL)) - { - JITDUMP("impArrayAccessIntrinsic: rejecting array intrinsic because actualElemClsHnd (%p) is not final\n", - dspPtr(actualElemClsHnd)); - return nullptr; - } - } - - unsigned arrayElemSize; - if (elemType == TYP_STRUCT) - { - assert(arrElemClsHnd); - arrayElemSize = info.compCompHnd->getClassSize(arrElemClsHnd); - } - else - { - arrayElemSize = genTypeSize(elemType); - } - - if ((unsigned char)arrayElemSize != arrayElemSize) - { - // arrayElemSize would be truncated as an unsigned char. - // This means the array element is too large. Don't do the optimization. - JITDUMP("impArrayAccessIntrinsic: rejecting array intrinsic because arrayElemSize (%d) is too large\n", - arrayElemSize); - return nullptr; - } - - GenTree* val = nullptr; - - if (intrinsicName == NI_Array_Set) - { - // Assignment of a struct is more work, and there are more gets than sets. - // TODO-CQ: support SET (`a[i,j,k] = s`) for struct element arrays. - if (elemType == TYP_STRUCT) - { - JITDUMP("impArrayAccessIntrinsic: rejecting SET array intrinsic because elemType is TYP_STRUCT" - " (implementation limitation)\n", - arrayElemSize); - return nullptr; - } - - val = impPopStack().val; - assert((genActualType(elemType) == genActualType(val->gtType)) || - (elemType == TYP_FLOAT && val->gtType == TYP_DOUBLE) || - (elemType == TYP_INT && val->gtType == TYP_BYREF) || - (elemType == TYP_DOUBLE && val->gtType == TYP_FLOAT)); - } - - // Here, we're committed to expanding the intrinsic and creating a GT_ARR_ELEM node. - optMethodFlags |= OMF_HAS_MDARRAYREF; - compCurBB->bbFlags |= BBF_HAS_MDARRAYREF; - - noway_assert((unsigned char)GT_ARR_MAX_RANK == GT_ARR_MAX_RANK); - - GenTree* inds[GT_ARR_MAX_RANK]; - for (unsigned k = rank; k > 0; k--) - { - // The indices should be converted to `int` type, as they would be if the intrinsic was not expanded. - GenTree* argVal = impPopStack().val; - if (impInlineRoot()->opts.compJitEarlyExpandMDArrays) - { - // This is only enabled when early MD expansion is set because it causes small - // asm diffs (only in some test cases) otherwise. The GT_ARR_ELEM lowering code "accidentally" does - // this cast, but the new code requires it to be explicit. - argVal = impImplicitIorI4Cast(argVal, TYP_INT); - } - inds[k - 1] = argVal; - } - - GenTree* arr = impPopStack().val; - assert(arr->gtType == TYP_REF); - - GenTree* arrElem = new (this, GT_ARR_ELEM) GenTreeArrElem(TYP_BYREF, arr, static_cast(rank), - static_cast(arrayElemSize), &inds[0]); - - if (intrinsicName != NI_Array_Address) - { - if (varTypeIsStruct(elemType)) - { - arrElem = gtNewObjNode(sig->retTypeClass, arrElem); - } - else - { - arrElem = gtNewOperNode(GT_IND, elemType, arrElem); - } - } - - if (intrinsicName == NI_Array_Set) - { - assert(val != nullptr); - return gtNewAssignNode(arrElem, val); - } - else - { - return arrElem; - } -} - -//------------------------------------------------------------------------ -// impKeepAliveIntrinsic: Import the GC.KeepAlive intrinsic call -// -// Imports the intrinsic as a GT_KEEPALIVE node, and, as an optimization, -// if the object to keep alive is a GT_BOX, removes its side effects and -// uses the address of a local (copied from the box's source if needed) -// as the operand for GT_KEEPALIVE. For the BOX optimization, if the class -// of the box has no GC fields, a GT_NOP is returned. -// -// Arguments: -// objToKeepAlive - the intrinisic call's argument -// -// Return Value: -// The imported GT_KEEPALIVE or GT_NOP - see description. -// -GenTree* Compiler::impKeepAliveIntrinsic(GenTree* objToKeepAlive) -{ - assert(objToKeepAlive->TypeIs(TYP_REF)); - - if (opts.OptimizationEnabled() && objToKeepAlive->IsBoxedValue()) - { - CORINFO_CLASS_HANDLE boxedClass = lvaGetDesc(objToKeepAlive->AsBox()->BoxOp()->AsLclVar())->lvClassHnd; - ClassLayout* layout = typGetObjLayout(boxedClass); - - if (!layout->HasGCPtr()) - { - gtTryRemoveBoxUpstreamEffects(objToKeepAlive, BR_REMOVE_AND_NARROW); - JITDUMP("\nBOX class has no GC fields, KEEPALIVE is a NOP"); - - return gtNewNothingNode(); - } - - GenTree* boxSrc = gtTryRemoveBoxUpstreamEffects(objToKeepAlive, BR_REMOVE_BUT_NOT_NARROW); - if (boxSrc != nullptr) - { - unsigned boxTempNum; - if (boxSrc->OperIs(GT_LCL_VAR)) - { - boxTempNum = boxSrc->AsLclVarCommon()->GetLclNum(); - } - else - { - boxTempNum = lvaGrabTemp(true DEBUGARG("Temp for the box source")); - GenTree* boxTempAsg = gtNewTempAssign(boxTempNum, boxSrc); - Statement* boxAsgStmt = objToKeepAlive->AsBox()->gtCopyStmtWhenInlinedBoxValue; - boxAsgStmt->SetRootNode(boxTempAsg); - } - - JITDUMP("\nImporting KEEPALIVE(BOX) as KEEPALIVE(ADDR(LCL_VAR V%02u))", boxTempNum); - - GenTree* boxTemp = gtNewLclvNode(boxTempNum, boxSrc->TypeGet()); - GenTree* boxTempAddr = gtNewOperNode(GT_ADDR, TYP_BYREF, boxTemp); - - return gtNewKeepAliveNode(boxTempAddr); - } - } - - return gtNewKeepAliveNode(objToKeepAlive); -} - -bool Compiler::verMergeEntryStates(BasicBlock* block, bool* changed) -{ - unsigned i; - - // do some basic checks first - if (block->bbStackDepthOnEntry() != verCurrentState.esStackDepth) - { - return false; - } - - if (verCurrentState.esStackDepth > 0) - { - // merge stack types - StackEntry* parentStack = block->bbStackOnEntry(); - StackEntry* childStack = verCurrentState.esStack; - - for (i = 0; i < verCurrentState.esStackDepth; i++, parentStack++, childStack++) - { - if (tiMergeToCommonParent(&parentStack->seTypeInfo, &childStack->seTypeInfo, changed) == false) - { - return false; - } - } - } - - // merge initialization status of this ptr - - if (verTrackObjCtorInitState) - { - // If we're tracking the CtorInitState, then it must not be unknown in the current state. - assert(verCurrentState.thisInitialized != TIS_Bottom); - - // If the successor block's thisInit state is unknown, copy it from the current state. - if (block->bbThisOnEntry() == TIS_Bottom) - { - *changed = true; - verSetThisInit(block, verCurrentState.thisInitialized); - } - else if (verCurrentState.thisInitialized != block->bbThisOnEntry()) - { - if (block->bbThisOnEntry() != TIS_Top) - { - *changed = true; - verSetThisInit(block, TIS_Top); - - if (block->bbFlags & BBF_FAILED_VERIFICATION) - { - // The block is bad. Control can flow through the block to any handler that catches the - // verification exception, but the importer ignores bad blocks and therefore won't model - // this flow in the normal way. To complete the merge into the bad block, the new state - // needs to be manually pushed to the handlers that may be reached after the verification - // exception occurs. - // - // Usually, the new state was already propagated to the relevant handlers while processing - // the predecessors of the bad block. The exception is when the bad block is at the start - // of a try region, meaning it is protected by additional handlers that do not protect its - // predecessors. - // - if (block->hasTryIndex() && ((block->bbFlags & BBF_TRY_BEG) != 0)) - { - // Push TIS_Top to the handlers that protect the bad block. Note that this can cause - // recursive calls back into this code path (if successors of the current bad block are - // also bad blocks). - // - ThisInitState origTIS = verCurrentState.thisInitialized; - verCurrentState.thisInitialized = TIS_Top; - impVerifyEHBlock(block, true); - verCurrentState.thisInitialized = origTIS; - } - } - } - } - } - else - { - assert(verCurrentState.thisInitialized == TIS_Bottom && block->bbThisOnEntry() == TIS_Bottom); - } - - return true; -} - -/***************************************************************************** - * 'logMsg' is true if a log message needs to be logged. false if the caller has - * already logged it (presumably in a more detailed fashion than done here) - */ - -void Compiler::verConvertBBToThrowVerificationException(BasicBlock* block DEBUGARG(bool logMsg)) -{ - block->bbJumpKind = BBJ_THROW; - block->bbFlags |= BBF_FAILED_VERIFICATION; - block->bbFlags &= ~BBF_IMPORTED; - - impCurStmtOffsSet(block->bbCodeOffs); - - // Clear the statement list as it exists so far; we're only going to have a verification exception. - impStmtList = impLastStmt = nullptr; - -#ifdef DEBUG - if (logMsg) - { - JITLOG((LL_ERROR, "Verification failure: while compiling %s near IL offset %x..%xh \n", info.compFullName, - block->bbCodeOffs, block->bbCodeOffsEnd)); - if (verbose) - { - printf("\n\nVerification failure: %s near IL %xh \n", info.compFullName, block->bbCodeOffs); - } - } - - if (JitConfig.DebugBreakOnVerificationFailure()) - { - DebugBreak(); - } -#endif - - impBeginTreeList(); - - // if the stack is non-empty evaluate all the side-effects - if (verCurrentState.esStackDepth > 0) - { - impEvalSideEffects(); - } - assert(verCurrentState.esStackDepth == 0); - - GenTree* op1 = gtNewHelperCallNode(CORINFO_HELP_VERIFICATION, TYP_VOID, gtNewIconNode(block->bbCodeOffs)); - // verCurrentState.esStackDepth = 0; - impAppendTree(op1, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); - - // The inliner is not able to handle methods that require throw block, so - // make sure this methods never gets inlined. - info.compCompHnd->setMethodAttribs(info.compMethodHnd, CORINFO_FLG_BAD_INLINEE); -} - -/***************************************************************************** - * - */ -void Compiler::verHandleVerificationFailure(BasicBlock* block DEBUGARG(bool logMsg)) -{ - verResetCurrentState(block, &verCurrentState); - verConvertBBToThrowVerificationException(block DEBUGARG(logMsg)); - -#ifdef DEBUG - impNoteLastILoffs(); // Remember at which BC offset the tree was finished -#endif // DEBUG -} - -/******************************************************************************/ -typeInfo Compiler::verMakeTypeInfo(CorInfoType ciType, CORINFO_CLASS_HANDLE clsHnd) -{ - assert(ciType < CORINFO_TYPE_COUNT); - - typeInfo tiResult; - switch (ciType) - { - case CORINFO_TYPE_STRING: - case CORINFO_TYPE_CLASS: - tiResult = verMakeTypeInfo(clsHnd); - if (!tiResult.IsType(TI_REF)) - { // type must be consistent with element type - return typeInfo(); - } - break; - -#ifdef TARGET_64BIT - case CORINFO_TYPE_NATIVEINT: - case CORINFO_TYPE_NATIVEUINT: - if (clsHnd) - { - // If we have more precise information, use it - return verMakeTypeInfo(clsHnd); - } - else - { - return typeInfo::nativeInt(); - } - break; -#endif // TARGET_64BIT - - case CORINFO_TYPE_VALUECLASS: - case CORINFO_TYPE_REFANY: - tiResult = verMakeTypeInfo(clsHnd); - // type must be constant with element type; - if (!tiResult.IsValueClass()) - { - return typeInfo(); - } - break; - case CORINFO_TYPE_VAR: - return verMakeTypeInfo(clsHnd); - - case CORINFO_TYPE_PTR: // for now, pointers are treated as an error - case CORINFO_TYPE_VOID: - return typeInfo(); - break; - - case CORINFO_TYPE_BYREF: - { - CORINFO_CLASS_HANDLE childClassHandle; - CorInfoType childType = info.compCompHnd->getChildType(clsHnd, &childClassHandle); - return ByRef(verMakeTypeInfo(childType, childClassHandle)); - } - break; - - default: - if (clsHnd) - { // If we have more precise information, use it - return typeInfo(TI_STRUCT, clsHnd); - } - else - { - return typeInfo(JITtype2tiType(ciType)); - } - } - return tiResult; -} - -/******************************************************************************/ - -typeInfo Compiler::verMakeTypeInfo(CORINFO_CLASS_HANDLE clsHnd, bool bashStructToRef /* = false */) -{ - if (clsHnd == nullptr) - { - return typeInfo(); - } - - // Byrefs should only occur in method and local signatures, which are accessed - // using ICorClassInfo and ICorClassInfo.getChildType. - // So findClass() and getClassAttribs() should not be called for byrefs - - if (JITtype2varType(info.compCompHnd->asCorInfoType(clsHnd)) == TYP_BYREF) - { - assert(!"Did findClass() return a Byref?"); - return typeInfo(); - } - - unsigned attribs = info.compCompHnd->getClassAttribs(clsHnd); - - if (attribs & CORINFO_FLG_VALUECLASS) - { - CorInfoType t = info.compCompHnd->getTypeForPrimitiveValueClass(clsHnd); - - // Meta-data validation should ensure that CORINF_TYPE_BYREF should - // not occur here, so we may want to change this to an assert instead. - if (t == CORINFO_TYPE_VOID || t == CORINFO_TYPE_BYREF || t == CORINFO_TYPE_PTR) - { - return typeInfo(); - } - -#ifdef TARGET_64BIT - if (t == CORINFO_TYPE_NATIVEINT || t == CORINFO_TYPE_NATIVEUINT) - { - return typeInfo::nativeInt(); - } -#endif // TARGET_64BIT - - if (t != CORINFO_TYPE_UNDEF) - { - return (typeInfo(JITtype2tiType(t))); - } - else if (bashStructToRef) - { - return (typeInfo(TI_REF, clsHnd)); - } - else - { - return (typeInfo(TI_STRUCT, clsHnd)); - } - } - else if (attribs & CORINFO_FLG_GENERIC_TYPE_VARIABLE) - { - // See comment in _typeInfo.h for why we do it this way. - return (typeInfo(TI_REF, clsHnd, true)); - } - else - { - return (typeInfo(TI_REF, clsHnd)); - } -} - -/******************************************************************************/ -bool Compiler::verIsSDArray(const typeInfo& ti) -{ - if (ti.IsNullObjRef()) - { // nulls are SD arrays - return true; - } - - if (!ti.IsType(TI_REF)) - { - return false; - } - - if (!info.compCompHnd->isSDArray(ti.GetClassHandleForObjRef())) - { - return false; - } - return true; -} - -/******************************************************************************/ -/* Given 'arrayObjectType' which is an array type, fetch the element type. */ -/* Returns an error type if anything goes wrong */ - -typeInfo Compiler::verGetArrayElemType(const typeInfo& arrayObjectType) -{ - assert(!arrayObjectType.IsNullObjRef()); // you need to check for null explicitly since that is a success case - - if (!verIsSDArray(arrayObjectType)) - { - return typeInfo(); - } - - CORINFO_CLASS_HANDLE childClassHandle = nullptr; - CorInfoType ciType = info.compCompHnd->getChildType(arrayObjectType.GetClassHandleForObjRef(), &childClassHandle); - - return verMakeTypeInfo(ciType, childClassHandle); -} - -/***************************************************************************** - */ -typeInfo Compiler::verParseArgSigToTypeInfo(CORINFO_SIG_INFO* sig, CORINFO_ARG_LIST_HANDLE args) -{ - CORINFO_CLASS_HANDLE classHandle; - CorInfoType ciType = strip(info.compCompHnd->getArgType(sig, args, &classHandle)); - - var_types type = JITtype2varType(ciType); - if (varTypeIsGC(type)) - { - // For efficiency, getArgType only returns something in classHandle for - // value types. For other types that have addition type info, you - // have to call back explicitly - classHandle = info.compCompHnd->getArgClass(sig, args); - if (!classHandle) - { - NO_WAY("Could not figure out Class specified in argument or local signature"); - } - } - - return verMakeTypeInfo(ciType, classHandle); -} - -bool Compiler::verIsByRefLike(const typeInfo& ti) -{ - if (ti.IsByRef()) - { - return true; - } - if (!ti.IsType(TI_STRUCT)) - { - return false; - } - return info.compCompHnd->getClassAttribs(ti.GetClassHandleForValueClass()) & CORINFO_FLG_BYREF_LIKE; -} - -bool Compiler::verIsSafeToReturnByRef(const typeInfo& ti) -{ - if (ti.IsPermanentHomeByRef()) - { - return true; - } - else - { - return false; - } -} - -bool Compiler::verIsBoxable(const typeInfo& ti) -{ - return (ti.IsPrimitiveType() || ti.IsObjRef() // includes boxed generic type variables - || ti.IsUnboxedGenericTypeVar() || - (ti.IsType(TI_STRUCT) && - // exclude byreflike structs - !(info.compCompHnd->getClassAttribs(ti.GetClassHandleForValueClass()) & CORINFO_FLG_BYREF_LIKE))); -} - -// Is it a boxed value type? -bool Compiler::verIsBoxedValueType(const typeInfo& ti) -{ - if (ti.GetType() == TI_REF) - { - CORINFO_CLASS_HANDLE clsHnd = ti.GetClassHandleForObjRef(); - return !!eeIsValueClass(clsHnd); - } - else - { - return false; - } -} - -/***************************************************************************** - * - * Check if a TailCall is legal. - */ - -bool Compiler::verCheckTailCallConstraint( - OPCODE opcode, - CORINFO_RESOLVED_TOKEN* pResolvedToken, - CORINFO_RESOLVED_TOKEN* pConstrainedResolvedToken, // Is this a "constrained." call on a type parameter? - bool speculative // If true, won't throw if verificatoin fails. Instead it will - // return false to the caller. - // If false, it will throw. - ) -{ - DWORD mflags; - CORINFO_SIG_INFO sig; - unsigned int popCount = 0; // we can't pop the stack since impImportCall needs it, so - // this counter is used to keep track of how many items have been - // virtually popped - - CORINFO_METHOD_HANDLE methodHnd = nullptr; - CORINFO_CLASS_HANDLE methodClassHnd = nullptr; - unsigned methodClassFlgs = 0; - - assert(impOpcodeIsCallOpcode(opcode)); - - if (compIsForInlining()) - { - return false; - } - - // for calli, VerifyOrReturn that this is not a virtual method - if (opcode == CEE_CALLI) - { - /* Get the call sig */ - eeGetSig(pResolvedToken->token, pResolvedToken->tokenScope, pResolvedToken->tokenContext, &sig); - - // We don't know the target method, so we have to infer the flags, or - // assume the worst-case. - mflags = (sig.callConv & CORINFO_CALLCONV_HASTHIS) ? 0 : CORINFO_FLG_STATIC; - } - else - { - methodHnd = pResolvedToken->hMethod; - - mflags = info.compCompHnd->getMethodAttribs(methodHnd); - - // When verifying generic code we pair the method handle with its - // owning class to get the exact method signature. - methodClassHnd = pResolvedToken->hClass; - assert(methodClassHnd); - - eeGetMethodSig(methodHnd, &sig, methodClassHnd); - - // opcode specific check - methodClassFlgs = info.compCompHnd->getClassAttribs(methodClassHnd); - } - - // We must have got the methodClassHnd if opcode is not CEE_CALLI - assert((methodHnd != nullptr && methodClassHnd != nullptr) || opcode == CEE_CALLI); - - if ((sig.callConv & CORINFO_CALLCONV_MASK) == CORINFO_CALLCONV_VARARG) - { - eeGetCallSiteSig(pResolvedToken->token, pResolvedToken->tokenScope, pResolvedToken->tokenContext, &sig); - } - - // check compatibility of the arguments - unsigned int argCount; - argCount = sig.numArgs; - CORINFO_ARG_LIST_HANDLE args; - args = sig.args; - while (argCount--) - { - typeInfo tiDeclared = verParseArgSigToTypeInfo(&sig, args).NormaliseForStack(); - - // check that the argument is not a byref for tailcalls - VerifyOrReturnSpeculative(!verIsByRefLike(tiDeclared), "tailcall on byrefs", speculative); - - // For unsafe code, we might have parameters containing pointer to the stack location. - // Disallow the tailcall for this kind. - CORINFO_CLASS_HANDLE classHandle; - CorInfoType ciType = strip(info.compCompHnd->getArgType(&sig, args, &classHandle)); - VerifyOrReturnSpeculative(ciType != CORINFO_TYPE_PTR, "tailcall on CORINFO_TYPE_PTR", speculative); - - args = info.compCompHnd->getArgNext(args); - } - - // update popCount - popCount += sig.numArgs; - - // check for 'this' which is on non-static methods, not called via NEWOBJ - if (!(mflags & CORINFO_FLG_STATIC)) - { - // Always update the popCount. - // This is crucial for the stack calculation to be correct. - typeInfo tiThis = impStackTop(popCount).seTypeInfo; - popCount++; - - if (opcode == CEE_CALLI) - { - // For CALLI, we don't know the methodClassHnd. Therefore, let's check the "this" object - // on the stack. - if (tiThis.IsValueClass()) - { - tiThis.MakeByRef(); - } - VerifyOrReturnSpeculative(!verIsByRefLike(tiThis), "byref in tailcall", speculative); - } - else - { - // Check type compatibility of the this argument - typeInfo tiDeclaredThis = verMakeTypeInfo(methodClassHnd); - if (tiDeclaredThis.IsValueClass()) - { - tiDeclaredThis.MakeByRef(); - } - - VerifyOrReturnSpeculative(!verIsByRefLike(tiDeclaredThis), "byref in tailcall", speculative); - } - } - - // Tail calls on constrained calls should be illegal too: - // when instantiated at a value type, a constrained call may pass the address of a stack allocated value - VerifyOrReturnSpeculative(!pConstrainedResolvedToken, "byref in constrained tailcall", speculative); - - // Get the exact view of the signature for an array method - if (sig.retType != CORINFO_TYPE_VOID) - { - if (methodClassFlgs & CORINFO_FLG_ARRAY) - { - assert(opcode != CEE_CALLI); - eeGetCallSiteSig(pResolvedToken->token, pResolvedToken->tokenScope, pResolvedToken->tokenContext, &sig); - } - } - - typeInfo tiCalleeRetType = verMakeTypeInfo(sig.retType, sig.retTypeClass); - typeInfo tiCallerRetType = - verMakeTypeInfo(info.compMethodInfo->args.retType, info.compMethodInfo->args.retTypeClass); - - // void return type gets morphed into the error type, so we have to treat them specially here - if (sig.retType == CORINFO_TYPE_VOID) - { - VerifyOrReturnSpeculative(info.compMethodInfo->args.retType == CORINFO_TYPE_VOID, "tailcall return mismatch", - speculative); - } - else - { - VerifyOrReturnSpeculative(tiCompatibleWith(NormaliseForStack(tiCalleeRetType), - NormaliseForStack(tiCallerRetType), true), - "tailcall return mismatch", speculative); - } - - // for tailcall, stack must be empty - VerifyOrReturnSpeculative(verCurrentState.esStackDepth == popCount, "stack non-empty on tailcall", speculative); - - return true; // Yes, tailcall is legal -} - -/***************************************************************************** - * - * Checks the IL verification rules for the call - */ - -void Compiler::verVerifyCall(OPCODE opcode, - CORINFO_RESOLVED_TOKEN* pResolvedToken, - CORINFO_RESOLVED_TOKEN* pConstrainedResolvedToken, - bool tailCall, - bool readonlyCall, - const BYTE* delegateCreateStart, - const BYTE* codeAddr, - CORINFO_CALL_INFO* callInfo DEBUGARG(const char* methodName)) -{ - DWORD mflags; - CORINFO_SIG_INFO* sig = nullptr; - unsigned int popCount = 0; // we can't pop the stack since impImportCall needs it, so - // this counter is used to keep track of how many items have been - // virtually popped - - // for calli, VerifyOrReturn that this is not a virtual method - if (opcode == CEE_CALLI) - { - Verify(false, "Calli not verifiable"); - return; - } - - // It would be nice to cache the rest of it, but eeFindMethod is the big ticket item. - mflags = callInfo->verMethodFlags; - - sig = &callInfo->verSig; - - if ((sig->callConv & CORINFO_CALLCONV_MASK) == CORINFO_CALLCONV_VARARG) - { - eeGetCallSiteSig(pResolvedToken->token, pResolvedToken->tokenScope, pResolvedToken->tokenContext, sig); - } - - // opcode specific check - unsigned methodClassFlgs = callInfo->classFlags; - switch (opcode) - { - case CEE_CALLVIRT: - // cannot do callvirt on valuetypes - VerifyOrReturn(!(methodClassFlgs & CORINFO_FLG_VALUECLASS), "callVirt on value class"); - VerifyOrReturn(sig->hasThis(), "CallVirt on static method"); - break; - - case CEE_NEWOBJ: - { - assert(!tailCall); // Importer should not allow this - VerifyOrReturn((mflags & CORINFO_FLG_CONSTRUCTOR) && !(mflags & CORINFO_FLG_STATIC), - "newobj must be on instance"); - - if (methodClassFlgs & CORINFO_FLG_DELEGATE) - { - VerifyOrReturn(sig->numArgs == 2, "wrong number args to delegate ctor"); - typeInfo tiDeclaredObj = verParseArgSigToTypeInfo(sig, sig->args).NormaliseForStack(); - typeInfo tiDeclaredFtn = - verParseArgSigToTypeInfo(sig, info.compCompHnd->getArgNext(sig->args)).NormaliseForStack(); - VerifyOrReturn(tiDeclaredFtn.IsNativeIntType(), "ftn arg needs to be a native int type"); - - assert(popCount == 0); - typeInfo tiActualObj = impStackTop(1).seTypeInfo; - typeInfo tiActualFtn = impStackTop(0).seTypeInfo; - - VerifyOrReturn(tiActualFtn.IsMethod(), "delegate needs method as first arg"); - VerifyOrReturn(tiCompatibleWith(tiActualObj, tiDeclaredObj, true), "delegate object type mismatch"); - VerifyOrReturn(tiActualObj.IsNullObjRef() || tiActualObj.IsType(TI_REF), - "delegate object type mismatch"); - - CORINFO_CLASS_HANDLE objTypeHandle = - tiActualObj.IsNullObjRef() ? nullptr : tiActualObj.GetClassHandleForObjRef(); - - // the method signature must be compatible with the delegate's invoke method - - // check that for virtual functions, the type of the object used to get the - // ftn ptr is the same as the type of the object passed to the delegate ctor. - // since this is a bit of work to determine in general, we pattern match stylized - // code sequences - - // the delegate creation code check, which used to be done later, is now done here - // so we can read delegateMethodRef directly from - // from the preceding LDFTN or CEE_LDVIRTFN instruction sequence; - // we then use it in our call to isCompatibleDelegate(). - - mdMemberRef delegateMethodRef = mdMemberRefNil; - VerifyOrReturn(verCheckDelegateCreation(delegateCreateStart, codeAddr, delegateMethodRef), - "must create delegates with certain IL"); - - CORINFO_RESOLVED_TOKEN delegateResolvedToken; - delegateResolvedToken.tokenContext = impTokenLookupContextHandle; - delegateResolvedToken.tokenScope = info.compScopeHnd; - delegateResolvedToken.token = delegateMethodRef; - delegateResolvedToken.tokenType = CORINFO_TOKENKIND_Method; - info.compCompHnd->resolveToken(&delegateResolvedToken); - - CORINFO_CALL_INFO delegateCallInfo; - eeGetCallInfo(&delegateResolvedToken, nullptr /* constraint typeRef */, CORINFO_CALLINFO_SECURITYCHECKS, - &delegateCallInfo); - - bool isOpenDelegate = false; - VerifyOrReturn(info.compCompHnd->isCompatibleDelegate(objTypeHandle, delegateResolvedToken.hClass, - tiActualFtn.GetMethod(), pResolvedToken->hClass, - &isOpenDelegate), - "function incompatible with delegate"); - - // check the constraints on the target method - VerifyOrReturn(info.compCompHnd->satisfiesClassConstraints(delegateResolvedToken.hClass), - "delegate target has unsatisfied class constraints"); - VerifyOrReturn(info.compCompHnd->satisfiesMethodConstraints(delegateResolvedToken.hClass, - tiActualFtn.GetMethod()), - "delegate target has unsatisfied method constraints"); - - // See ECMA spec section 1.8.1.5.2 (Delegating via instance dispatch) - // for additional verification rules for delegates - CORINFO_METHOD_HANDLE actualMethodHandle = tiActualFtn.GetMethod(); - DWORD actualMethodAttribs = info.compCompHnd->getMethodAttribs(actualMethodHandle); - if (impIsLDFTN_TOKEN(delegateCreateStart, codeAddr)) - { - - if ((actualMethodAttribs & CORINFO_FLG_VIRTUAL) && ((actualMethodAttribs & CORINFO_FLG_FINAL) == 0)) - { - VerifyOrReturn((tiActualObj.IsThisPtr() && lvaIsOriginalThisReadOnly()) || - verIsBoxedValueType(tiActualObj), - "The 'this' parameter to the call must be either the calling method's " - "'this' parameter or " - "a boxed value type."); - } - } - - if (actualMethodAttribs & CORINFO_FLG_PROTECTED) - { - bool targetIsStatic = actualMethodAttribs & CORINFO_FLG_STATIC; - - Verify(targetIsStatic || !isOpenDelegate, - "Unverifiable creation of an open instance delegate for a protected member."); - - CORINFO_CLASS_HANDLE instanceClassHnd = (tiActualObj.IsNullObjRef() || targetIsStatic) - ? info.compClassHnd - : tiActualObj.GetClassHandleForObjRef(); - - // In the case of protected methods, it is a requirement that the 'this' - // pointer be a subclass of the current context. Perform this check. - Verify(info.compCompHnd->canAccessFamily(info.compMethodHnd, instanceClassHnd), - "Accessing protected method through wrong type."); - } - goto DONE_ARGS; - } - } - // fall thru to default checks - FALLTHROUGH; - default: - VerifyOrReturn(!(mflags & CORINFO_FLG_ABSTRACT), "method abstract"); - } - VerifyOrReturn(!((mflags & CORINFO_FLG_CONSTRUCTOR) && (methodClassFlgs & CORINFO_FLG_DELEGATE)), - "can only newobj a delegate constructor"); - - // check compatibility of the arguments - unsigned int argCount; - argCount = sig->numArgs; - CORINFO_ARG_LIST_HANDLE args; - args = sig->args; - while (argCount--) - { - typeInfo tiActual = impStackTop(popCount + argCount).seTypeInfo; - - typeInfo tiDeclared = verParseArgSigToTypeInfo(sig, args).NormaliseForStack(); - VerifyOrReturn(tiCompatibleWith(tiActual, tiDeclared, true), "type mismatch"); - - args = info.compCompHnd->getArgNext(args); - } - -DONE_ARGS: - - // update popCount - popCount += sig->numArgs; - - // check for 'this' which are is non-static methods, not called via NEWOBJ - CORINFO_CLASS_HANDLE instanceClassHnd = info.compClassHnd; - if (!(mflags & CORINFO_FLG_STATIC) && (opcode != CEE_NEWOBJ)) - { - typeInfo tiThis = impStackTop(popCount).seTypeInfo; - popCount++; - - // If it is null, we assume we can access it (since it will AV shortly) - // If it is anything but a reference class, there is no hierarchy, so - // again, we don't need the precise instance class to compute 'protected' access - if (tiThis.IsType(TI_REF)) - { - instanceClassHnd = tiThis.GetClassHandleForObjRef(); - } - - // Check type compatibility of the this argument - typeInfo tiDeclaredThis = verMakeTypeInfo(pResolvedToken->hClass); - if (tiDeclaredThis.IsValueClass()) - { - tiDeclaredThis.MakeByRef(); - } - - // If this is a call to the base class .ctor, set thisPtr Init for - // this block. - if (mflags & CORINFO_FLG_CONSTRUCTOR) - { - if (verTrackObjCtorInitState && tiThis.IsThisPtr() && - verIsCallToInitThisPtr(info.compClassHnd, pResolvedToken->hClass)) - { - assert(verCurrentState.thisInitialized != - TIS_Bottom); // This should never be the case just from the logic of the verifier. - VerifyOrReturn(verCurrentState.thisInitialized == TIS_Uninit, - "Call to base class constructor when 'this' is possibly initialized"); - // Otherwise, 'this' is now initialized. - verCurrentState.thisInitialized = TIS_Init; - tiThis.SetInitialisedObjRef(); - } - else - { - // We allow direct calls to value type constructors - // NB: we have to check that the contents of tiThis is a value type, otherwise we could use a - // constrained callvirt to illegally re-enter a .ctor on a value of reference type. - VerifyOrReturn(tiThis.IsByRef() && DereferenceByRef(tiThis).IsValueClass(), - "Bad call to a constructor"); - } - } - - if (pConstrainedResolvedToken != nullptr) - { - VerifyOrReturn(tiThis.IsByRef(), "non-byref this type in constrained call"); - - typeInfo tiConstraint = verMakeTypeInfo(pConstrainedResolvedToken->hClass); - - // We just dereference this and test for equality - tiThis.DereferenceByRef(); - VerifyOrReturn(typeInfo::AreEquivalent(tiThis, tiConstraint), - "this type mismatch with constrained type operand"); - - // Now pretend the this type is the boxed constrained type, for the sake of subsequent checks - tiThis = typeInfo(TI_REF, pConstrainedResolvedToken->hClass); - } - - // To support direct calls on readonly byrefs, just pretend tiDeclaredThis is readonly too - if (tiDeclaredThis.IsByRef() && tiThis.IsReadonlyByRef()) - { - tiDeclaredThis.SetIsReadonlyByRef(); - } - - VerifyOrReturn(tiCompatibleWith(tiThis, tiDeclaredThis, true), "this type mismatch"); - - if (tiThis.IsByRef()) - { - // Find the actual type where the method exists (as opposed to what is declared - // in the metadata). This is to prevent passing a byref as the "this" argument - // while calling methods like System.ValueType.GetHashCode() which expect boxed objects. - - CORINFO_CLASS_HANDLE actualClassHnd = info.compCompHnd->getMethodClass(pResolvedToken->hMethod); - VerifyOrReturn(eeIsValueClass(actualClassHnd), - "Call to base type of valuetype (which is never a valuetype)"); - } - - // Rules for non-virtual call to a non-final virtual method: - - // Define: - // The "this" pointer is considered to be "possibly written" if - // 1. Its address have been taken (LDARGA 0) anywhere in the method. - // (or) - // 2. It has been stored to (STARG.0) anywhere in the method. - - // A non-virtual call to a non-final virtual method is only allowed if - // 1. The this pointer passed to the callee is an instance of a boxed value type. - // (or) - // 2. The this pointer passed to the callee is the current method's this pointer. - // (and) The current method's this pointer is not "possibly written". - - // Thus the rule is that if you assign to this ANYWHERE you can't make "base" calls to - // virtual methods. (Luckily this does affect .ctors, since they are not virtual). - // This is stronger that is strictly needed, but implementing a laxer rule is significantly - // hard and more error prone. - - if (opcode == CEE_CALL && (mflags & CORINFO_FLG_VIRTUAL) && ((mflags & CORINFO_FLG_FINAL) == 0)) - { - VerifyOrReturn((tiThis.IsThisPtr() && lvaIsOriginalThisReadOnly()) || verIsBoxedValueType(tiThis), - "The 'this' parameter to the call must be either the calling method's 'this' parameter or " - "a boxed value type."); - } - } - - // check any constraints on the callee's class and type parameters - VerifyOrReturn(info.compCompHnd->satisfiesClassConstraints(pResolvedToken->hClass), - "method has unsatisfied class constraints"); - VerifyOrReturn(info.compCompHnd->satisfiesMethodConstraints(pResolvedToken->hClass, pResolvedToken->hMethod), - "method has unsatisfied method constraints"); - - if (mflags & CORINFO_FLG_PROTECTED) - { - VerifyOrReturn(info.compCompHnd->canAccessFamily(info.compMethodHnd, instanceClassHnd), - "Can't access protected method"); - } - - // Get the exact view of the signature for an array method - if (sig->retType != CORINFO_TYPE_VOID) - { - eeGetMethodSig(pResolvedToken->hMethod, sig, pResolvedToken->hClass); - } - - // "readonly." prefixed calls only allowed for the Address operation on arrays. - // The methods supported by array types are under the control of the EE - // so we can trust that only the Address operation returns a byref. - if (readonlyCall) - { - typeInfo tiCalleeRetType = verMakeTypeInfo(sig->retType, sig->retTypeClass); - VerifyOrReturn((methodClassFlgs & CORINFO_FLG_ARRAY) && tiCalleeRetType.IsByRef(), - "unexpected use of readonly prefix"); - } - - // Verify the tailcall - if (tailCall) - { - verCheckTailCallConstraint(opcode, pResolvedToken, pConstrainedResolvedToken, false); - } -} - -/***************************************************************************** - * Checks that a delegate creation is done using the following pattern: - * dup - * ldvirtftn targetMemberRef - * OR - * ldftn targetMemberRef - * - * 'delegateCreateStart' points at the last dup or ldftn in this basic block (null if - * not in this basic block) - * - * targetMemberRef is read from the code sequence. - * targetMemberRef is validated iff verificationNeeded. - */ - -bool Compiler::verCheckDelegateCreation(const BYTE* delegateCreateStart, - const BYTE* codeAddr, - mdMemberRef& targetMemberRef) -{ - if (impIsLDFTN_TOKEN(delegateCreateStart, codeAddr)) - { - targetMemberRef = getU4LittleEndian(&delegateCreateStart[2]); - return true; - } - else if (impIsDUP_LDVIRTFTN_TOKEN(delegateCreateStart, codeAddr)) - { - targetMemberRef = getU4LittleEndian(&delegateCreateStart[3]); - return true; - } - - return false; -} - -typeInfo Compiler::verVerifySTIND(const typeInfo& tiTo, const typeInfo& value, const typeInfo& instrType) -{ - Verify(!tiTo.IsReadonlyByRef(), "write to readonly byref"); - typeInfo ptrVal = verVerifyLDIND(tiTo, instrType); - typeInfo normPtrVal = typeInfo(ptrVal).NormaliseForStack(); - if (!tiCompatibleWith(value, normPtrVal, true)) - { - Verify(tiCompatibleWith(value, normPtrVal, true), "type mismatch"); - } - return ptrVal; -} - -typeInfo Compiler::verVerifyLDIND(const typeInfo& ptr, const typeInfo& instrType) -{ - assert(!instrType.IsStruct()); - - typeInfo ptrVal; - if (ptr.IsByRef()) - { - ptrVal = DereferenceByRef(ptr); - if (instrType.IsObjRef() && !ptrVal.IsObjRef()) - { - Verify(false, "bad pointer"); - } - else if (!instrType.IsObjRef() && !typeInfo::AreEquivalent(instrType, ptrVal)) - { - Verify(false, "pointer not consistent with instr"); - } - } - else - { - Verify(false, "pointer not byref"); - } - - return ptrVal; -} - -// Verify that the field is used properly. 'tiThis' is NULL for statics, -// 'fieldFlags' is the fields attributes, and mutator is true if it is a -// ld*flda or a st*fld. -// 'enclosingClass' is given if we are accessing a field in some specific type. - -void Compiler::verVerifyField(CORINFO_RESOLVED_TOKEN* pResolvedToken, - const CORINFO_FIELD_INFO& fieldInfo, - const typeInfo* tiThis, - bool mutator, - bool allowPlainStructAsThis) -{ - CORINFO_CLASS_HANDLE enclosingClass = pResolvedToken->hClass; - unsigned fieldFlags = fieldInfo.fieldFlags; - CORINFO_CLASS_HANDLE instanceClass = - info.compClassHnd; // for statics, we imagine the instance is the current class. - - bool isStaticField = ((fieldFlags & CORINFO_FLG_FIELD_STATIC) != 0); - if (mutator) - { - Verify(!(fieldFlags & CORINFO_FLG_FIELD_UNMANAGED), "mutating an RVA bases static"); - if ((fieldFlags & CORINFO_FLG_FIELD_FINAL)) - { - Verify((info.compFlags & CORINFO_FLG_CONSTRUCTOR) && enclosingClass == info.compClassHnd && - info.compIsStatic == isStaticField, - "bad use of initonly field (set or address taken)"); - } - } - - if (tiThis == nullptr) - { - Verify(isStaticField, "used static opcode with non-static field"); - } - else - { - typeInfo tThis = *tiThis; - - if (allowPlainStructAsThis && tThis.IsValueClass()) - { - tThis.MakeByRef(); - } - - // If it is null, we assume we can access it (since it will AV shortly) - // If it is anything but a refernce class, there is no hierarchy, so - // again, we don't need the precise instance class to compute 'protected' access - if (tiThis->IsType(TI_REF)) - { - instanceClass = tiThis->GetClassHandleForObjRef(); - } - - // Note that even if the field is static, we require that the this pointer - // satisfy the same constraints as a non-static field This happens to - // be simpler and seems reasonable - typeInfo tiDeclaredThis = verMakeTypeInfo(enclosingClass); - if (tiDeclaredThis.IsValueClass()) - { - tiDeclaredThis.MakeByRef(); - - // we allow read-only tThis, on any field access (even stores!), because if the - // class implementor wants to prohibit stores he should make the field private. - // we do this by setting the read-only bit on the type we compare tThis to. - tiDeclaredThis.SetIsReadonlyByRef(); - } - else if (verTrackObjCtorInitState && tThis.IsThisPtr()) - { - // Any field access is legal on "uninitialized" this pointers. - // The easiest way to implement this is to simply set the - // initialized bit for the duration of the type check on the - // field access only. It does not change the state of the "this" - // for the function as a whole. Note that the "tThis" is a copy - // of the original "this" type (*tiThis) passed in. - tThis.SetInitialisedObjRef(); - } - - Verify(tiCompatibleWith(tThis, tiDeclaredThis, true), "this type mismatch"); - } - - // Presently the JIT does not check that we don't store or take the address of init-only fields - // since we cannot guarantee their immutability and it is not a security issue. - - // check any constraints on the fields's class --- accessing the field might cause a class constructor to run. - VerifyOrReturn(info.compCompHnd->satisfiesClassConstraints(enclosingClass), - "field has unsatisfied class constraints"); - if (fieldFlags & CORINFO_FLG_FIELD_PROTECTED) - { - Verify(info.compCompHnd->canAccessFamily(info.compMethodHnd, instanceClass), - "Accessing protected method through wrong type."); - } -} - -void Compiler::verVerifyCond(const typeInfo& tiOp1, const typeInfo& tiOp2, unsigned opcode) -{ - if (tiOp1.IsNumberType()) - { -#ifdef TARGET_64BIT - Verify(tiCompatibleWith(tiOp1, tiOp2, true), "Cond type mismatch"); -#else // TARGET_64BIT - // [10/17/2013] Consider changing this: to put on my verification lawyer hat, - // this is non-conforming to the ECMA Spec: types don't have to be equivalent, - // but compatible, since we can coalesce native int with int32 (see section III.1.5). - Verify(typeInfo::AreEquivalent(tiOp1, tiOp2), "Cond type mismatch"); -#endif // !TARGET_64BIT - } - else if (tiOp1.IsObjRef()) - { - switch (opcode) - { - case CEE_BEQ_S: - case CEE_BEQ: - case CEE_BNE_UN_S: - case CEE_BNE_UN: - case CEE_CEQ: - case CEE_CGT_UN: - break; - default: - Verify(false, "Cond not allowed on object types"); - } - Verify(tiOp2.IsObjRef(), "Cond type mismatch"); - } - else if (tiOp1.IsByRef()) - { - Verify(tiOp2.IsByRef(), "Cond type mismatch"); - } - else - { - Verify(tiOp1.IsMethod() && tiOp2.IsMethod(), "Cond type mismatch"); - } -} - -void Compiler::verVerifyThisPtrInitialised() -{ - if (verTrackObjCtorInitState) - { - Verify(verCurrentState.thisInitialized == TIS_Init, "this ptr is not initialized"); - } -} - -bool Compiler::verIsCallToInitThisPtr(CORINFO_CLASS_HANDLE context, CORINFO_CLASS_HANDLE target) -{ - // Either target == context, in this case calling an alternate .ctor - // Or target is the immediate parent of context - - return ((target == context) || (target == info.compCompHnd->getParentType(context))); -} - -GenTree* Compiler::impImportLdvirtftn(GenTree* thisPtr, - CORINFO_RESOLVED_TOKEN* pResolvedToken, - CORINFO_CALL_INFO* pCallInfo) -{ - if ((pCallInfo->methodFlags & CORINFO_FLG_EnC) && !(pCallInfo->classFlags & CORINFO_FLG_INTERFACE)) - { - NO_WAY("Virtual call to a function added via EnC is not supported"); - } - - // NativeAOT generic virtual method - if ((pCallInfo->sig.sigInst.methInstCount != 0) && IsTargetAbi(CORINFO_NATIVEAOT_ABI)) - { - GenTree* runtimeMethodHandle = - impLookupToTree(pResolvedToken, &pCallInfo->codePointerLookup, GTF_ICON_METHOD_HDL, pCallInfo->hMethod); - return gtNewHelperCallNode(CORINFO_HELP_GVMLOOKUP_FOR_SLOT, TYP_I_IMPL, thisPtr, runtimeMethodHandle); - } - -#ifdef FEATURE_READYTORUN - if (opts.IsReadyToRun()) - { - if (!pCallInfo->exactContextNeedsRuntimeLookup) - { - GenTreeCall* call = gtNewHelperCallNode(CORINFO_HELP_READYTORUN_VIRTUAL_FUNC_PTR, TYP_I_IMPL, thisPtr); - - call->setEntryPoint(pCallInfo->codePointerLookup.constLookup); - - return call; - } - - // We need a runtime lookup. NativeAOT has a ReadyToRun helper for that too. - if (IsTargetAbi(CORINFO_NATIVEAOT_ABI)) - { - GenTree* ctxTree = getRuntimeContextTree(pCallInfo->codePointerLookup.lookupKind.runtimeLookupKind); - - return impReadyToRunHelperToTree(pResolvedToken, CORINFO_HELP_READYTORUN_GENERIC_HANDLE, TYP_I_IMPL, - &pCallInfo->codePointerLookup.lookupKind, ctxTree); - } - } -#endif - - // Get the exact descriptor for the static callsite - GenTree* exactTypeDesc = impParentClassTokenToHandle(pResolvedToken); - if (exactTypeDesc == nullptr) - { // compDonotInline() - return nullptr; - } - - GenTree* exactMethodDesc = impTokenToHandle(pResolvedToken); - if (exactMethodDesc == nullptr) - { // compDonotInline() - return nullptr; - } - - // Call helper function. This gets the target address of the final destination callsite. - - return gtNewHelperCallNode(CORINFO_HELP_VIRTUAL_FUNC_PTR, TYP_I_IMPL, thisPtr, exactTypeDesc, exactMethodDesc); -} - -//------------------------------------------------------------------------ -// impBoxPatternMatch: match and import common box idioms -// -// Arguments: -// pResolvedToken - resolved token from the box operation -// codeAddr - position in IL stream after the box instruction -// codeEndp - end of IL stream -// opts - dictate pattern matching behavior -// -// Return Value: -// Number of IL bytes matched and imported, -1 otherwise -// -// Notes: -// pResolvedToken is known to be a value type; ref type boxing -// is handled in the CEE_BOX clause. - -int Compiler::impBoxPatternMatch(CORINFO_RESOLVED_TOKEN* pResolvedToken, - const BYTE* codeAddr, - const BYTE* codeEndp, - BoxPatterns opts) -{ - if (codeAddr >= codeEndp) - { - return -1; - } - - switch (codeAddr[0]) - { - case CEE_UNBOX_ANY: - // box + unbox.any - if (codeAddr + 1 + sizeof(mdToken) <= codeEndp) - { - if (opts == BoxPatterns::MakeInlineObservation) - { - compInlineResult->Note(InlineObservation::CALLEE_FOLDABLE_BOX); - return 1 + sizeof(mdToken); - } - - CORINFO_RESOLVED_TOKEN unboxResolvedToken; - - impResolveToken(codeAddr + 1, &unboxResolvedToken, CORINFO_TOKENKIND_Class); - - // See if the resolved tokens describe types that are equal. - const TypeCompareState compare = - info.compCompHnd->compareTypesForEquality(unboxResolvedToken.hClass, pResolvedToken->hClass); - - bool optimize = false; - - // If so, box/unbox.any is a nop. - if (compare == TypeCompareState::Must) - { - optimize = true; - } - else if (compare == TypeCompareState::MustNot) - { - // An attempt to catch cases where we mix enums and primitives, e.g.: - // (IntEnum)(object)myInt - // (byte)(object)myByteEnum - // - CorInfoType typ = info.compCompHnd->getTypeForPrimitiveValueClass(unboxResolvedToken.hClass); - if ((typ >= CORINFO_TYPE_BYTE) && (typ <= CORINFO_TYPE_ULONG) && - (info.compCompHnd->getTypeForPrimitiveValueClass(pResolvedToken->hClass) == typ)) - { - optimize = true; - } - } - - if (optimize) - { - JITDUMP("\n Importing BOX; UNBOX.ANY as NOP\n"); - // Skip the next unbox.any instruction - return 1 + sizeof(mdToken); - } - } - break; - - case CEE_BRTRUE: - case CEE_BRTRUE_S: - case CEE_BRFALSE: - case CEE_BRFALSE_S: - // box + br_true/false - if ((codeAddr + ((codeAddr[0] >= CEE_BRFALSE) ? 5 : 2)) <= codeEndp) - { - if (opts == BoxPatterns::MakeInlineObservation) - { - compInlineResult->Note(InlineObservation::CALLEE_FOLDABLE_BOX); - return 0; - } - - GenTree* const treeToBox = impStackTop().val; - bool canOptimize = true; - GenTree* treeToNullcheck = nullptr; - - // Can the thing being boxed cause a side effect? - if ((treeToBox->gtFlags & GTF_SIDE_EFFECT) != 0) - { - // Is this a side effect we can replicate cheaply? - if (((treeToBox->gtFlags & GTF_SIDE_EFFECT) == GTF_EXCEPT) && - treeToBox->OperIs(GT_OBJ, GT_BLK, GT_IND)) - { - // If the only side effect comes from the dereference itself, yes. - GenTree* const addr = treeToBox->AsOp()->gtGetOp1(); - - if ((addr->gtFlags & GTF_SIDE_EFFECT) != 0) - { - canOptimize = false; - } - else if (fgAddrCouldBeNull(addr)) - { - treeToNullcheck = addr; - } - } - else - { - canOptimize = false; - } - } - - if (canOptimize) - { - if ((opts == BoxPatterns::IsByRefLike) || - info.compCompHnd->getBoxHelper(pResolvedToken->hClass) == CORINFO_HELP_BOX) - { - JITDUMP("\n Importing BOX; BR_TRUE/FALSE as %sconstant\n", - treeToNullcheck == nullptr ? "" : "nullcheck+"); - impPopStack(); - - GenTree* result = gtNewIconNode(1); - - if (treeToNullcheck != nullptr) - { - GenTree* nullcheck = gtNewNullCheck(treeToNullcheck, compCurBB); - result = gtNewOperNode(GT_COMMA, TYP_INT, nullcheck, result); - } - - impPushOnStack(result, typeInfo(TI_INT)); - return 0; - } - } - } - break; - - case CEE_ISINST: - if (codeAddr + 1 + sizeof(mdToken) + 1 <= codeEndp) - { - const BYTE* nextCodeAddr = codeAddr + 1 + sizeof(mdToken); - - switch (nextCodeAddr[0]) - { - // box + isinst + br_true/false - case CEE_BRTRUE: - case CEE_BRTRUE_S: - case CEE_BRFALSE: - case CEE_BRFALSE_S: - if ((nextCodeAddr + ((nextCodeAddr[0] >= CEE_BRFALSE) ? 5 : 2)) <= codeEndp) - { - if (opts == BoxPatterns::MakeInlineObservation) - { - compInlineResult->Note(InlineObservation::CALLEE_FOLDABLE_BOX); - return 1 + sizeof(mdToken); - } - - if ((impStackTop().val->gtFlags & GTF_SIDE_EFFECT) == 0) - { - CorInfoHelpFunc foldAsHelper; - if (opts == BoxPatterns::IsByRefLike) - { - // Treat ByRefLike types as if they were regular boxing operations - // so they can be elided. - foldAsHelper = CORINFO_HELP_BOX; - } - else - { - foldAsHelper = info.compCompHnd->getBoxHelper(pResolvedToken->hClass); - } - - if (foldAsHelper == CORINFO_HELP_BOX) - { - CORINFO_RESOLVED_TOKEN isInstResolvedToken; - - impResolveToken(codeAddr + 1, &isInstResolvedToken, CORINFO_TOKENKIND_Casting); - - TypeCompareState castResult = - info.compCompHnd->compareTypesForCast(pResolvedToken->hClass, - isInstResolvedToken.hClass); - if (castResult != TypeCompareState::May) - { - JITDUMP("\n Importing BOX; ISINST; BR_TRUE/FALSE as constant\n"); - impPopStack(); - - impPushOnStack(gtNewIconNode((castResult == TypeCompareState::Must) ? 1 : 0), - typeInfo(TI_INT)); - - // Skip the next isinst instruction - return 1 + sizeof(mdToken); - } - } - else if (foldAsHelper == CORINFO_HELP_BOX_NULLABLE) - { - // For nullable we're going to fold it to "ldfld hasValue + brtrue/brfalse" or - // "ldc.i4.0 + brtrue/brfalse" in case if the underlying type is not castable to - // the target type. - CORINFO_RESOLVED_TOKEN isInstResolvedToken; - impResolveToken(codeAddr + 1, &isInstResolvedToken, CORINFO_TOKENKIND_Casting); - - CORINFO_CLASS_HANDLE nullableCls = pResolvedToken->hClass; - CORINFO_CLASS_HANDLE underlyingCls = info.compCompHnd->getTypeForBox(nullableCls); - - TypeCompareState castResult = - info.compCompHnd->compareTypesForCast(underlyingCls, - isInstResolvedToken.hClass); - - if (castResult == TypeCompareState::Must) - { - const CORINFO_FIELD_HANDLE hasValueFldHnd = - info.compCompHnd->getFieldInClass(nullableCls, 0); - - assert(info.compCompHnd->getFieldOffset(hasValueFldHnd) == 0); - assert(!strcmp(info.compCompHnd->getFieldName(hasValueFldHnd, nullptr), - "hasValue")); - - GenTree* objToBox = impPopStack().val; - - // Spill struct to get its address (to access hasValue field) - objToBox = - impGetStructAddr(objToBox, nullableCls, (unsigned)CHECK_SPILL_ALL, true); - - impPushOnStack(gtNewFieldRef(TYP_BOOL, hasValueFldHnd, objToBox, 0), - typeInfo(TI_INT)); - - JITDUMP("\n Importing BOX; ISINST; BR_TRUE/FALSE as nullableVT.hasValue\n"); - return 1 + sizeof(mdToken); - } - else if (castResult == TypeCompareState::MustNot) - { - impPopStack(); - impPushOnStack(gtNewIconNode(0), typeInfo(TI_INT)); - JITDUMP("\n Importing BOX; ISINST; BR_TRUE/FALSE as constant (false)\n"); - return 1 + sizeof(mdToken); - } - } - } - } - break; - - // box + isinst + unbox.any - case CEE_UNBOX_ANY: - if ((nextCodeAddr + 1 + sizeof(mdToken)) <= codeEndp) - { - if (opts == BoxPatterns::MakeInlineObservation) - { - compInlineResult->Note(InlineObservation::CALLEE_FOLDABLE_BOX); - return 2 + sizeof(mdToken) * 2; - } - - // See if the resolved tokens in box, isinst and unbox.any describe types that are equal. - CORINFO_RESOLVED_TOKEN isinstResolvedToken = {}; - impResolveToken(codeAddr + 1, &isinstResolvedToken, CORINFO_TOKENKIND_Class); - - if (info.compCompHnd->compareTypesForEquality(isinstResolvedToken.hClass, - pResolvedToken->hClass) == - TypeCompareState::Must) - { - CORINFO_RESOLVED_TOKEN unboxResolvedToken = {}; - impResolveToken(nextCodeAddr + 1, &unboxResolvedToken, CORINFO_TOKENKIND_Class); - - // If so, box + isinst + unbox.any is a nop. - if (info.compCompHnd->compareTypesForEquality(unboxResolvedToken.hClass, - pResolvedToken->hClass) == - TypeCompareState::Must) - { - JITDUMP("\n Importing BOX; ISINST, UNBOX.ANY as NOP\n"); - return 2 + sizeof(mdToken) * 2; - } - } - } - break; - } - } - break; - - default: - break; - } - - return -1; -} - -//------------------------------------------------------------------------ -// impImportAndPushBox: build and import a value-type box -// -// Arguments: -// pResolvedToken - resolved token from the box operation -// -// Return Value: -// None. -// -// Side Effects: -// The value to be boxed is popped from the stack, and a tree for -// the boxed value is pushed. This method may create upstream -// statements, spill side effecting trees, and create new temps. -// -// If importing an inlinee, we may also discover the inline must -// fail. If so there is no new value pushed on the stack. Callers -// should use CompDoNotInline after calling this method to see if -// ongoing importation should be aborted. -// -// Notes: -// Boxing of ref classes results in the same value as the value on -// the top of the stack, so is handled inline in impImportBlockCode -// for the CEE_BOX case. Only value or primitive type boxes make it -// here. -// -// Boxing for nullable types is done via a helper call; boxing -// of other value types is expanded inline or handled via helper -// call, depending on the jit's codegen mode. -// -// When the jit is operating in size and time constrained modes, -// using a helper call here can save jit time and code size. But it -// also may inhibit cleanup optimizations that could have also had a -// even greater benefit effect on code size and jit time. An optimal -// strategy may need to peek ahead and see if it is easy to tell how -// the box is being used. For now, we defer. - -void Compiler::impImportAndPushBox(CORINFO_RESOLVED_TOKEN* pResolvedToken) -{ - // Spill any special side effects - impSpillSpecialSideEff(); - - // Get get the expression to box from the stack. - GenTree* op1 = nullptr; - GenTree* op2 = nullptr; - StackEntry se = impPopStack(); - CORINFO_CLASS_HANDLE operCls = se.seTypeInfo.GetClassHandle(); - GenTree* exprToBox = se.val; - - // Look at what helper we should use. - CorInfoHelpFunc boxHelper = info.compCompHnd->getBoxHelper(pResolvedToken->hClass); - - // Determine what expansion to prefer. - // - // In size/time/debuggable constrained modes, the helper call - // expansion for box is generally smaller and is preferred, unless - // the value to box is a struct that comes from a call. In that - // case the call can construct its return value directly into the - // box payload, saving possibly some up-front zeroing. - // - // Currently primitive type boxes always get inline expanded. We may - // want to do the same for small structs if they don't come from - // calls and don't have GC pointers, since explicitly copying such - // structs is cheap. - JITDUMP("\nCompiler::impImportAndPushBox -- handling BOX(value class) via"); - bool canExpandInline = (boxHelper == CORINFO_HELP_BOX); - bool optForSize = !exprToBox->IsCall() && (operCls != nullptr) && opts.OptimizationDisabled(); - bool expandInline = canExpandInline && !optForSize; - - if (expandInline) - { - JITDUMP(" inline allocate/copy sequence\n"); - - // we are doing 'normal' boxing. This means that we can inline the box operation - // Box(expr) gets morphed into - // temp = new(clsHnd) - // cpobj(temp+4, expr, clsHnd) - // push temp - // The code paths differ slightly below for structs and primitives because - // "cpobj" differs in these cases. In one case you get - // impAssignStructPtr(temp+4, expr, clsHnd) - // and the other you get - // *(temp+4) = expr - - if (opts.OptimizationDisabled()) - { - // For minopts/debug code, try and minimize the total number - // of box temps by reusing an existing temp when possible. - if (impBoxTempInUse || impBoxTemp == BAD_VAR_NUM) - { - impBoxTemp = lvaGrabTemp(true DEBUGARG("Reusable Box Helper")); - } - } - else - { - // When optimizing, use a new temp for each box operation - // since we then know the exact class of the box temp. - impBoxTemp = lvaGrabTemp(true DEBUGARG("Single-def Box Helper")); - lvaTable[impBoxTemp].lvType = TYP_REF; - lvaTable[impBoxTemp].lvSingleDef = 1; - JITDUMP("Marking V%02u as a single def local\n", impBoxTemp); - const bool isExact = true; - lvaSetClass(impBoxTemp, pResolvedToken->hClass, isExact); - } - - // needs to stay in use until this box expression is appended - // some other node. We approximate this by keeping it alive until - // the opcode stack becomes empty - impBoxTempInUse = true; - - // Remember the current last statement in case we need to move - // a range of statements to ensure the box temp is initialized - // before it's used. - // - Statement* const cursor = impLastStmt; - - const bool useParent = false; - op1 = gtNewAllocObjNode(pResolvedToken, useParent); - if (op1 == nullptr) - { - // If we fail to create the newobj node, we must be inlining - // and have run across a type we can't describe. - // - assert(compDonotInline()); - return; - } - - // Remember that this basic block contains 'new' of an object, - // and so does this method - // - compCurBB->bbFlags |= BBF_HAS_NEWOBJ; - optMethodFlags |= OMF_HAS_NEWOBJ; - - // Assign the boxed object to the box temp. - // - GenTree* asg = gtNewTempAssign(impBoxTemp, op1); - Statement* asgStmt = impAppendTree(asg, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); - - // If the exprToBox is a call that returns its value via a ret buf arg, - // move the assignment statement(s) before the call (which must be a top level tree). - // - // We do this because impAssignStructPtr (invoked below) will - // back-substitute into a call when it sees a GT_RET_EXPR and the call - // has a hidden buffer pointer, So we need to reorder things to avoid - // creating out-of-sequence IR. - // - if (varTypeIsStruct(exprToBox) && exprToBox->OperIs(GT_RET_EXPR)) - { - GenTreeCall* const call = exprToBox->AsRetExpr()->gtInlineCandidate->AsCall(); - - if (call->ShouldHaveRetBufArg()) - { - JITDUMP("Must insert newobj stmts for box before call [%06u]\n", dspTreeID(call)); - - // Walk back through the statements in this block, looking for the one - // that has this call as the root node. - // - // Because gtNewTempAssign (above) may have added statements that - // feed into the actual assignment we need to move this set of added - // statements as a group. - // - // Note boxed allocations are side-effect free (no com or finalizer) so - // our only worries here are (correctness) not overlapping the box temp - // lifetime and (perf) stretching the temp lifetime across the inlinee - // body. - // - // Since this is an inline candidate, we must be optimizing, and so we have - // a unique box temp per call. So no worries about overlap. - // - assert(!opts.OptimizationDisabled()); - - // Lifetime stretching could addressed with some extra cleverness--sinking - // the allocation back down to just before the copy, once we figure out - // where the copy is. We defer for now. - // - Statement* insertBeforeStmt = cursor; - noway_assert(insertBeforeStmt != nullptr); - - while (true) - { - if (insertBeforeStmt->GetRootNode() == call) - { - break; - } - - // If we've searched all the statements in the block and failed to - // find the call, then something's wrong. - // - noway_assert(insertBeforeStmt != impStmtList); - - insertBeforeStmt = insertBeforeStmt->GetPrevStmt(); - } - - // Found the call. Move the statements comprising the assignment. - // - JITDUMP("Moving " FMT_STMT "..." FMT_STMT " before " FMT_STMT "\n", cursor->GetNextStmt()->GetID(), - asgStmt->GetID(), insertBeforeStmt->GetID()); - assert(asgStmt == impLastStmt); - do - { - Statement* movingStmt = impExtractLastStmt(); - impInsertStmtBefore(movingStmt, insertBeforeStmt); - insertBeforeStmt = movingStmt; - } while (impLastStmt != cursor); - } - } - - // Create a pointer to the box payload in op1. - // - op1 = gtNewLclvNode(impBoxTemp, TYP_REF); - op2 = gtNewIconNode(TARGET_POINTER_SIZE, TYP_I_IMPL); - op1 = gtNewOperNode(GT_ADD, TYP_BYREF, op1, op2); - - // Copy from the exprToBox to the box payload. - // - if (varTypeIsStruct(exprToBox)) - { - assert(info.compCompHnd->getClassSize(pResolvedToken->hClass) == info.compCompHnd->getClassSize(operCls)); - op1 = impAssignStructPtr(op1, exprToBox, operCls, (unsigned)CHECK_SPILL_ALL); - } - else - { - var_types lclTyp = exprToBox->TypeGet(); - if (lclTyp == TYP_BYREF) - { - lclTyp = TYP_I_IMPL; - } - CorInfoType jitType = info.compCompHnd->asCorInfoType(pResolvedToken->hClass); - if (impIsPrimitive(jitType)) - { - lclTyp = JITtype2varType(jitType); - } - - var_types srcTyp = exprToBox->TypeGet(); - var_types dstTyp = lclTyp; - - // We allow float <-> double mismatches and implicit truncation for small types. - assert((genActualType(srcTyp) == genActualType(dstTyp)) || - (varTypeIsFloating(srcTyp) == varTypeIsFloating(dstTyp))); - - // Note regarding small types. - // We are going to store to the box here via an indirection, so the cast added below is - // redundant, since the store has an implicit truncation semantic. The reason we still - // add this cast is so that the code which deals with GT_BOX optimizations does not have - // to account for this implicit truncation (e. g. understand that BOX(0xFF + 1) is - // actually BOX(0) or deal with signedness mismatch and other GT_CAST complexities). - if (srcTyp != dstTyp) - { - exprToBox = gtNewCastNode(genActualType(dstTyp), exprToBox, false, dstTyp); - } - - op1 = gtNewAssignNode(gtNewOperNode(GT_IND, dstTyp, op1), exprToBox); - } - - // Spill eval stack to flush out any pending side effects. - impSpillSideEffects(true, (unsigned)CHECK_SPILL_ALL DEBUGARG("impImportAndPushBox")); - - // Set up this copy as a second assignment. - Statement* copyStmt = impAppendTree(op1, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); - - op1 = gtNewLclvNode(impBoxTemp, TYP_REF); - - // Record that this is a "box" node and keep track of the matching parts. - op1 = new (this, GT_BOX) GenTreeBox(TYP_REF, op1, asgStmt, copyStmt); - - // If it is a value class, mark the "box" node. We can use this information - // to optimise several cases: - // "box(x) == null" --> false - // "(box(x)).CallAnInterfaceMethod(...)" --> "(&x).CallAValueTypeMethod" - // "(box(x)).CallAnObjectMethod(...)" --> "(&x).CallAValueTypeMethod" - - op1->gtFlags |= GTF_BOX_VALUE; - assert(op1->IsBoxedValue()); - assert(asg->gtOper == GT_ASG); - } - else - { - // Don't optimize, just call the helper and be done with it. - JITDUMP(" helper call because: %s\n", canExpandInline ? "optimizing for size" : "nullable"); - assert(operCls != nullptr); - - // Ensure that the value class is restored - op2 = impTokenToHandle(pResolvedToken, nullptr, true /* mustRestoreHandle */); - if (op2 == nullptr) - { - // We must be backing out of an inline. - assert(compDonotInline()); - return; - } - - op1 = gtNewHelperCallNode(boxHelper, TYP_REF, op2, - impGetStructAddr(exprToBox, operCls, (unsigned)CHECK_SPILL_ALL, true)); - } - - /* Push the result back on the stack, */ - /* even if clsHnd is a value class we want the TI_REF */ - typeInfo tiRetVal = typeInfo(TI_REF, info.compCompHnd->getTypeForBox(pResolvedToken->hClass)); - impPushOnStack(op1, tiRetVal); -} - -//------------------------------------------------------------------------ -// impImportNewObjArray: Build and import `new` of multi-dimensional array -// -// Arguments: -// pResolvedToken - The CORINFO_RESOLVED_TOKEN that has been initialized -// by a call to CEEInfo::resolveToken(). -// pCallInfo - The CORINFO_CALL_INFO that has been initialized -// by a call to CEEInfo::getCallInfo(). -// -// Assumptions: -// The multi-dimensional array constructor arguments (array dimensions) are -// pushed on the IL stack on entry to this method. -// -// Notes: -// Multi-dimensional array constructors are imported as calls to a JIT -// helper, not as regular calls. -// -void Compiler::impImportNewObjArray(CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_CALL_INFO* pCallInfo) -{ - GenTree* classHandle = impParentClassTokenToHandle(pResolvedToken); - if (classHandle == nullptr) - { // compDonotInline() - return; - } - - assert(pCallInfo->sig.numArgs); - - GenTree* node; - - // Reuse the temp used to pass the array dimensions to avoid bloating - // the stack frame in case there are multiple calls to multi-dim array - // constructors within a single method. - if (lvaNewObjArrayArgs == BAD_VAR_NUM) - { - lvaNewObjArrayArgs = lvaGrabTemp(false DEBUGARG("NewObjArrayArgs")); - lvaTable[lvaNewObjArrayArgs].lvType = TYP_BLK; - lvaTable[lvaNewObjArrayArgs].lvExactSize = 0; - } - - // Increase size of lvaNewObjArrayArgs to be the largest size needed to hold 'numArgs' integers - // for our call to CORINFO_HELP_NEW_MDARR. - lvaTable[lvaNewObjArrayArgs].lvExactSize = - max(lvaTable[lvaNewObjArrayArgs].lvExactSize, pCallInfo->sig.numArgs * sizeof(INT32)); - - // The side-effects may include allocation of more multi-dimensional arrays. Spill all side-effects - // to ensure that the shared lvaNewObjArrayArgs local variable is only ever used to pass arguments - // to one allocation at a time. - impSpillSideEffects(true, (unsigned)CHECK_SPILL_ALL DEBUGARG("impImportNewObjArray")); - - // - // The arguments of the CORINFO_HELP_NEW_MDARR helper are: - // - Array class handle - // - Number of dimension arguments - // - Pointer to block of int32 dimensions: address of lvaNewObjArrayArgs temp. - // - - node = gtNewLclvNode(lvaNewObjArrayArgs, TYP_BLK); - node = gtNewOperNode(GT_ADDR, TYP_I_IMPL, node); - - // Pop dimension arguments from the stack one at a time and store it - // into lvaNewObjArrayArgs temp. - for (int i = pCallInfo->sig.numArgs - 1; i >= 0; i--) - { - GenTree* arg = impImplicitIorI4Cast(impPopStack().val, TYP_INT); - - GenTree* dest = gtNewLclvNode(lvaNewObjArrayArgs, TYP_BLK); - dest = gtNewOperNode(GT_ADDR, TYP_I_IMPL, dest); - dest = gtNewOperNode(GT_ADD, TYP_I_IMPL, dest, - new (this, GT_CNS_INT) GenTreeIntCon(TYP_I_IMPL, sizeof(INT32) * i)); - dest = gtNewOperNode(GT_IND, TYP_INT, dest); - - node = gtNewOperNode(GT_COMMA, node->TypeGet(), gtNewAssignNode(dest, arg), node); - } - - node = - gtNewHelperCallNode(CORINFO_HELP_NEW_MDARR, TYP_REF, classHandle, gtNewIconNode(pCallInfo->sig.numArgs), node); - - node->AsCall()->compileTimeHelperArgumentHandle = (CORINFO_GENERIC_HANDLE)pResolvedToken->hClass; - - // Remember that this function contains 'new' of a MD array. - optMethodFlags |= OMF_HAS_MDNEWARRAY; - - impPushOnStack(node, typeInfo(TI_REF, pResolvedToken->hClass)); -} - -GenTree* Compiler::impTransformThis(GenTree* thisPtr, - CORINFO_RESOLVED_TOKEN* pConstrainedResolvedToken, - CORINFO_THIS_TRANSFORM transform) -{ - switch (transform) - { - case CORINFO_DEREF_THIS: - { - GenTree* obj = thisPtr; - - // This does a LDIND on the obj, which should be a byref. pointing to a ref - impBashVarAddrsToI(obj); - assert(genActualType(obj->gtType) == TYP_I_IMPL || obj->gtType == TYP_BYREF); - CorInfoType constraintTyp = info.compCompHnd->asCorInfoType(pConstrainedResolvedToken->hClass); - - obj = gtNewOperNode(GT_IND, JITtype2varType(constraintTyp), obj); - // ldind could point anywhere, example a boxed class static int - obj->gtFlags |= (GTF_EXCEPT | GTF_GLOB_REF); - - return obj; - } - - case CORINFO_BOX_THIS: - { - // Constraint calls where there might be no - // unboxed entry point require us to implement the call via helper. - // These only occur when a possible target of the call - // may have inherited an implementation of an interface - // method from System.Object or System.ValueType. The EE does not provide us with - // "unboxed" versions of these methods. - - GenTree* obj = thisPtr; - - assert(obj->TypeGet() == TYP_BYREF || obj->TypeGet() == TYP_I_IMPL); - obj = gtNewObjNode(pConstrainedResolvedToken->hClass, obj); - obj->gtFlags |= GTF_EXCEPT; - - CorInfoType jitTyp = info.compCompHnd->asCorInfoType(pConstrainedResolvedToken->hClass); - if (impIsPrimitive(jitTyp)) - { - if (obj->OperIsBlk()) - { - obj->ChangeOperUnchecked(GT_IND); - obj->AsOp()->gtOp2 = nullptr; // must be zero for tree walkers - } - - obj->gtType = JITtype2varType(jitTyp); - assert(varTypeIsArithmetic(obj->gtType)); - } - - // This pushes on the dereferenced byref - // This is then used immediately to box. - impPushOnStack(obj, verMakeTypeInfo(pConstrainedResolvedToken->hClass).NormaliseForStack()); - - // This pops off the byref-to-a-value-type remaining on the stack and - // replaces it with a boxed object. - // This is then used as the object to the virtual call immediately below. - impImportAndPushBox(pConstrainedResolvedToken); - if (compDonotInline()) - { - return nullptr; - } - - obj = impPopStack().val; - return obj; - } - case CORINFO_NO_THIS_TRANSFORM: - default: - return thisPtr; - } -} - -//------------------------------------------------------------------------ -// impCanPInvokeInline: check whether PInvoke inlining should enabled in current method. -// -// Return Value: -// true if PInvoke inlining should be enabled in current method, false otherwise -// -// Notes: -// Checks a number of ambient conditions where we could pinvoke but choose not to - -bool Compiler::impCanPInvokeInline() -{ - return getInlinePInvokeEnabled() && (!opts.compDbgCode) && (compCodeOpt() != SMALL_CODE) && - (!opts.compNoPInvokeInlineCB) // profiler is preventing inline pinvoke - ; -} - -//------------------------------------------------------------------------ -// impCanPInvokeInlineCallSite: basic legality checks using information -// from a call to see if the call qualifies as an inline pinvoke. -// -// Arguments: -// block - block contaning the call, or for inlinees, block -// containing the call being inlined -// -// Return Value: -// true if this call can legally qualify as an inline pinvoke, false otherwise -// -// Notes: -// For runtimes that support exception handling interop there are -// restrictions on using inline pinvoke in handler regions. -// -// * We have to disable pinvoke inlining inside of filters because -// in case the main execution (i.e. in the try block) is inside -// unmanaged code, we cannot reuse the inlined stub (we still need -// the original state until we are in the catch handler) -// -// * We disable pinvoke inlining inside handlers since the GSCookie -// is in the inlined Frame (see -// CORINFO_EE_INFO::InlinedCallFrameInfo::offsetOfGSCookie), but -// this would not protect framelets/return-address of handlers. -// -// These restrictions are currently also in place for CoreCLR but -// can be relaxed when coreclr/#8459 is addressed. - -bool Compiler::impCanPInvokeInlineCallSite(BasicBlock* block) -{ - if (block->hasHndIndex()) - { - return false; - } - - // The remaining limitations do not apply to NativeAOT - if (IsTargetAbi(CORINFO_NATIVEAOT_ABI)) - { - return true; - } - -#ifdef TARGET_64BIT - // On 64-bit platforms, we disable pinvoke inlining inside of try regions. - // Note that this could be needed on other architectures too, but we - // haven't done enough investigation to know for sure at this point. - // - // Here is the comment from JIT64 explaining why: - // [VSWhidbey: 611015] - because the jitted code links in the - // Frame (instead of the stub) we rely on the Frame not being - // 'active' until inside the stub. This normally happens by the - // stub setting the return address pointer in the Frame object - // inside the stub. On a normal return, the return address - // pointer is zeroed out so the Frame can be safely re-used, but - // if an exception occurs, nobody zeros out the return address - // pointer. Thus if we re-used the Frame object, it would go - // 'active' as soon as we link it into the Frame chain. - // - // Technically we only need to disable PInvoke inlining if we're - // in a handler or if we're in a try body with a catch or - // filter/except where other non-handler code in this method - // might run and try to re-use the dirty Frame object. - // - // A desktop test case where this seems to matter is - // jit\jit64\ebvts\mcpp\sources2\ijw\__clrcall\vector_ctor_dtor.02\deldtor_clr.exe - if (block->hasTryIndex()) - { - // This does not apply to the raw pinvoke call that is inside the pinvoke - // ILStub. In this case, we have to inline the raw pinvoke call into the stub, - // otherwise we would end up with a stub that recursively calls itself, and end - // up with a stack overflow. - if (opts.jitFlags->IsSet(JitFlags::JIT_FLAG_IL_STUB) && opts.ShouldUsePInvokeHelpers()) - { - return true; - } - - return false; - } -#endif // TARGET_64BIT - - return true; -} - -//------------------------------------------------------------------------ -// impCheckForPInvokeCall examine call to see if it is a pinvoke and if so -// if it can be expressed as an inline pinvoke. -// -// Arguments: -// call - tree for the call -// methHnd - handle for the method being called (may be null) -// sig - signature of the method being called -// mflags - method flags for the method being called -// block - block contaning the call, or for inlinees, block -// containing the call being inlined -// -// Notes: -// Sets GTF_CALL_M_PINVOKE on the call for pinvokes. -// -// Also sets GTF_CALL_UNMANAGED on call for inline pinvokes if the -// call passes a combination of legality and profitability checks. -// -// If GTF_CALL_UNMANAGED is set, increments info.compUnmanagedCallCountWithGCTransition - -void Compiler::impCheckForPInvokeCall( - GenTreeCall* call, CORINFO_METHOD_HANDLE methHnd, CORINFO_SIG_INFO* sig, unsigned mflags, BasicBlock* block) -{ - CorInfoCallConvExtension unmanagedCallConv; - - // If VM flagged it as Pinvoke, flag the call node accordingly - if ((mflags & CORINFO_FLG_PINVOKE) != 0) - { - call->gtCallMoreFlags |= GTF_CALL_M_PINVOKE; - } - - bool suppressGCTransition = false; - if (methHnd) - { - if ((mflags & CORINFO_FLG_PINVOKE) == 0) - { - return; - } - - unmanagedCallConv = info.compCompHnd->getUnmanagedCallConv(methHnd, nullptr, &suppressGCTransition); - } - else - { - if (sig->getCallConv() == CORINFO_CALLCONV_DEFAULT || sig->getCallConv() == CORINFO_CALLCONV_VARARG) - { - return; - } - - unmanagedCallConv = info.compCompHnd->getUnmanagedCallConv(nullptr, sig, &suppressGCTransition); - - assert(!call->gtCallCookie); - } - - if (suppressGCTransition) - { - call->gtCallMoreFlags |= GTF_CALL_M_SUPPRESS_GC_TRANSITION; - } - - if ((unmanagedCallConv == CorInfoCallConvExtension::Thiscall) && (sig->numArgs == 0)) - { - BADCODE("thiscall with 0 arguments"); - } - - // If we can't get the unmanaged calling convention or the calling convention is unsupported in the JIT, - // return here without inlining the native call. - if (unmanagedCallConv == CorInfoCallConvExtension::Managed || - unmanagedCallConv == CorInfoCallConvExtension::Fastcall || - unmanagedCallConv == CorInfoCallConvExtension::FastcallMemberFunction) - { - return; - } - optNativeCallCount++; - - if (methHnd == nullptr && (opts.jitFlags->IsSet(JitFlags::JIT_FLAG_IL_STUB) || IsTargetAbi(CORINFO_NATIVEAOT_ABI))) - { - // PInvoke in NativeAOT ABI must be always inlined. Non-inlineable CALLI cases have been - // converted to regular method calls earlier using convertPInvokeCalliToCall. - - // PInvoke CALLI in IL stubs must be inlined - } - else - { - // Check legality - if (!impCanPInvokeInlineCallSite(block)) - { - return; - } - - // Legal PInvoke CALL in PInvoke IL stubs must be inlined to avoid infinite recursive - // inlining in NativeAOT. Skip the ambient conditions checks and profitability checks. - if (!IsTargetAbi(CORINFO_NATIVEAOT_ABI) || (info.compFlags & CORINFO_FLG_PINVOKE) == 0) - { - if (opts.jitFlags->IsSet(JitFlags::JIT_FLAG_IL_STUB) && opts.ShouldUsePInvokeHelpers()) - { - // Raw PInvoke call in PInvoke IL stub generated must be inlined to avoid infinite - // recursive calls to the stub. - } - else - { - if (!impCanPInvokeInline()) - { - return; - } - - // Size-speed tradeoff: don't use inline pinvoke at rarely - // executed call sites. The non-inline version is more - // compact. - if (block->isRunRarely()) - { - return; - } - } - } - - // The expensive check should be last - if (info.compCompHnd->pInvokeMarshalingRequired(methHnd, sig)) - { - return; - } - } - - JITLOG((LL_INFO1000000, "\nInline a CALLI PINVOKE call from method %s\n", info.compFullName)); - - call->gtFlags |= GTF_CALL_UNMANAGED; - call->unmgdCallConv = unmanagedCallConv; - if (!call->IsSuppressGCTransition()) - { - info.compUnmanagedCallCountWithGCTransition++; - } - - // AMD64 convention is same for native and managed - if (unmanagedCallConv == CorInfoCallConvExtension::C || - unmanagedCallConv == CorInfoCallConvExtension::CMemberFunction) - { - call->gtFlags |= GTF_CALL_POP_ARGS; - } -} - -GenTreeCall* Compiler::impImportIndirectCall(CORINFO_SIG_INFO* sig, const DebugInfo& di) -{ - var_types callRetTyp = JITtype2varType(sig->retType); - - /* The function pointer is on top of the stack - It may be a - * complex expression. As it is evaluated after the args, - * it may cause registered args to be spilled. Simply spill it. - */ - - // Ignore this trivial case. - if (impStackTop().val->gtOper != GT_LCL_VAR) - { - impSpillStackEntry(verCurrentState.esStackDepth - 1, - BAD_VAR_NUM DEBUGARG(false) DEBUGARG("impImportIndirectCall")); - } - - /* Get the function pointer */ - - GenTree* fptr = impPopStack().val; - - // The function pointer is typically a sized to match the target pointer size - // However, stubgen IL optimization can change LDC.I8 to LDC.I4 - // See ILCodeStream::LowerOpcode - assert(genActualType(fptr->gtType) == TYP_I_IMPL || genActualType(fptr->gtType) == TYP_INT); - -#ifdef DEBUG - // This temporary must never be converted to a double in stress mode, - // because that can introduce a call to the cast helper after the - // arguments have already been evaluated. - - if (fptr->OperGet() == GT_LCL_VAR) - { - lvaTable[fptr->AsLclVarCommon()->GetLclNum()].lvKeepType = 1; - } -#endif - - /* Create the call node */ - - GenTreeCall* call = gtNewIndCallNode(fptr, callRetTyp, di); - - call->gtFlags |= GTF_EXCEPT | (fptr->gtFlags & GTF_GLOB_EFFECT); -#ifdef UNIX_X86_ABI - call->gtFlags &= ~GTF_CALL_POP_ARGS; -#endif - - return call; -} - -/*****************************************************************************/ - -void Compiler::impPopArgsForUnmanagedCall(GenTreeCall* call, CORINFO_SIG_INFO* sig) -{ - assert(call->gtFlags & GTF_CALL_UNMANAGED); - - /* Since we push the arguments in reverse order (i.e. right -> left) - * spill any side effects from the stack - * - * OBS: If there is only one side effect we do not need to spill it - * thus we have to spill all side-effects except last one - */ - - unsigned lastLevelWithSideEffects = UINT_MAX; - - unsigned argsToReverse = sig->numArgs; - - // For "thiscall", the first argument goes in a register. Since its - // order does not need to be changed, we do not need to spill it - - if (call->unmgdCallConv == CorInfoCallConvExtension::Thiscall) - { - assert(argsToReverse); - argsToReverse--; - } - -#ifndef TARGET_X86 - // Don't reverse args on ARM or x64 - first four args always placed in regs in order - argsToReverse = 0; -#endif - - for (unsigned level = verCurrentState.esStackDepth - argsToReverse; level < verCurrentState.esStackDepth; level++) - { - if (verCurrentState.esStack[level].val->gtFlags & GTF_ORDER_SIDEEFF) - { - assert(lastLevelWithSideEffects == UINT_MAX); - - impSpillStackEntry(level, - BAD_VAR_NUM DEBUGARG(false) DEBUGARG("impPopArgsForUnmanagedCall - other side effect")); - } - else if (verCurrentState.esStack[level].val->gtFlags & GTF_SIDE_EFFECT) - { - if (lastLevelWithSideEffects != UINT_MAX) - { - /* We had a previous side effect - must spill it */ - impSpillStackEntry(lastLevelWithSideEffects, - BAD_VAR_NUM DEBUGARG(false) DEBUGARG("impPopArgsForUnmanagedCall - side effect")); - - /* Record the level for the current side effect in case we will spill it */ - lastLevelWithSideEffects = level; - } - else - { - /* This is the first side effect encountered - record its level */ - - lastLevelWithSideEffects = level; - } - } - } - - /* The argument list is now "clean" - no out-of-order side effects - * Pop the argument list in reverse order */ - - impPopReverseCallArgs(sig, call, sig->numArgs - argsToReverse); - - if (call->unmgdCallConv == CorInfoCallConvExtension::Thiscall) - { - GenTree* thisPtr = call->gtArgs.GetArgByIndex(0)->GetNode(); - impBashVarAddrsToI(thisPtr); - assert(thisPtr->TypeGet() == TYP_I_IMPL || thisPtr->TypeGet() == TYP_BYREF); - } - - for (CallArg& arg : call->gtArgs.Args()) - { - GenTree* argNode = arg.GetEarlyNode(); - - // We should not be passing gc typed args to an unmanaged call. - if (varTypeIsGC(argNode->TypeGet())) - { - // Tolerate byrefs by retyping to native int. - // - // This is needed or we'll generate inconsistent GC info - // for this arg at the call site (gc info says byref, - // pinvoke sig says native int). - // - if (argNode->TypeGet() == TYP_BYREF) - { - argNode->ChangeType(TYP_I_IMPL); - } - else - { - assert(!"*** invalid IL: gc ref passed to unmanaged call"); - } - } - } -} - -//------------------------------------------------------------------------ -// impInitClass: Build a node to initialize the class before accessing the -// field if necessary -// -// Arguments: -// pResolvedToken - The CORINFO_RESOLVED_TOKEN that has been initialized -// by a call to CEEInfo::resolveToken(). -// -// Return Value: If needed, a pointer to the node that will perform the class -// initializtion. Otherwise, nullptr. -// - -GenTree* Compiler::impInitClass(CORINFO_RESOLVED_TOKEN* pResolvedToken) -{ - CorInfoInitClassResult initClassResult = - info.compCompHnd->initClass(pResolvedToken->hField, info.compMethodHnd, impTokenLookupContextHandle); - - if ((initClassResult & CORINFO_INITCLASS_USE_HELPER) == 0) - { - return nullptr; - } - bool runtimeLookup; - - GenTree* node = impParentClassTokenToHandle(pResolvedToken, &runtimeLookup); - - if (node == nullptr) - { - assert(compDonotInline()); - return nullptr; - } - - if (runtimeLookup) - { - node = gtNewHelperCallNode(CORINFO_HELP_INITCLASS, TYP_VOID, node); - } - else - { - // Call the shared non gc static helper, as its the fastest - node = fgGetSharedCCtor(pResolvedToken->hClass); - } - - return node; -} - -GenTree* Compiler::impImportStaticReadOnlyField(void* fldAddr, var_types lclTyp) -{ - GenTree* op1 = nullptr; - -#if defined(DEBUG) - // If we're replaying under SuperPMI, we're going to read the data stored by SuperPMI and use it - // for optimization. Unfortunately, SuperPMI doesn't implement a guarantee on the alignment of - // this data, so for some platforms which don't allow unaligned access (e.g., Linux arm32), - // this can fault. We should fix SuperPMI to guarantee alignment, but that is a big change. - // Instead, simply fix up the data here for future use. - - // This variable should be the largest size element, with the largest alignment requirement, - // and the native C++ compiler should guarantee sufficient alignment. - double aligned_data = 0.0; - void* p_aligned_data = &aligned_data; - if (info.compMethodSuperPMIIndex != -1) - { - switch (lclTyp) - { - case TYP_BOOL: - case TYP_BYTE: - case TYP_UBYTE: - static_assert_no_msg(sizeof(unsigned __int8) == sizeof(bool)); - static_assert_no_msg(sizeof(unsigned __int8) == sizeof(signed char)); - static_assert_no_msg(sizeof(unsigned __int8) == sizeof(unsigned char)); - // No alignment necessary for byte. - break; - - case TYP_SHORT: - case TYP_USHORT: - static_assert_no_msg(sizeof(unsigned __int16) == sizeof(short)); - static_assert_no_msg(sizeof(unsigned __int16) == sizeof(unsigned short)); - if ((size_t)fldAddr % sizeof(unsigned __int16) != 0) - { - *(unsigned __int16*)p_aligned_data = GET_UNALIGNED_16(fldAddr); - fldAddr = p_aligned_data; - } - break; - - case TYP_INT: - case TYP_UINT: - case TYP_FLOAT: - static_assert_no_msg(sizeof(unsigned __int32) == sizeof(int)); - static_assert_no_msg(sizeof(unsigned __int32) == sizeof(unsigned int)); - static_assert_no_msg(sizeof(unsigned __int32) == sizeof(float)); - if ((size_t)fldAddr % sizeof(unsigned __int32) != 0) - { - *(unsigned __int32*)p_aligned_data = GET_UNALIGNED_32(fldAddr); - fldAddr = p_aligned_data; - } - break; - - case TYP_LONG: - case TYP_ULONG: - case TYP_DOUBLE: - static_assert_no_msg(sizeof(unsigned __int64) == sizeof(__int64)); - static_assert_no_msg(sizeof(unsigned __int64) == sizeof(double)); - if ((size_t)fldAddr % sizeof(unsigned __int64) != 0) - { - *(unsigned __int64*)p_aligned_data = GET_UNALIGNED_64(fldAddr); - fldAddr = p_aligned_data; - } - break; - - default: - assert(!"Unexpected lclTyp"); - break; - } - } -#endif // DEBUG - - switch (lclTyp) - { - int ival; - __int64 lval; - double dval; - - case TYP_BOOL: - ival = *((bool*)fldAddr); - goto IVAL_COMMON; - - case TYP_BYTE: - ival = *((signed char*)fldAddr); - goto IVAL_COMMON; - - case TYP_UBYTE: - ival = *((unsigned char*)fldAddr); - goto IVAL_COMMON; - - case TYP_SHORT: - ival = *((short*)fldAddr); - goto IVAL_COMMON; - - case TYP_USHORT: - ival = *((unsigned short*)fldAddr); - goto IVAL_COMMON; - - case TYP_UINT: - case TYP_INT: - ival = *((int*)fldAddr); - IVAL_COMMON: - op1 = gtNewIconNode(ival); - break; - - case TYP_LONG: - case TYP_ULONG: - lval = *((__int64*)fldAddr); - op1 = gtNewLconNode(lval); - break; - - case TYP_FLOAT: - dval = *((float*)fldAddr); - op1 = gtNewDconNode(dval); - op1->gtType = TYP_FLOAT; - break; - - case TYP_DOUBLE: - dval = *((double*)fldAddr); - op1 = gtNewDconNode(dval); - break; - - default: - assert(!"Unexpected lclTyp"); - break; - } - - return op1; -} - -GenTree* Compiler::impImportStaticFieldAccess(CORINFO_RESOLVED_TOKEN* pResolvedToken, - CORINFO_ACCESS_FLAGS access, - CORINFO_FIELD_INFO* pFieldInfo, - var_types lclTyp) -{ - // Ordinary static fields never overlap. RVA statics, however, can overlap (if they're - // mapped to the same ".data" declaration). That said, such mappings only appear to be - // possible with ILASM, and in ILASM-produced (ILONLY) images, RVA statics are always - // read-only (using "stsfld" on them is UB). In mixed-mode assemblies, RVA statics can - // be mutable, but the only current producer of such images, the C++/CLI compiler, does - // not appear to support mapping different fields to the same address. So we will say - // that "mutable overlapping RVA statics" are UB as well. - - // For statics that are not "boxed", the initial address tree will contain the field sequence. - // For those that are, we will attach it later, when adding the indirection for the box, since - // that tree will represent the true address. - bool isBoxedStatic = (pFieldInfo->fieldFlags & CORINFO_FLG_FIELD_STATIC_IN_HEAP) != 0; - bool isSharedStatic = (pFieldInfo->fieldAccessor == CORINFO_FIELD_STATIC_GENERICS_STATIC_HELPER) || - (pFieldInfo->fieldAccessor == CORINFO_FIELD_STATIC_READYTORUN_HELPER); - FieldSeq::FieldKind fieldKind = - isSharedStatic ? FieldSeq::FieldKind::SharedStatic : FieldSeq::FieldKind::SimpleStatic; - - FieldSeq* innerFldSeq; - FieldSeq* outerFldSeq; - if (isBoxedStatic) - { - innerFldSeq = nullptr; - outerFldSeq = GetFieldSeqStore()->Create(pResolvedToken->hField, TARGET_POINTER_SIZE, fieldKind); - } - else - { - bool hasConstAddr = (pFieldInfo->fieldAccessor == CORINFO_FIELD_STATIC_ADDRESS) || - (pFieldInfo->fieldAccessor == CORINFO_FIELD_STATIC_RVA_ADDRESS); - - ssize_t offset; - if (hasConstAddr) - { - offset = reinterpret_cast(info.compCompHnd->getFieldAddress(pResolvedToken->hField)); - assert(offset != 0); - } - else - { - offset = pFieldInfo->offset; - } - - innerFldSeq = GetFieldSeqStore()->Create(pResolvedToken->hField, offset, fieldKind); - outerFldSeq = nullptr; - } - - GenTree* op1; - switch (pFieldInfo->fieldAccessor) - { - case CORINFO_FIELD_STATIC_GENERICS_STATIC_HELPER: - { - assert(!compIsForInlining()); - - // We first call a special helper to get the statics base pointer - op1 = impParentClassTokenToHandle(pResolvedToken); - - // compIsForInlining() is false so we should not get NULL here - assert(op1 != nullptr); - - var_types type = TYP_BYREF; - - switch (pFieldInfo->helper) - { - case CORINFO_HELP_GETGENERICS_NONGCTHREADSTATIC_BASE: - type = TYP_I_IMPL; - break; - case CORINFO_HELP_GETGENERICS_GCSTATIC_BASE: - case CORINFO_HELP_GETGENERICS_NONGCSTATIC_BASE: - case CORINFO_HELP_GETGENERICS_GCTHREADSTATIC_BASE: - break; - default: - assert(!"unknown generic statics helper"); - break; - } - - op1 = gtNewHelperCallNode(pFieldInfo->helper, type, op1); - op1 = gtNewOperNode(GT_ADD, type, op1, gtNewIconNode(pFieldInfo->offset, innerFldSeq)); - } - break; - - case CORINFO_FIELD_STATIC_SHARED_STATIC_HELPER: - { -#ifdef FEATURE_READYTORUN - if (opts.IsReadyToRun()) - { - GenTreeFlags callFlags = GTF_EMPTY; - - if (info.compCompHnd->getClassAttribs(pResolvedToken->hClass) & CORINFO_FLG_BEFOREFIELDINIT) - { - callFlags |= GTF_CALL_HOISTABLE; - } - - op1 = gtNewHelperCallNode(CORINFO_HELP_READYTORUN_STATIC_BASE, TYP_BYREF); - op1->gtFlags |= callFlags; - - op1->AsCall()->setEntryPoint(pFieldInfo->fieldLookup); - } - else -#endif - { - op1 = fgGetStaticsCCtorHelper(pResolvedToken->hClass, pFieldInfo->helper); - } - - op1 = gtNewOperNode(GT_ADD, op1->TypeGet(), op1, gtNewIconNode(pFieldInfo->offset, innerFldSeq)); - break; - } - - case CORINFO_FIELD_STATIC_READYTORUN_HELPER: - { -#ifdef FEATURE_READYTORUN - assert(opts.IsReadyToRun()); - assert(!compIsForInlining()); - CORINFO_LOOKUP_KIND kind; - info.compCompHnd->getLocationOfThisType(info.compMethodHnd, &kind); - assert(kind.needsRuntimeLookup); - - GenTree* ctxTree = getRuntimeContextTree(kind.runtimeLookupKind); - - GenTreeFlags callFlags = GTF_EMPTY; - - if (info.compCompHnd->getClassAttribs(pResolvedToken->hClass) & CORINFO_FLG_BEFOREFIELDINIT) - { - callFlags |= GTF_CALL_HOISTABLE; - } - var_types type = TYP_BYREF; - op1 = gtNewHelperCallNode(CORINFO_HELP_READYTORUN_GENERIC_STATIC_BASE, type, ctxTree); - op1->gtFlags |= callFlags; - - op1->AsCall()->setEntryPoint(pFieldInfo->fieldLookup); - op1 = gtNewOperNode(GT_ADD, type, op1, gtNewIconNode(pFieldInfo->offset, innerFldSeq)); -#else - unreached(); -#endif // FEATURE_READYTORUN - } - break; - - default: - { - // Do we need the address of a static field? - // - if (access & CORINFO_ACCESS_ADDRESS) - { - void** pFldAddr = nullptr; - void* fldAddr = info.compCompHnd->getFieldAddress(pResolvedToken->hField, (void**)&pFldAddr); - - // We should always be able to access this static's address directly. - assert(pFldAddr == nullptr); - - // Create the address node. - GenTreeFlags handleKind = isBoxedStatic ? GTF_ICON_STATIC_BOX_PTR : GTF_ICON_STATIC_HDL; - op1 = gtNewIconHandleNode((size_t)fldAddr, handleKind, innerFldSeq); - INDEBUG(op1->AsIntCon()->gtTargetHandle = reinterpret_cast(pResolvedToken->hField)); - - if (pFieldInfo->fieldFlags & CORINFO_FLG_FIELD_INITCLASS) - { - op1->gtFlags |= GTF_ICON_INITCLASS; - } - } - else // We need the value of a static field - { - // In future, it may be better to just create the right tree here instead of folding it later. - op1 = gtNewFieldRef(lclTyp, pResolvedToken->hField); - - if (pFieldInfo->fieldFlags & CORINFO_FLG_FIELD_INITCLASS) - { - op1->gtFlags |= GTF_FLD_INITCLASS; - } - - if (isBoxedStatic) - { - op1->ChangeType(TYP_REF); // points at boxed object - op1 = gtNewOperNode(GT_ADD, TYP_BYREF, op1, gtNewIconNode(TARGET_POINTER_SIZE, outerFldSeq)); - - if (varTypeIsStruct(lclTyp)) - { - // Constructor adds GTF_GLOB_REF. Note that this is *not* GTF_EXCEPT. - op1 = gtNewObjNode(pFieldInfo->structType, op1); - } - else - { - op1 = gtNewOperNode(GT_IND, lclTyp, op1); - op1->gtFlags |= (GTF_GLOB_REF | GTF_IND_NONFAULTING); - } - } - - return op1; - } - break; - } - } - - if (isBoxedStatic) - { - op1 = gtNewOperNode(GT_IND, TYP_REF, op1); - op1->gtFlags |= (GTF_IND_INVARIANT | GTF_IND_NONFAULTING | GTF_IND_NONNULL); - - op1 = gtNewOperNode(GT_ADD, TYP_BYREF, op1, gtNewIconNode(TARGET_POINTER_SIZE, outerFldSeq)); - } - - if (!(access & CORINFO_ACCESS_ADDRESS)) - { - if (varTypeIsStruct(lclTyp)) - { - // Constructor adds GTF_GLOB_REF. Note that this is *not* GTF_EXCEPT. - op1 = gtNewObjNode(pFieldInfo->structType, op1); - } - else - { - op1 = gtNewOperNode(GT_IND, lclTyp, op1); - op1->gtFlags |= GTF_GLOB_REF; - } - } - - return op1; -} - -// In general try to call this before most of the verification work. Most people expect the access -// exceptions before the verification exceptions. If you do this after, that usually doesn't happen. Turns -// out if you can't access something we also think that you're unverifiable for other reasons. -void Compiler::impHandleAccessAllowed(CorInfoIsAccessAllowedResult result, CORINFO_HELPER_DESC* helperCall) -{ - if (result != CORINFO_ACCESS_ALLOWED) - { - impHandleAccessAllowedInternal(result, helperCall); - } -} - -void Compiler::impHandleAccessAllowedInternal(CorInfoIsAccessAllowedResult result, CORINFO_HELPER_DESC* helperCall) -{ - switch (result) - { - case CORINFO_ACCESS_ALLOWED: - break; - case CORINFO_ACCESS_ILLEGAL: - // if we're verifying, then we need to reject the illegal access to ensure that we don't think the - // method is verifiable. Otherwise, delay the exception to runtime. - if (compIsForImportOnly()) - { - info.compCompHnd->ThrowExceptionForHelper(helperCall); - } - else - { - impInsertHelperCall(helperCall); - } - break; - } -} - -void Compiler::impInsertHelperCall(CORINFO_HELPER_DESC* helperInfo) -{ - assert(helperInfo->helperNum != CORINFO_HELP_UNDEF); - - /* TODO-Review: - * Mark as CSE'able, and hoistable. Consider marking hoistable unless you're in the inlinee. - * Also, consider sticking this in the first basic block. - */ - GenTreeCall* callout = gtNewHelperCallNode(helperInfo->helperNum, TYP_VOID); - // Add the arguments - for (unsigned i = helperInfo->numArgs; i > 0; --i) - { - const CORINFO_HELPER_ARG& helperArg = helperInfo->args[i - 1]; - GenTree* currentArg = nullptr; - switch (helperArg.argType) - { - case CORINFO_HELPER_ARG_TYPE_Field: - info.compCompHnd->classMustBeLoadedBeforeCodeIsRun( - info.compCompHnd->getFieldClass(helperArg.fieldHandle)); - currentArg = gtNewIconEmbFldHndNode(helperArg.fieldHandle); - break; - case CORINFO_HELPER_ARG_TYPE_Method: - info.compCompHnd->methodMustBeLoadedBeforeCodeIsRun(helperArg.methodHandle); - currentArg = gtNewIconEmbMethHndNode(helperArg.methodHandle); - break; - case CORINFO_HELPER_ARG_TYPE_Class: - info.compCompHnd->classMustBeLoadedBeforeCodeIsRun(helperArg.classHandle); - currentArg = gtNewIconEmbClsHndNode(helperArg.classHandle); - break; - case CORINFO_HELPER_ARG_TYPE_Module: - currentArg = gtNewIconEmbScpHndNode(helperArg.moduleHandle); - break; - case CORINFO_HELPER_ARG_TYPE_Const: - currentArg = gtNewIconNode(helperArg.constant); - break; - default: - NO_WAY("Illegal helper arg type"); - } - callout->gtArgs.PushFront(this, NewCallArg::Primitive(currentArg)); - } - - impAppendTree(callout, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); -} - -//------------------------------------------------------------------------ -// impTailCallRetTypeCompatible: Checks whether the return types of caller -// and callee are compatible so that calle can be tail called. -// sizes are not supported integral type sizes return values to temps. -// -// Arguments: -// allowWidening -- whether to allow implicit widening by the callee. -// For instance, allowing int32 -> int16 tailcalls. -// The managed calling convention allows this, but -// we don't want explicit tailcalls to depend on this -// detail of the managed calling convention. -// callerRetType -- the caller's return type -// callerRetTypeClass - the caller's return struct type -// callerCallConv -- calling convention of the caller -// calleeRetType -- the callee's return type -// calleeRetTypeClass - the callee return struct type -// calleeCallConv -- calling convention of the callee -// -// Returns: -// True if the tailcall types are compatible. -// -// Remarks: -// Note that here we don't check compatibility in IL Verifier sense, but on the -// lines of return types getting returned in the same return register. -bool Compiler::impTailCallRetTypeCompatible(bool allowWidening, - var_types callerRetType, - CORINFO_CLASS_HANDLE callerRetTypeClass, - CorInfoCallConvExtension callerCallConv, - var_types calleeRetType, - CORINFO_CLASS_HANDLE calleeRetTypeClass, - CorInfoCallConvExtension calleeCallConv) -{ - // Early out if the types are the same. - if (callerRetType == calleeRetType) - { - return true; - } - - // For integral types the managed calling convention dictates that callee - // will widen the return value to 4 bytes, so we can allow implicit widening - // in managed to managed tailcalls when dealing with <= 4 bytes. - bool isManaged = - (callerCallConv == CorInfoCallConvExtension::Managed) && (calleeCallConv == CorInfoCallConvExtension::Managed); - - if (allowWidening && isManaged && varTypeIsIntegral(callerRetType) && varTypeIsIntegral(calleeRetType) && - (genTypeSize(callerRetType) <= 4) && (genTypeSize(calleeRetType) <= genTypeSize(callerRetType))) - { - return true; - } - - // If the class handles are the same and not null, the return types are compatible. - if ((callerRetTypeClass != nullptr) && (callerRetTypeClass == calleeRetTypeClass)) - { - return true; - } - -#if defined(TARGET_AMD64) || defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) - // Jit64 compat: - if (callerRetType == TYP_VOID) - { - // This needs to be allowed to support the following IL pattern that Jit64 allows: - // tail.call - // pop - // ret - // - // Note that the above IL pattern is not valid as per IL verification rules. - // Therefore, only full trust code can take advantage of this pattern. - return true; - } - - // These checks return true if the return value type sizes are the same and - // get returned in the same return register i.e. caller doesn't need to normalize - // return value. Some of the tail calls permitted by below checks would have - // been rejected by IL Verifier before we reached here. Therefore, only full - // trust code can make those tail calls. - unsigned callerRetTypeSize = 0; - unsigned calleeRetTypeSize = 0; - bool isCallerRetTypMBEnreg = VarTypeIsMultiByteAndCanEnreg(callerRetType, callerRetTypeClass, &callerRetTypeSize, - true, info.compIsVarArgs, callerCallConv); - bool isCalleeRetTypMBEnreg = VarTypeIsMultiByteAndCanEnreg(calleeRetType, calleeRetTypeClass, &calleeRetTypeSize, - true, info.compIsVarArgs, calleeCallConv); - - if (varTypeIsIntegral(callerRetType) || isCallerRetTypMBEnreg) - { - return (varTypeIsIntegral(calleeRetType) || isCalleeRetTypMBEnreg) && (callerRetTypeSize == calleeRetTypeSize); - } -#endif // TARGET_AMD64 || TARGET_ARM64 || TARGET_LOONGARCH64 - - return false; -} - -/******************************************************************************** - * - * Returns true if the current opcode and and the opcodes following it correspond - * to a supported tail call IL pattern. - * - */ -bool Compiler::impIsTailCallILPattern( - bool tailPrefixed, OPCODE curOpcode, const BYTE* codeAddrOfNextOpcode, const BYTE* codeEnd, bool isRecursive) -{ - // Bail out if the current opcode is not a call. - if (!impOpcodeIsCallOpcode(curOpcode)) - { - return false; - } - -#if !FEATURE_TAILCALL_OPT_SHARED_RETURN - // If shared ret tail opt is not enabled, we will enable - // it for recursive methods. - if (isRecursive) -#endif - { - // we can actually handle if the ret is in a fallthrough block, as long as that is the only part of the - // sequence. Make sure we don't go past the end of the IL however. - codeEnd = min(codeEnd + 1, info.compCode + info.compILCodeSize); - } - - // Bail out if there is no next opcode after call - if (codeAddrOfNextOpcode >= codeEnd) - { - return false; - } - - OPCODE nextOpcode = (OPCODE)getU1LittleEndian(codeAddrOfNextOpcode); - - return (nextOpcode == CEE_RET); -} - -/***************************************************************************** - * - * Determine whether the call could be converted to an implicit tail call - * - */ -bool Compiler::impIsImplicitTailCallCandidate( - OPCODE opcode, const BYTE* codeAddrOfNextOpcode, const BYTE* codeEnd, int prefixFlags, bool isRecursive) -{ - -#if FEATURE_TAILCALL_OPT - if (!opts.compTailCallOpt) - { - return false; - } - - if (opts.OptimizationDisabled()) - { - return false; - } - - // must not be tail prefixed - if (prefixFlags & PREFIX_TAILCALL_EXPLICIT) - { - return false; - } - -#if !FEATURE_TAILCALL_OPT_SHARED_RETURN - // the block containing call is marked as BBJ_RETURN - // We allow shared ret tail call optimization on recursive calls even under - // !FEATURE_TAILCALL_OPT_SHARED_RETURN. - if (!isRecursive && (compCurBB->bbJumpKind != BBJ_RETURN)) - return false; -#endif // !FEATURE_TAILCALL_OPT_SHARED_RETURN - - // must be call+ret or call+pop+ret - if (!impIsTailCallILPattern(false, opcode, codeAddrOfNextOpcode, codeEnd, isRecursive)) - { - return false; - } - - return true; -#else - return false; -#endif // FEATURE_TAILCALL_OPT -} - -//------------------------------------------------------------------------ -// impImportCall: import a call-inspiring opcode -// -// Arguments: -// opcode - opcode that inspires the call -// pResolvedToken - resolved token for the call target -// pConstrainedResolvedToken - resolved constraint token (or nullptr) -// newObjThis - tree for this pointer or uninitialized newobj temp (or nullptr) -// prefixFlags - IL prefix flags for the call -// callInfo - EE supplied info for the call -// rawILOffset - IL offset of the opcode, used for guarded devirtualization. -// -// Returns: -// Type of the call's return value. -// If we're importing an inlinee and have realized the inline must fail, the call return type should be TYP_UNDEF. -// However we can't assert for this here yet because there are cases we miss. See issue #13272. -// -// -// Notes: -// opcode can be CEE_CALL, CEE_CALLI, CEE_CALLVIRT, or CEE_NEWOBJ. -// -// For CEE_NEWOBJ, newobjThis should be the temp grabbed for the allocated -// uninitialized object. - -#ifdef _PREFAST_ -#pragma warning(push) -#pragma warning(disable : 21000) // Suppress PREFast warning about overly large function -#endif - -var_types Compiler::impImportCall(OPCODE opcode, - CORINFO_RESOLVED_TOKEN* pResolvedToken, - CORINFO_RESOLVED_TOKEN* pConstrainedResolvedToken, - GenTree* newobjThis, - int prefixFlags, - CORINFO_CALL_INFO* callInfo, - IL_OFFSET rawILOffset) -{ - assert(opcode == CEE_CALL || opcode == CEE_CALLVIRT || opcode == CEE_NEWOBJ || opcode == CEE_CALLI); - - // The current statement DI may not refer to the exact call, but for calls - // we wish to be able to attach the exact IL instruction to get "return - // value" support in the debugger, so create one with the exact IL offset. - DebugInfo di = impCreateDIWithCurrentStackInfo(rawILOffset, true); - - var_types callRetTyp = TYP_COUNT; - CORINFO_SIG_INFO* sig = nullptr; - CORINFO_METHOD_HANDLE methHnd = nullptr; - CORINFO_CLASS_HANDLE clsHnd = nullptr; - unsigned clsFlags = 0; - unsigned mflags = 0; - GenTree* call = nullptr; - CORINFO_THIS_TRANSFORM constraintCallThisTransform = CORINFO_NO_THIS_TRANSFORM; - CORINFO_CONTEXT_HANDLE exactContextHnd = nullptr; - bool exactContextNeedsRuntimeLookup = false; - bool canTailCall = true; - const char* szCanTailCallFailReason = nullptr; - const int tailCallFlags = (prefixFlags & PREFIX_TAILCALL); - const bool isReadonlyCall = (prefixFlags & PREFIX_READONLY) != 0; - - methodPointerInfo* ldftnInfo = nullptr; - - // Synchronized methods need to call CORINFO_HELP_MON_EXIT at the end. We could - // do that before tailcalls, but that is probably not the intended - // semantic. So just disallow tailcalls from synchronized methods. - // Also, popping arguments in a varargs function is more work and NYI - // If we have a security object, we have to keep our frame around for callers - // to see any imperative security. - // Reverse P/Invokes need a call to CORINFO_HELP_JIT_REVERSE_PINVOKE_EXIT - // at the end, so tailcalls should be disabled. - if (info.compFlags & CORINFO_FLG_SYNCH) - { - canTailCall = false; - szCanTailCallFailReason = "Caller is synchronized"; - } - else if (opts.IsReversePInvoke()) - { - canTailCall = false; - szCanTailCallFailReason = "Caller is Reverse P/Invoke"; - } -#if !FEATURE_FIXED_OUT_ARGS - else if (info.compIsVarArgs) - { - canTailCall = false; - szCanTailCallFailReason = "Caller is varargs"; - } -#endif // FEATURE_FIXED_OUT_ARGS - - // We only need to cast the return value of pinvoke inlined calls that return small types - - bool checkForSmallType = false; - bool bIntrinsicImported = false; - - CORINFO_SIG_INFO calliSig; - NewCallArg extraArg; - - /*------------------------------------------------------------------------- - * First create the call node - */ - - if (opcode == CEE_CALLI) - { - if (IsTargetAbi(CORINFO_NATIVEAOT_ABI)) - { - // See comment in impCheckForPInvokeCall - BasicBlock* block = compIsForInlining() ? impInlineInfo->iciBlock : compCurBB; - if (info.compCompHnd->convertPInvokeCalliToCall(pResolvedToken, !impCanPInvokeInlineCallSite(block))) - { - eeGetCallInfo(pResolvedToken, nullptr, CORINFO_CALLINFO_ALLOWINSTPARAM, callInfo); - return impImportCall(CEE_CALL, pResolvedToken, nullptr, nullptr, prefixFlags, callInfo, rawILOffset); - } - } - - /* Get the call site sig */ - eeGetSig(pResolvedToken->token, pResolvedToken->tokenScope, pResolvedToken->tokenContext, &calliSig); - - callRetTyp = JITtype2varType(calliSig.retType); - - call = impImportIndirectCall(&calliSig, di); - - // We don't know the target method, so we have to infer the flags, or - // assume the worst-case. - mflags = (calliSig.callConv & CORINFO_CALLCONV_HASTHIS) ? 0 : CORINFO_FLG_STATIC; - -#ifdef DEBUG - if (verbose) - { - unsigned structSize = (callRetTyp == TYP_STRUCT) ? eeTryGetClassSize(calliSig.retTypeSigClass) : 0; - printf("\nIn Compiler::impImportCall: opcode is %s, kind=%d, callRetType is %s, structSize is %u\n", - opcodeNames[opcode], callInfo->kind, varTypeName(callRetTyp), structSize); - } -#endif - sig = &calliSig; - } - else // (opcode != CEE_CALLI) - { - NamedIntrinsic ni = NI_Illegal; - - // Passing CORINFO_CALLINFO_ALLOWINSTPARAM indicates that this JIT is prepared to - // supply the instantiation parameters necessary to make direct calls to underlying - // shared generic code, rather than calling through instantiating stubs. If the - // returned signature has CORINFO_CALLCONV_PARAMTYPE then this indicates that the JIT - // must indeed pass an instantiation parameter. - - methHnd = callInfo->hMethod; - - sig = &(callInfo->sig); - callRetTyp = JITtype2varType(sig->retType); - - mflags = callInfo->methodFlags; - -#ifdef DEBUG - if (verbose) - { - unsigned structSize = (callRetTyp == TYP_STRUCT) ? eeTryGetClassSize(sig->retTypeSigClass) : 0; - printf("\nIn Compiler::impImportCall: opcode is %s, kind=%d, callRetType is %s, structSize is %u\n", - opcodeNames[opcode], callInfo->kind, varTypeName(callRetTyp), structSize); - } -#endif - if (compIsForInlining()) - { - /* Does the inlinee use StackCrawlMark */ - - if (mflags & CORINFO_FLG_DONT_INLINE_CALLER) - { - compInlineResult->NoteFatal(InlineObservation::CALLEE_STACK_CRAWL_MARK); - return TYP_UNDEF; - } - - /* For now ignore varargs */ - if ((sig->callConv & CORINFO_CALLCONV_MASK) == CORINFO_CALLCONV_NATIVEVARARG) - { - compInlineResult->NoteFatal(InlineObservation::CALLEE_HAS_NATIVE_VARARGS); - return TYP_UNDEF; - } - - if ((sig->callConv & CORINFO_CALLCONV_MASK) == CORINFO_CALLCONV_VARARG) - { - compInlineResult->NoteFatal(InlineObservation::CALLEE_HAS_MANAGED_VARARGS); - return TYP_UNDEF; - } - - if ((mflags & CORINFO_FLG_VIRTUAL) && (sig->sigInst.methInstCount != 0) && (opcode == CEE_CALLVIRT)) - { - compInlineResult->NoteFatal(InlineObservation::CALLEE_IS_GENERIC_VIRTUAL); - return TYP_UNDEF; - } - } - - clsHnd = pResolvedToken->hClass; - - clsFlags = callInfo->classFlags; - -#ifdef DEBUG - // If this is a call to JitTestLabel.Mark, do "early inlining", and record the test attribute. - - // This recognition should really be done by knowing the methHnd of the relevant Mark method(s). - // These should be in corelib.h, and available through a JIT/EE interface call. - const char* modName; - const char* className; - const char* methodName; - if ((className = eeGetClassName(clsHnd)) != nullptr && - strcmp(className, "System.Runtime.CompilerServices.JitTestLabel") == 0 && - (methodName = eeGetMethodName(methHnd, &modName)) != nullptr && strcmp(methodName, "Mark") == 0) - { - return impImportJitTestLabelMark(sig->numArgs); - } -#endif // DEBUG - - // Factor this into getCallInfo - bool isSpecialIntrinsic = false; - if ((mflags & CORINFO_FLG_INTRINSIC) != 0) - { - const bool isTailCall = canTailCall && (tailCallFlags != 0); - - call = - impIntrinsic(newobjThis, clsHnd, methHnd, sig, mflags, pResolvedToken->token, isReadonlyCall, - isTailCall, pConstrainedResolvedToken, callInfo->thisTransform, &ni, &isSpecialIntrinsic); - - if (compDonotInline()) - { - return TYP_UNDEF; - } - - if (call != nullptr) - { -#ifdef FEATURE_READYTORUN - if (call->OperGet() == GT_INTRINSIC) - { - if (opts.IsReadyToRun()) - { - noway_assert(callInfo->kind == CORINFO_CALL); - call->AsIntrinsic()->gtEntryPoint = callInfo->codePointerLookup.constLookup; - } - else - { - call->AsIntrinsic()->gtEntryPoint.addr = nullptr; - call->AsIntrinsic()->gtEntryPoint.accessType = IAT_VALUE; - } - } -#endif - - bIntrinsicImported = true; - goto DONE_CALL; - } - } - -#ifdef FEATURE_SIMD - call = impSIMDIntrinsic(opcode, newobjThis, clsHnd, methHnd, sig, mflags, pResolvedToken->token); - if (call != nullptr) - { - bIntrinsicImported = true; - goto DONE_CALL; - } -#endif // FEATURE_SIMD - - if ((mflags & CORINFO_FLG_VIRTUAL) && (mflags & CORINFO_FLG_EnC) && (opcode == CEE_CALLVIRT)) - { - NO_WAY("Virtual call to a function added via EnC is not supported"); - } - - if ((sig->callConv & CORINFO_CALLCONV_MASK) != CORINFO_CALLCONV_DEFAULT && - (sig->callConv & CORINFO_CALLCONV_MASK) != CORINFO_CALLCONV_VARARG && - (sig->callConv & CORINFO_CALLCONV_MASK) != CORINFO_CALLCONV_NATIVEVARARG) - { - BADCODE("Bad calling convention"); - } - - //------------------------------------------------------------------------- - // Construct the call node - // - // Work out what sort of call we're making. - // Dispense with virtual calls implemented via LDVIRTFTN immediately. - - constraintCallThisTransform = callInfo->thisTransform; - exactContextHnd = callInfo->contextHandle; - exactContextNeedsRuntimeLookup = callInfo->exactContextNeedsRuntimeLookup; - - switch (callInfo->kind) - { - case CORINFO_VIRTUALCALL_STUB: - { - assert(!(mflags & CORINFO_FLG_STATIC)); // can't call a static method - assert(!(clsFlags & CORINFO_FLG_VALUECLASS)); - if (callInfo->stubLookup.lookupKind.needsRuntimeLookup) - { - if (callInfo->stubLookup.lookupKind.runtimeLookupKind == CORINFO_LOOKUP_NOT_SUPPORTED) - { - // Runtime does not support inlining of all shapes of runtime lookups - // Inlining has to be aborted in such a case - compInlineResult->NoteFatal(InlineObservation::CALLSITE_HAS_COMPLEX_HANDLE); - return TYP_UNDEF; - } - - GenTree* stubAddr = impRuntimeLookupToTree(pResolvedToken, &callInfo->stubLookup, methHnd); - assert(!compDonotInline()); - - // This is the rough code to set up an indirect stub call - assert(stubAddr != nullptr); - - // The stubAddr may be a - // complex expression. As it is evaluated after the args, - // it may cause registered args to be spilled. Simply spill it. - - unsigned lclNum = lvaGrabTemp(true DEBUGARG("VirtualCall with runtime lookup")); - impAssignTempGen(lclNum, stubAddr, (unsigned)CHECK_SPILL_NONE); - stubAddr = gtNewLclvNode(lclNum, TYP_I_IMPL); - - // Create the actual call node - - assert((sig->callConv & CORINFO_CALLCONV_MASK) != CORINFO_CALLCONV_VARARG && - (sig->callConv & CORINFO_CALLCONV_MASK) != CORINFO_CALLCONV_NATIVEVARARG); - - call = gtNewIndCallNode(stubAddr, callRetTyp); - - call->gtFlags |= GTF_EXCEPT | (stubAddr->gtFlags & GTF_GLOB_EFFECT); - call->gtFlags |= GTF_CALL_VIRT_STUB; - -#ifdef TARGET_X86 - // No tailcalls allowed for these yet... - canTailCall = false; - szCanTailCallFailReason = "VirtualCall with runtime lookup"; -#endif - } - else - { - // The stub address is known at compile time - call = gtNewCallNode(CT_USER_FUNC, callInfo->hMethod, callRetTyp, di); - call->AsCall()->gtStubCallStubAddr = callInfo->stubLookup.constLookup.addr; - call->gtFlags |= GTF_CALL_VIRT_STUB; - assert(callInfo->stubLookup.constLookup.accessType != IAT_PPVALUE && - callInfo->stubLookup.constLookup.accessType != IAT_RELPVALUE); - if (callInfo->stubLookup.constLookup.accessType == IAT_PVALUE) - { - call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_VIRTSTUB_REL_INDIRECT; - } - } - -#ifdef FEATURE_READYTORUN - if (opts.IsReadyToRun()) - { - // Null check is sometimes needed for ready to run to handle - // non-virtual <-> virtual changes between versions - if (callInfo->nullInstanceCheck) - { - call->gtFlags |= GTF_CALL_NULLCHECK; - } - } -#endif - - break; - } - - case CORINFO_VIRTUALCALL_VTABLE: - { - assert(!(mflags & CORINFO_FLG_STATIC)); // can't call a static method - assert(!(clsFlags & CORINFO_FLG_VALUECLASS)); - call = gtNewCallNode(CT_USER_FUNC, callInfo->hMethod, callRetTyp, di); - call->gtFlags |= GTF_CALL_VIRT_VTABLE; - - // Should we expand virtual call targets early for this method? - // - if (opts.compExpandCallsEarly) - { - // Mark this method to expand the virtual call target early in fgMorpgCall - call->AsCall()->SetExpandedEarly(); - } - break; - } - - case CORINFO_VIRTUALCALL_LDVIRTFTN: - { - if (compIsForInlining()) - { - compInlineResult->NoteFatal(InlineObservation::CALLSITE_HAS_CALL_VIA_LDVIRTFTN); - return TYP_UNDEF; - } - - assert(!(mflags & CORINFO_FLG_STATIC)); // can't call a static method - assert(!(clsFlags & CORINFO_FLG_VALUECLASS)); - // OK, We've been told to call via LDVIRTFTN, so just - // take the call now.... - call = gtNewIndCallNode(nullptr, callRetTyp, di); - - impPopCallArgs(sig, call->AsCall()); - - GenTree* thisPtr = impPopStack().val; - thisPtr = impTransformThis(thisPtr, pConstrainedResolvedToken, callInfo->thisTransform); - assert(thisPtr != nullptr); - - // Clone the (possibly transformed) "this" pointer - GenTree* thisPtrCopy; - thisPtr = impCloneExpr(thisPtr, &thisPtrCopy, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, - nullptr DEBUGARG("LDVIRTFTN this pointer")); - - GenTree* fptr = impImportLdvirtftn(thisPtr, pResolvedToken, callInfo); - assert(fptr != nullptr); - - call->AsCall() - ->gtArgs.PushFront(this, NewCallArg::Primitive(thisPtrCopy).WellKnown(WellKnownArg::ThisPointer)); - - // Now make an indirect call through the function pointer - - unsigned lclNum = lvaGrabTemp(true DEBUGARG("VirtualCall through function pointer")); - impAssignTempGen(lclNum, fptr, (unsigned)CHECK_SPILL_ALL); - fptr = gtNewLclvNode(lclNum, TYP_I_IMPL); - - call->AsCall()->gtCallAddr = fptr; - call->gtFlags |= GTF_EXCEPT | (fptr->gtFlags & GTF_GLOB_EFFECT); - - if ((sig->sigInst.methInstCount != 0) && IsTargetAbi(CORINFO_NATIVEAOT_ABI)) - { - // NativeAOT generic virtual method: need to handle potential fat function pointers - addFatPointerCandidate(call->AsCall()); - } -#ifdef FEATURE_READYTORUN - if (opts.IsReadyToRun()) - { - // Null check is needed for ready to run to handle - // non-virtual <-> virtual changes between versions - call->gtFlags |= GTF_CALL_NULLCHECK; - } -#endif - - // Sine we are jumping over some code, check that its OK to skip that code - assert((sig->callConv & CORINFO_CALLCONV_MASK) != CORINFO_CALLCONV_VARARG && - (sig->callConv & CORINFO_CALLCONV_MASK) != CORINFO_CALLCONV_NATIVEVARARG); - goto DONE; - } - - case CORINFO_CALL: - { - // This is for a non-virtual, non-interface etc. call - call = gtNewCallNode(CT_USER_FUNC, callInfo->hMethod, callRetTyp, di); - - // We remove the nullcheck for the GetType call intrinsic. - // TODO-CQ: JIT64 does not introduce the null check for many more helper calls - // and intrinsics. - if (callInfo->nullInstanceCheck && - !((mflags & CORINFO_FLG_INTRINSIC) != 0 && (ni == NI_System_Object_GetType))) - { - call->gtFlags |= GTF_CALL_NULLCHECK; - } - -#ifdef FEATURE_READYTORUN - if (opts.IsReadyToRun()) - { - call->AsCall()->setEntryPoint(callInfo->codePointerLookup.constLookup); - } -#endif - break; - } - - case CORINFO_CALL_CODE_POINTER: - { - // The EE has asked us to call by computing a code pointer and then doing an - // indirect call. This is because a runtime lookup is required to get the code entry point. - - // These calls always follow a uniform calling convention, i.e. no extra hidden params - assert((sig->callConv & CORINFO_CALLCONV_PARAMTYPE) == 0); - - assert((sig->callConv & CORINFO_CALLCONV_MASK) != CORINFO_CALLCONV_VARARG); - assert((sig->callConv & CORINFO_CALLCONV_MASK) != CORINFO_CALLCONV_NATIVEVARARG); - - GenTree* fptr = - impLookupToTree(pResolvedToken, &callInfo->codePointerLookup, GTF_ICON_FTN_ADDR, callInfo->hMethod); - - if (compDonotInline()) - { - return TYP_UNDEF; - } - - // Now make an indirect call through the function pointer - - unsigned lclNum = lvaGrabTemp(true DEBUGARG("Indirect call through function pointer")); - impAssignTempGen(lclNum, fptr, (unsigned)CHECK_SPILL_ALL); - fptr = gtNewLclvNode(lclNum, TYP_I_IMPL); - - call = gtNewIndCallNode(fptr, callRetTyp, di); - call->gtFlags |= GTF_EXCEPT | (fptr->gtFlags & GTF_GLOB_EFFECT); - if (callInfo->nullInstanceCheck) - { - call->gtFlags |= GTF_CALL_NULLCHECK; - } - - break; - } - - default: - assert(!"unknown call kind"); - break; - } - - //------------------------------------------------------------------------- - // Set more flags - - PREFIX_ASSUME(call != nullptr); - - if (mflags & CORINFO_FLG_NOGCCHECK) - { - call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_NOGCCHECK; - } - - // Mark call if it's one of the ones we will maybe treat as an intrinsic - if (isSpecialIntrinsic) - { - call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_SPECIAL_INTRINSIC; - } - } - assert(sig); - assert(clsHnd || (opcode == CEE_CALLI)); // We're never verifying for CALLI, so this is not set. - - /* Some sanity checks */ - - // CALL_VIRT and NEWOBJ must have a THIS pointer - assert((opcode != CEE_CALLVIRT && opcode != CEE_NEWOBJ) || (sig->callConv & CORINFO_CALLCONV_HASTHIS)); - // static bit and hasThis are negations of one another - assert(((mflags & CORINFO_FLG_STATIC) != 0) == ((sig->callConv & CORINFO_CALLCONV_HASTHIS) == 0)); - assert(call != nullptr); - - /*------------------------------------------------------------------------- - * Check special-cases etc - */ - - /* Special case - Check if it is a call to Delegate.Invoke(). */ - - if (mflags & CORINFO_FLG_DELEGATE_INVOKE) - { - assert(!(mflags & CORINFO_FLG_STATIC)); // can't call a static method - assert(mflags & CORINFO_FLG_FINAL); - - /* Set the delegate flag */ - call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_DELEGATE_INV; - - if (callInfo->wrapperDelegateInvoke) - { - call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_WRAPPER_DELEGATE_INV; - } - - if (opcode == CEE_CALLVIRT) - { - assert(mflags & CORINFO_FLG_FINAL); - - /* It should have the GTF_CALL_NULLCHECK flag set. Reset it */ - assert(call->gtFlags & GTF_CALL_NULLCHECK); - call->gtFlags &= ~GTF_CALL_NULLCHECK; - } - } - - CORINFO_CLASS_HANDLE actualMethodRetTypeSigClass; - actualMethodRetTypeSigClass = sig->retTypeSigClass; - - /* Check for varargs */ - if (!compFeatureVarArg() && ((sig->callConv & CORINFO_CALLCONV_MASK) == CORINFO_CALLCONV_VARARG || - (sig->callConv & CORINFO_CALLCONV_MASK) == CORINFO_CALLCONV_NATIVEVARARG)) - { - BADCODE("Varargs not supported."); - } - - if ((sig->callConv & CORINFO_CALLCONV_MASK) == CORINFO_CALLCONV_VARARG || - (sig->callConv & CORINFO_CALLCONV_MASK) == CORINFO_CALLCONV_NATIVEVARARG) - { - assert(!compIsForInlining()); - - /* Set the right flags */ - - call->gtFlags |= GTF_CALL_POP_ARGS; - call->AsCall()->gtArgs.SetIsVarArgs(); - - /* Can't allow tailcall for varargs as it is caller-pop. The caller - will be expecting to pop a certain number of arguments, but if we - tailcall to a function with a different number of arguments, we - are hosed. There are ways around this (caller remembers esp value, - varargs is not caller-pop, etc), but not worth it. */ - CLANG_FORMAT_COMMENT_ANCHOR; - -#ifdef TARGET_X86 - if (canTailCall) - { - canTailCall = false; - szCanTailCallFailReason = "Callee is varargs"; - } -#endif - - /* Get the total number of arguments - this is already correct - * for CALLI - for methods we have to get it from the call site */ - - if (opcode != CEE_CALLI) - { -#ifdef DEBUG - unsigned numArgsDef = sig->numArgs; -#endif - eeGetCallSiteSig(pResolvedToken->token, pResolvedToken->tokenScope, pResolvedToken->tokenContext, sig); - - // For vararg calls we must be sure to load the return type of the - // method actually being called, as well as the return types of the - // specified in the vararg signature. With type equivalency, these types - // may not be the same. - if (sig->retTypeSigClass != actualMethodRetTypeSigClass) - { - if (actualMethodRetTypeSigClass != nullptr && sig->retType != CORINFO_TYPE_CLASS && - sig->retType != CORINFO_TYPE_BYREF && sig->retType != CORINFO_TYPE_PTR && - sig->retType != CORINFO_TYPE_VAR) - { - // Make sure that all valuetypes (including enums) that we push are loaded. - // This is to guarantee that if a GC is triggerred from the prestub of this methods, - // all valuetypes in the method signature are already loaded. - // We need to be able to find the size of the valuetypes, but we cannot - // do a class-load from within GC. - info.compCompHnd->classMustBeLoadedBeforeCodeIsRun(actualMethodRetTypeSigClass); - } - } - - assert(numArgsDef <= sig->numArgs); - } - - /* We will have "cookie" as the last argument but we cannot push - * it on the operand stack because we may overflow, so we append it - * to the arg list next after we pop them */ - } - - //--------------------------- Inline NDirect ------------------------------ - - // For inline cases we technically should look at both the current - // block and the call site block (or just the latter if we've - // fused the EH trees). However the block-related checks pertain to - // EH and we currently won't inline a method with EH. So for - // inlinees, just checking the call site block is sufficient. - { - // New lexical block here to avoid compilation errors because of GOTOs. - BasicBlock* block = compIsForInlining() ? impInlineInfo->iciBlock : compCurBB; - impCheckForPInvokeCall(call->AsCall(), methHnd, sig, mflags, block); - } - -#ifdef UNIX_X86_ABI - // On Unix x86 we use caller-cleaned convention. - if ((call->gtFlags & GTF_CALL_UNMANAGED) == 0) - call->gtFlags |= GTF_CALL_POP_ARGS; -#endif // UNIX_X86_ABI - - if (call->gtFlags & GTF_CALL_UNMANAGED) - { - // We set up the unmanaged call by linking the frame, disabling GC, etc - // This needs to be cleaned up on return. - // In addition, native calls have different normalization rules than managed code - // (managed calling convention always widens return values in the callee) - if (canTailCall) - { - canTailCall = false; - szCanTailCallFailReason = "Callee is native"; - } - - checkForSmallType = true; - - impPopArgsForUnmanagedCall(call->AsCall(), sig); - - goto DONE; - } - else if ((opcode == CEE_CALLI) && ((sig->callConv & CORINFO_CALLCONV_MASK) != CORINFO_CALLCONV_DEFAULT) && - ((sig->callConv & CORINFO_CALLCONV_MASK) != CORINFO_CALLCONV_VARARG)) - { - if (!info.compCompHnd->canGetCookieForPInvokeCalliSig(sig)) - { - // Normally this only happens with inlining. - // However, a generic method (or type) being NGENd into another module - // can run into this issue as well. There's not an easy fall-back for NGEN - // so instead we fallback to JIT. - if (compIsForInlining()) - { - compInlineResult->NoteFatal(InlineObservation::CALLSITE_CANT_EMBED_PINVOKE_COOKIE); - } - else - { - IMPL_LIMITATION("Can't get PInvoke cookie (cross module generics)"); - } - - return TYP_UNDEF; - } - - GenTree* cookie = eeGetPInvokeCookie(sig); - - // This cookie is required to be either a simple GT_CNS_INT or - // an indirection of a GT_CNS_INT - // - GenTree* cookieConst = cookie; - if (cookie->gtOper == GT_IND) - { - cookieConst = cookie->AsOp()->gtOp1; - } - assert(cookieConst->gtOper == GT_CNS_INT); - - // Setting GTF_DONT_CSE on the GT_CNS_INT as well as on the GT_IND (if it exists) will ensure that - // we won't allow this tree to participate in any CSE logic - // - cookie->gtFlags |= GTF_DONT_CSE; - cookieConst->gtFlags |= GTF_DONT_CSE; - - call->AsCall()->gtCallCookie = cookie; - - if (canTailCall) - { - canTailCall = false; - szCanTailCallFailReason = "PInvoke calli"; - } - } - - /*------------------------------------------------------------------------- - * Create the argument list - */ - - //------------------------------------------------------------------------- - // Special case - for varargs we have an implicit last argument - - if ((sig->callConv & CORINFO_CALLCONV_MASK) == CORINFO_CALLCONV_VARARG) - { - assert(!compIsForInlining()); - - void *varCookie, *pVarCookie; - if (!info.compCompHnd->canGetVarArgsHandle(sig)) - { - compInlineResult->NoteFatal(InlineObservation::CALLSITE_CANT_EMBED_VARARGS_COOKIE); - return TYP_UNDEF; - } - - varCookie = info.compCompHnd->getVarArgsHandle(sig, &pVarCookie); - assert((!varCookie) != (!pVarCookie)); - GenTree* cookieNode = gtNewIconEmbHndNode(varCookie, pVarCookie, GTF_ICON_VARG_HDL, sig); - assert(extraArg.Node == nullptr); - extraArg = NewCallArg::Primitive(cookieNode).WellKnown(WellKnownArg::VarArgsCookie); - } - - //------------------------------------------------------------------------- - // Extra arg for shared generic code and array methods - // - // Extra argument containing instantiation information is passed in the - // following circumstances: - // (a) To the "Address" method on array classes; the extra parameter is - // the array's type handle (a TypeDesc) - // (b) To shared-code instance methods in generic structs; the extra parameter - // is the struct's type handle (a vtable ptr) - // (c) To shared-code per-instantiation non-generic static methods in generic - // classes and structs; the extra parameter is the type handle - // (d) To shared-code generic methods; the extra parameter is an - // exact-instantiation MethodDesc - // - // We also set the exact type context associated with the call so we can - // inline the call correctly later on. - - if (sig->callConv & CORINFO_CALLCONV_PARAMTYPE) - { - assert(call->AsCall()->gtCallType == CT_USER_FUNC); - if (clsHnd == nullptr) - { - NO_WAY("CALLI on parameterized type"); - } - - assert(opcode != CEE_CALLI); - - GenTree* instParam; - bool runtimeLookup; - - // Instantiated generic method - if (((SIZE_T)exactContextHnd & CORINFO_CONTEXTFLAGS_MASK) == CORINFO_CONTEXTFLAGS_METHOD) - { - assert(exactContextHnd != METHOD_BEING_COMPILED_CONTEXT()); - - CORINFO_METHOD_HANDLE exactMethodHandle = - (CORINFO_METHOD_HANDLE)((SIZE_T)exactContextHnd & ~CORINFO_CONTEXTFLAGS_MASK); - - if (!exactContextNeedsRuntimeLookup) - { -#ifdef FEATURE_READYTORUN - if (opts.IsReadyToRun()) - { - instParam = - impReadyToRunLookupToTree(&callInfo->instParamLookup, GTF_ICON_METHOD_HDL, exactMethodHandle); - if (instParam == nullptr) - { - assert(compDonotInline()); - return TYP_UNDEF; - } - } - else -#endif - { - instParam = gtNewIconEmbMethHndNode(exactMethodHandle); - info.compCompHnd->methodMustBeLoadedBeforeCodeIsRun(exactMethodHandle); - } - } - else - { - instParam = impTokenToHandle(pResolvedToken, &runtimeLookup, true /*mustRestoreHandle*/); - if (instParam == nullptr) - { - assert(compDonotInline()); - return TYP_UNDEF; - } - } - } - - // otherwise must be an instance method in a generic struct, - // a static method in a generic type, or a runtime-generated array method - else - { - assert(((SIZE_T)exactContextHnd & CORINFO_CONTEXTFLAGS_MASK) == CORINFO_CONTEXTFLAGS_CLASS); - CORINFO_CLASS_HANDLE exactClassHandle = eeGetClassFromContext(exactContextHnd); - - if (compIsForInlining() && (clsFlags & CORINFO_FLG_ARRAY) != 0) - { - compInlineResult->NoteFatal(InlineObservation::CALLEE_IS_ARRAY_METHOD); - return TYP_UNDEF; - } - - if ((clsFlags & CORINFO_FLG_ARRAY) && isReadonlyCall) - { - // We indicate "readonly" to the Address operation by using a null - // instParam. - instParam = gtNewIconNode(0, TYP_REF); - } - else if (!exactContextNeedsRuntimeLookup) - { -#ifdef FEATURE_READYTORUN - if (opts.IsReadyToRun()) - { - instParam = - impReadyToRunLookupToTree(&callInfo->instParamLookup, GTF_ICON_CLASS_HDL, exactClassHandle); - if (instParam == nullptr) - { - assert(compDonotInline()); - return TYP_UNDEF; - } - } - else -#endif - { - instParam = gtNewIconEmbClsHndNode(exactClassHandle); - info.compCompHnd->classMustBeLoadedBeforeCodeIsRun(exactClassHandle); - } - } - else - { - instParam = impParentClassTokenToHandle(pResolvedToken, &runtimeLookup, true /*mustRestoreHandle*/); - if (instParam == nullptr) - { - assert(compDonotInline()); - return TYP_UNDEF; - } - } - } - - assert(extraArg.Node == nullptr); - extraArg = NewCallArg::Primitive(instParam).WellKnown(WellKnownArg::InstParam); - } - - if ((opcode == CEE_NEWOBJ) && ((clsFlags & CORINFO_FLG_DELEGATE) != 0)) - { - // Only verifiable cases are supported. - // dup; ldvirtftn; newobj; or ldftn; newobj. - // IL test could contain unverifiable sequence, in this case optimization should not be done. - if (impStackHeight() > 0) - { - typeInfo delegateTypeInfo = impStackTop().seTypeInfo; - if (delegateTypeInfo.IsMethod()) - { - ldftnInfo = delegateTypeInfo.GetMethodPointerInfo(); - } - } - } - - //------------------------------------------------------------------------- - // The main group of arguments - - impPopCallArgs(sig, call->AsCall()); - if (extraArg.Node != nullptr) - { - if (Target::g_tgtArgOrder == Target::ARG_ORDER_R2L) - { - call->AsCall()->gtArgs.PushFront(this, extraArg); - } - else - { - call->AsCall()->gtArgs.PushBack(this, extraArg); - } - - call->gtFlags |= extraArg.Node->gtFlags & GTF_GLOB_EFFECT; - } - - //------------------------------------------------------------------------- - // The "this" pointer - - if (((mflags & CORINFO_FLG_STATIC) == 0) && ((sig->callConv & CORINFO_CALLCONV_EXPLICITTHIS) == 0) && - !((opcode == CEE_NEWOBJ) && (newobjThis == nullptr))) - { - GenTree* obj; - - if (opcode == CEE_NEWOBJ) - { - obj = newobjThis; - } - else - { - obj = impPopStack().val; - obj = impTransformThis(obj, pConstrainedResolvedToken, constraintCallThisTransform); - if (compDonotInline()) - { - return TYP_UNDEF; - } - } - - // Store the "this" value in the call - call->gtFlags |= obj->gtFlags & GTF_GLOB_EFFECT; - call->AsCall()->gtArgs.PushFront(this, NewCallArg::Primitive(obj).WellKnown(WellKnownArg::ThisPointer)); - - if (impIsThis(obj)) - { - call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_NONVIRT_SAME_THIS; - } - } - - bool probing; - probing = impConsiderCallProbe(call->AsCall(), rawILOffset); - - // See if we can devirt if we aren't probing. - if (!probing && opts.OptimizationEnabled()) - { - if (call->AsCall()->IsVirtual()) - { - // only true object pointers can be virtual - assert(call->AsCall()->gtArgs.HasThisPointer() && - call->AsCall()->gtArgs.GetThisArg()->GetNode()->TypeIs(TYP_REF)); - - // See if we can devirtualize. - - const bool isExplicitTailCall = (tailCallFlags & PREFIX_TAILCALL_EXPLICIT) != 0; - const bool isLateDevirtualization = false; - impDevirtualizeCall(call->AsCall(), pResolvedToken, &callInfo->hMethod, &callInfo->methodFlags, - &callInfo->contextHandle, &exactContextHnd, isLateDevirtualization, isExplicitTailCall, - // Take care to pass raw IL offset here as the 'debug info' might be different for - // inlinees. - rawILOffset); - - // Devirtualization may change which method gets invoked. Update our local cache. - // - methHnd = callInfo->hMethod; - } - else if (call->AsCall()->IsDelegateInvoke()) - { - considerGuardedDevirtualization(call->AsCall(), rawILOffset, false, NO_METHOD_HANDLE, NO_CLASS_HANDLE, - nullptr); - } - } - - //------------------------------------------------------------------------- - // The "this" pointer for "newobj" - - if (opcode == CEE_NEWOBJ) - { - if (clsFlags & CORINFO_FLG_VAROBJSIZE) - { - assert(!(clsFlags & CORINFO_FLG_ARRAY)); // arrays handled separately - // This is a 'new' of a variable sized object, wher - // the constructor is to return the object. In this case - // the constructor claims to return VOID but we know it - // actually returns the new object - assert(callRetTyp == TYP_VOID); - callRetTyp = TYP_REF; - call->gtType = TYP_REF; - impSpillSpecialSideEff(); - - impPushOnStack(call, typeInfo(TI_REF, clsHnd)); - } - else - { - if (clsFlags & CORINFO_FLG_DELEGATE) - { - // New inliner morph it in impImportCall. - // This will allow us to inline the call to the delegate constructor. - call = fgOptimizeDelegateConstructor(call->AsCall(), &exactContextHnd, ldftnInfo); - } - - if (!bIntrinsicImported) - { - -#if defined(DEBUG) || defined(INLINE_DATA) - - // Keep track of the raw IL offset of the call - call->AsCall()->gtRawILOffset = rawILOffset; - -#endif // defined(DEBUG) || defined(INLINE_DATA) - - // Is it an inline candidate? - impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, rawILOffset); - } - - // append the call node. - impAppendTree(call, (unsigned)CHECK_SPILL_ALL, impCurStmtDI); - - // Now push the value of the 'new onto the stack - - // This is a 'new' of a non-variable sized object. - // Append the new node (op1) to the statement list, - // and then push the local holding the value of this - // new instruction on the stack. - - if (clsFlags & CORINFO_FLG_VALUECLASS) - { - assert(newobjThis->gtOper == GT_ADDR && newobjThis->AsOp()->gtOp1->gtOper == GT_LCL_VAR); - - unsigned tmp = newobjThis->AsOp()->gtOp1->AsLclVarCommon()->GetLclNum(); - impPushOnStack(gtNewLclvNode(tmp, lvaGetRealType(tmp)), verMakeTypeInfo(clsHnd).NormaliseForStack()); - } - else - { - if (newobjThis->gtOper == GT_COMMA) - { - // We must have inserted the callout. Get the real newobj. - newobjThis = newobjThis->AsOp()->gtOp2; - } - - assert(newobjThis->gtOper == GT_LCL_VAR); - impPushOnStack(gtNewLclvNode(newobjThis->AsLclVarCommon()->GetLclNum(), TYP_REF), - typeInfo(TI_REF, clsHnd)); - } - } - return callRetTyp; - } - -DONE: - -#ifdef DEBUG - // In debug we want to be able to register callsites with the EE. - assert(call->AsCall()->callSig == nullptr); - call->AsCall()->callSig = new (this, CMK_Generic) CORINFO_SIG_INFO; - *call->AsCall()->callSig = *sig; -#endif - - // Final importer checks for calls flagged as tail calls. - // - if (tailCallFlags != 0) - { - const bool isExplicitTailCall = (tailCallFlags & PREFIX_TAILCALL_EXPLICIT) != 0; - const bool isImplicitTailCall = (tailCallFlags & PREFIX_TAILCALL_IMPLICIT) != 0; - const bool isStressTailCall = (tailCallFlags & PREFIX_TAILCALL_STRESS) != 0; - - // Exactly one of these should be true. - assert(isExplicitTailCall != isImplicitTailCall); - - // This check cannot be performed for implicit tail calls for the reason - // that impIsImplicitTailCallCandidate() is not checking whether return - // types are compatible before marking a call node with PREFIX_TAILCALL_IMPLICIT. - // As a result it is possible that in the following case, we find that - // the type stack is non-empty if Callee() is considered for implicit - // tail calling. - // int Caller(..) { .... void Callee(); ret val; ... } - // - // Note that we cannot check return type compatibility before ImpImportCall() - // as we don't have required info or need to duplicate some of the logic of - // ImpImportCall(). - // - // For implicit tail calls, we perform this check after return types are - // known to be compatible. - if (isExplicitTailCall && (verCurrentState.esStackDepth != 0)) - { - BADCODE("Stack should be empty after tailcall"); - } - - // For opportunistic tailcalls we allow implicit widening, i.e. tailcalls from int32 -> int16, since the - // managed calling convention dictates that the callee widens the value. For explicit tailcalls we don't - // want to require this detail of the calling convention to bubble up to the tailcall helpers - bool allowWidening = isImplicitTailCall; - if (canTailCall && - !impTailCallRetTypeCompatible(allowWidening, info.compRetType, info.compMethodInfo->args.retTypeClass, - info.compCallConv, callRetTyp, sig->retTypeClass, - call->AsCall()->GetUnmanagedCallConv())) - { - canTailCall = false; - szCanTailCallFailReason = "Return types are not tail call compatible"; - } - - // Stack empty check for implicit tail calls. - if (canTailCall && isImplicitTailCall && (verCurrentState.esStackDepth != 0)) - { -#ifdef TARGET_AMD64 - // JIT64 Compatibility: Opportunistic tail call stack mismatch throws a VerificationException - // in JIT64, not an InvalidProgramException. - Verify(false, "Stack should be empty after tailcall"); -#else // TARGET_64BIT - BADCODE("Stack should be empty after tailcall"); -#endif //! TARGET_64BIT - } - - // assert(compCurBB is not a catch, finally or filter block); - // assert(compCurBB is not a try block protected by a finally block); - assert(!isExplicitTailCall || compCurBB->bbJumpKind == BBJ_RETURN); - - // Ask VM for permission to tailcall - if (canTailCall) - { - // True virtual or indirect calls, shouldn't pass in a callee handle. - CORINFO_METHOD_HANDLE exactCalleeHnd = - ((call->AsCall()->gtCallType != CT_USER_FUNC) || call->AsCall()->IsVirtual()) ? nullptr : methHnd; - - if (info.compCompHnd->canTailCall(info.compMethodHnd, methHnd, exactCalleeHnd, isExplicitTailCall)) - { - if (isExplicitTailCall) - { - // In case of explicit tail calls, mark it so that it is not considered - // for in-lining. - call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_EXPLICIT_TAILCALL; - JITDUMP("\nGTF_CALL_M_EXPLICIT_TAILCALL set for call [%06u]\n", dspTreeID(call)); - - if (isStressTailCall) - { - call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_STRESS_TAILCALL; - JITDUMP("\nGTF_CALL_M_STRESS_TAILCALL set for call [%06u]\n", dspTreeID(call)); - } - } - else - { -#if FEATURE_TAILCALL_OPT - // Must be an implicit tail call. - assert(isImplicitTailCall); - - // It is possible that a call node is both an inline candidate and marked - // for opportunistic tail calling. In-lining happens before morhphing of - // trees. If in-lining of an in-line candidate gets aborted for whatever - // reason, it will survive to the morphing stage at which point it will be - // transformed into a tail call after performing additional checks. - - call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_IMPLICIT_TAILCALL; - JITDUMP("\nGTF_CALL_M_IMPLICIT_TAILCALL set for call [%06u]\n", dspTreeID(call)); - -#else //! FEATURE_TAILCALL_OPT - NYI("Implicit tail call prefix on a target which doesn't support opportunistic tail calls"); - -#endif // FEATURE_TAILCALL_OPT - } - - // This might or might not turn into a tailcall. We do more - // checks in morph. For explicit tailcalls we need more - // information in morph in case it turns out to be a - // helper-based tailcall. - if (isExplicitTailCall) - { - assert(call->AsCall()->tailCallInfo == nullptr); - call->AsCall()->tailCallInfo = new (this, CMK_CorTailCallInfo) TailCallSiteInfo; - switch (opcode) - { - case CEE_CALLI: - call->AsCall()->tailCallInfo->SetCalli(sig); - break; - case CEE_CALLVIRT: - call->AsCall()->tailCallInfo->SetCallvirt(sig, pResolvedToken); - break; - default: - call->AsCall()->tailCallInfo->SetCall(sig, pResolvedToken); - break; - } - } - } - else - { - // canTailCall reported its reasons already - canTailCall = false; - JITDUMP("\ninfo.compCompHnd->canTailCall returned false for call [%06u]\n", dspTreeID(call)); - } - } - else - { - // If this assert fires it means that canTailCall was set to false without setting a reason! - assert(szCanTailCallFailReason != nullptr); - JITDUMP("\nRejecting %splicit tail call for [%06u], reason: '%s'\n", isExplicitTailCall ? "ex" : "im", - dspTreeID(call), szCanTailCallFailReason); - info.compCompHnd->reportTailCallDecision(info.compMethodHnd, methHnd, isExplicitTailCall, TAILCALL_FAIL, - szCanTailCallFailReason); - } - } - - // Note: we assume that small return types are already normalized by the managed callee - // or by the pinvoke stub for calls to unmanaged code. - - if (!bIntrinsicImported) - { - // - // Things needed to be checked when bIntrinsicImported is false. - // - - assert(call->gtOper == GT_CALL); - assert(callInfo != nullptr); - - if (compIsForInlining() && opcode == CEE_CALLVIRT) - { - assert(call->AsCall()->gtArgs.HasThisPointer()); - GenTree* callObj = call->AsCall()->gtArgs.GetThisArg()->GetEarlyNode(); - - if ((call->AsCall()->IsVirtual() || (call->gtFlags & GTF_CALL_NULLCHECK)) && - impInlineIsGuaranteedThisDerefBeforeAnySideEffects(nullptr, &call->AsCall()->gtArgs, callObj, - impInlineInfo->inlArgInfo)) - { - impInlineInfo->thisDereferencedFirst = true; - } - } - -#if defined(DEBUG) || defined(INLINE_DATA) - - // Keep track of the raw IL offset of the call - call->AsCall()->gtRawILOffset = rawILOffset; - -#endif // defined(DEBUG) || defined(INLINE_DATA) - - // Is it an inline candidate? - impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, rawILOffset); - } - - // Extra checks for tail calls and tail recursion. - // - // A tail recursive call is a potential loop from the current block to the start of the root method. - // If we see a tail recursive call, mark the blocks from the call site back to the entry as potentially - // being in a loop. - // - // Note: if we're importing an inlinee we don't mark the right set of blocks, but by then it's too - // late. Currently this doesn't lead to problems. See GitHub issue 33529. - // - // OSR also needs to handle tail calls specially: - // * block profiling in OSR methods needs to ensure probes happen before tail calls, not after. - // * the root method entry must be imported if there's a recursive tail call or a potentially - // inlineable tail call. - // - if ((tailCallFlags != 0) && canTailCall) - { - if (gtIsRecursiveCall(methHnd)) - { - assert(verCurrentState.esStackDepth == 0); - BasicBlock* loopHead = nullptr; - if (!compIsForInlining() && opts.IsOSR()) - { - // For root method OSR we may branch back to the actual method entry, - // which is not fgFirstBB, and which we will need to import. - assert(fgEntryBB != nullptr); - loopHead = fgEntryBB; - } - else - { - // For normal jitting we may branch back to the firstBB; this - // should already be imported. - loopHead = fgFirstBB; - } - - JITDUMP("\nTail recursive call [%06u] in the method. Mark " FMT_BB " to " FMT_BB - " as having a backward branch.\n", - dspTreeID(call), loopHead->bbNum, compCurBB->bbNum); - fgMarkBackwardJump(loopHead, compCurBB); - } - - // We only do these OSR checks in the root method because: - // * If we fail to import the root method entry when importing the root method, we can't go back - // and import it during inlining. So instead of checking jsut for recursive tail calls we also - // have to check for anything that might introduce a recursive tail call. - // * We only instrument root method blocks in OSR methods, - // - if (opts.IsOSR() && !compIsForInlining()) - { - // If a root method tail call candidate block is not a BBJ_RETURN, it should have a unique - // BBJ_RETURN successor. Mark that successor so we can handle it specially during profile - // instrumentation. - // - if (compCurBB->bbJumpKind != BBJ_RETURN) - { - BasicBlock* const successor = compCurBB->GetUniqueSucc(); - assert(successor->bbJumpKind == BBJ_RETURN); - successor->bbFlags |= BBF_TAILCALL_SUCCESSOR; - optMethodFlags |= OMF_HAS_TAILCALL_SUCCESSOR; - } - - // If this call might eventually turn into a loop back to method entry, make sure we - // import the method entry. - // - assert(call->IsCall()); - GenTreeCall* const actualCall = call->AsCall(); - const bool mustImportEntryBlock = gtIsRecursiveCall(methHnd) || actualCall->IsInlineCandidate() || - actualCall->IsGuardedDevirtualizationCandidate(); - - // Only schedule importation if we're not currently importing. - // - if (mustImportEntryBlock && (compCurBB != fgEntryBB)) - { - JITDUMP("\nOSR: inlineable or recursive tail call [%06u] in the method, so scheduling " FMT_BB - " for importation\n", - dspTreeID(call), fgEntryBB->bbNum); - impImportBlockPending(fgEntryBB); - } - } - } - - if ((sig->flags & CORINFO_SIGFLAG_FAT_CALL) != 0) - { - assert(opcode == CEE_CALLI || callInfo->kind == CORINFO_CALL_CODE_POINTER); - addFatPointerCandidate(call->AsCall()); - } - -DONE_CALL: - // Push or append the result of the call - if (callRetTyp == TYP_VOID) - { - if (opcode == CEE_NEWOBJ) - { - // we actually did push something, so don't spill the thing we just pushed. - assert(verCurrentState.esStackDepth > 0); - impAppendTree(call, verCurrentState.esStackDepth - 1, impCurStmtDI); - } - else - { - impAppendTree(call, (unsigned)CHECK_SPILL_ALL, impCurStmtDI); - } - } - else - { - impSpillSpecialSideEff(); - - if (clsFlags & CORINFO_FLG_ARRAY) - { - eeGetCallSiteSig(pResolvedToken->token, pResolvedToken->tokenScope, pResolvedToken->tokenContext, sig); - } - - typeInfo tiRetVal = verMakeTypeInfo(sig->retType, sig->retTypeClass); - tiRetVal.NormaliseForStack(); - - // The CEE_READONLY prefix modifies the verification semantics of an Address - // operation on an array type. - if ((clsFlags & CORINFO_FLG_ARRAY) && isReadonlyCall && tiRetVal.IsByRef()) - { - tiRetVal.SetIsReadonlyByRef(); - } - - if (call->IsCall()) - { - // Sometimes "call" is not a GT_CALL (if we imported an intrinsic that didn't turn into a call) - - GenTreeCall* origCall = call->AsCall(); - - const bool isFatPointerCandidate = origCall->IsFatPointerCandidate(); - const bool isInlineCandidate = origCall->IsInlineCandidate(); - const bool isGuardedDevirtualizationCandidate = origCall->IsGuardedDevirtualizationCandidate(); - - if (varTypeIsStruct(callRetTyp)) - { - // Need to treat all "split tree" cases here, not just inline candidates - call = impFixupCallStructReturn(call->AsCall(), sig->retTypeClass); - } - - // TODO: consider handling fatcalli cases this way too...? - if (isInlineCandidate || isGuardedDevirtualizationCandidate) - { - // We should not have made any adjustments in impFixupCallStructReturn - // as we defer those until we know the fate of the call. - assert(call == origCall); - - assert(opts.OptEnabled(CLFLG_INLINING)); - assert(!isFatPointerCandidate); // We should not try to inline calli. - - // Make the call its own tree (spill the stack if needed). - // Do not consume the debug info here. This is particularly - // important if we give up on the inline, in which case the - // call will typically end up in the statement that contains - // the GT_RET_EXPR that we leave on the stack. - impAppendTree(call, (unsigned)CHECK_SPILL_ALL, impCurStmtDI, false); - - // TODO: Still using the widened type. - GenTree* retExpr = gtNewInlineCandidateReturnExpr(call, genActualType(callRetTyp), compCurBB->bbFlags); - - // Link the retExpr to the call so if necessary we can manipulate it later. - origCall->gtInlineCandidateInfo->retExpr = retExpr; - - // Propagate retExpr as the placeholder for the call. - call = retExpr; - } - else - { - // If the call is virtual, and has a generics context, and is not going to have a class probe, - // record the context for possible use during late devirt. - // - // If we ever want to devirt at Tier0, and/or see issues where OSR methods under PGO lose - // important devirtualizations, we'll want to allow both a class probe and a captured context. - // - if (origCall->IsVirtual() && (origCall->gtCallType != CT_INDIRECT) && (exactContextHnd != nullptr) && - (origCall->gtHandleHistogramProfileCandidateInfo == nullptr)) - { - JITDUMP("\nSaving context %p for call [%06u]\n", exactContextHnd, dspTreeID(origCall)); - origCall->gtCallMoreFlags |= GTF_CALL_M_HAS_LATE_DEVIRT_INFO; - LateDevirtualizationInfo* const info = new (this, CMK_Inlining) LateDevirtualizationInfo; - info->exactContextHnd = exactContextHnd; - origCall->gtLateDevirtualizationInfo = info; - } - - if (isFatPointerCandidate) - { - // fatPointer candidates should be in statements of the form call() or var = call(). - // Such form allows to find statements with fat calls without walking through whole trees - // and removes problems with cutting trees. - assert(!bIntrinsicImported); - assert(IsTargetAbi(CORINFO_NATIVEAOT_ABI)); - if (call->OperGet() != GT_LCL_VAR) // can be already converted by impFixupCallStructReturn. - { - unsigned calliSlot = lvaGrabTemp(true DEBUGARG("calli")); - LclVarDsc* varDsc = lvaGetDesc(calliSlot); - varDsc->lvVerTypeInfo = tiRetVal; - impAssignTempGen(calliSlot, call, tiRetVal.GetClassHandle(), (unsigned)CHECK_SPILL_NONE); - // impAssignTempGen can change src arg list and return type for call that returns struct. - var_types type = genActualType(lvaTable[calliSlot].TypeGet()); - call = gtNewLclvNode(calliSlot, type); - } - } - - // For non-candidates we must also spill, since we - // might have locals live on the eval stack that this - // call can modify. - // - // Suppress this for certain well-known call targets - // that we know won't modify locals, eg calls that are - // recognized in gtCanOptimizeTypeEquality. Otherwise - // we may break key fragile pattern matches later on. - bool spillStack = true; - if (call->IsCall()) - { - GenTreeCall* callNode = call->AsCall(); - if ((callNode->gtCallType == CT_HELPER) && (gtIsTypeHandleToRuntimeTypeHelper(callNode) || - gtIsTypeHandleToRuntimeTypeHandleHelper(callNode))) - { - spillStack = false; - } - else if ((callNode->gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC) != 0) - { - spillStack = false; - } - } - - if (spillStack) - { - impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("non-inline candidate call")); - } - } - } - - if (!bIntrinsicImported) - { - //------------------------------------------------------------------------- - // - /* If the call is of a small type and the callee is managed, the callee will normalize the result - before returning. - However, we need to normalize small type values returned by unmanaged - functions (pinvoke). The pinvoke stub does the normalization, but we need to do it here - if we use the shorter inlined pinvoke stub. */ - - if (checkForSmallType && varTypeIsIntegral(callRetTyp) && genTypeSize(callRetTyp) < genTypeSize(TYP_INT)) - { - call = gtNewCastNode(genActualType(callRetTyp), call, false, callRetTyp); - } - } - - impPushOnStack(call, tiRetVal); - } - - // VSD functions get a new call target each time we getCallInfo, so clear the cache. - // Also, the call info cache for CALLI instructions is largely incomplete, so clear it out. - // if ( (opcode == CEE_CALLI) || (callInfoCache.fetchCallInfo().kind == CORINFO_VIRTUALCALL_STUB)) - // callInfoCache.uncacheCallInfo(); - - return callRetTyp; -} -#ifdef _PREFAST_ -#pragma warning(pop) -#endif - -bool Compiler::impMethodInfo_hasRetBuffArg(CORINFO_METHOD_INFO* methInfo, CorInfoCallConvExtension callConv) -{ - CorInfoType corType = methInfo->args.retType; - - if ((corType == CORINFO_TYPE_VALUECLASS) || (corType == CORINFO_TYPE_REFANY)) - { - // We have some kind of STRUCT being returned - structPassingKind howToReturnStruct = SPK_Unknown; - - var_types returnType = getReturnTypeForStruct(methInfo->args.retTypeClass, callConv, &howToReturnStruct); - - if (howToReturnStruct == SPK_ByReference) - { - return true; - } - } - - return false; -} - -#ifdef DEBUG -// -var_types Compiler::impImportJitTestLabelMark(int numArgs) -{ - TestLabelAndNum tlAndN; - if (numArgs == 2) - { - tlAndN.m_num = 0; - StackEntry se = impPopStack(); - assert(se.seTypeInfo.GetType() == TI_INT); - GenTree* val = se.val; - assert(val->IsCnsIntOrI()); - tlAndN.m_tl = (TestLabel)val->AsIntConCommon()->IconValue(); - } - else if (numArgs == 3) - { - StackEntry se = impPopStack(); - assert(se.seTypeInfo.GetType() == TI_INT); - GenTree* val = se.val; - assert(val->IsCnsIntOrI()); - tlAndN.m_num = val->AsIntConCommon()->IconValue(); - se = impPopStack(); - assert(se.seTypeInfo.GetType() == TI_INT); - val = se.val; - assert(val->IsCnsIntOrI()); - tlAndN.m_tl = (TestLabel)val->AsIntConCommon()->IconValue(); - } - else - { - assert(false); - } - - StackEntry expSe = impPopStack(); - GenTree* node = expSe.val; - - // There are a small number of special cases, where we actually put the annotation on a subnode. - if (tlAndN.m_tl == TL_LoopHoist && tlAndN.m_num >= 100) - { - // A loop hoist annotation with value >= 100 means that the expression should be a static field access, - // a GT_IND of a static field address, which should be the sum of a (hoistable) helper call and possibly some - // offset within the static field block whose address is returned by the helper call. - // The annotation is saying that this address calculation, but not the entire access, should be hoisted. - assert(node->OperGet() == GT_IND); - tlAndN.m_num -= 100; - GetNodeTestData()->Set(node->AsOp()->gtOp1, tlAndN); - GetNodeTestData()->Remove(node); - } - else - { - GetNodeTestData()->Set(node, tlAndN); - } - - impPushOnStack(node, expSe.seTypeInfo); - return node->TypeGet(); -} -#endif // DEBUG - -//----------------------------------------------------------------------------------- -// impFixupCallStructReturn: For a call node that returns a struct do one of the following: -// - set the flag to indicate struct return via retbuf arg; -// - adjust the return type to a SIMD type if it is returned in 1 reg; -// - spill call result into a temp if it is returned into 2 registers or more and not tail call or inline candidate. -// -// Arguments: -// call - GT_CALL GenTree node -// retClsHnd - Class handle of return type of the call -// -// Return Value: -// Returns new GenTree node after fixing struct return of call node -// -GenTree* Compiler::impFixupCallStructReturn(GenTreeCall* call, CORINFO_CLASS_HANDLE retClsHnd) -{ - if (!varTypeIsStruct(call)) - { - return call; - } - - call->gtRetClsHnd = retClsHnd; - -#if FEATURE_MULTIREG_RET - call->InitializeStructReturnType(this, retClsHnd, call->GetUnmanagedCallConv()); - const ReturnTypeDesc* retTypeDesc = call->GetReturnTypeDesc(); - const unsigned retRegCount = retTypeDesc->GetReturnRegCount(); -#else // !FEATURE_MULTIREG_RET - const unsigned retRegCount = 1; -#endif // !FEATURE_MULTIREG_RET - - structPassingKind howToReturnStruct; - var_types returnType = getReturnTypeForStruct(retClsHnd, call->GetUnmanagedCallConv(), &howToReturnStruct); - - if (howToReturnStruct == SPK_ByReference) - { - assert(returnType == TYP_UNKNOWN); - call->gtCallMoreFlags |= GTF_CALL_M_RETBUFFARG; - return call; - } - - // Recognize SIMD types as we do for LCL_VARs, - // note it could be not the ABI specific type, for example, on x64 we can set 'TYP_SIMD8` - // for `System.Numerics.Vector2` here but lower will change it to long as ABI dictates. - var_types simdReturnType = impNormStructType(call->gtRetClsHnd); - if (simdReturnType != call->TypeGet()) - { - assert(varTypeIsSIMD(simdReturnType)); - JITDUMP("changing the type of a call [%06u] from %s to %s\n", dspTreeID(call), varTypeName(call->TypeGet()), - varTypeName(simdReturnType)); - call->ChangeType(simdReturnType); - } - - if (retRegCount == 1) - { - return call; - } - -#if FEATURE_MULTIREG_RET - assert(varTypeIsStruct(call)); // It could be a SIMD returned in several regs. - assert(returnType == TYP_STRUCT); - assert((howToReturnStruct == SPK_ByValueAsHfa) || (howToReturnStruct == SPK_ByValue)); - -#ifdef UNIX_AMD64_ABI - // must be a struct returned in two registers - assert(retRegCount == 2); -#else // not UNIX_AMD64_ABI - assert(retRegCount >= 2); -#endif // not UNIX_AMD64_ABI - - if (!call->CanTailCall() && !call->IsInlineCandidate()) - { - // Force a call returning multi-reg struct to be always of the IR form - // tmp = call - // - // No need to assign a multi-reg struct to a local var if: - // - It is a tail call or - // - The call is marked for in-lining later - return impAssignMultiRegTypeToVar(call, retClsHnd DEBUGARG(call->GetUnmanagedCallConv())); - } - return call; -#endif // FEATURE_MULTIREG_RET -} - -/***************************************************************************** - For struct return values, re-type the operand in the case where the ABI - does not use a struct return buffer - */ - -//------------------------------------------------------------------------ -// impFixupStructReturnType: For struct return values it sets appropriate flags in MULTIREG returns case; -// in non-multiref case it handles two special helpers: `CORINFO_HELP_GETFIELDSTRUCT`, `CORINFO_HELP_UNBOX_NULLABLE`. -// -// Arguments: -// op - the return value; -// retClsHnd - the struct handle; -// unmgdCallConv - the calling convention of the function that returns this struct. -// -// Return Value: -// the result tree that does the return. -// -GenTree* Compiler::impFixupStructReturnType(GenTree* op, - CORINFO_CLASS_HANDLE retClsHnd, - CorInfoCallConvExtension unmgdCallConv) -{ - assert(varTypeIsStruct(info.compRetType)); - assert(info.compRetBuffArg == BAD_VAR_NUM); - - JITDUMP("\nimpFixupStructReturnType: retyping\n"); - DISPTREE(op); - -#if defined(TARGET_XARCH) - -#if FEATURE_MULTIREG_RET - // No VarArgs for CoreCLR on x64 Unix - UNIX_AMD64_ABI_ONLY(assert(!info.compIsVarArgs)); - - // Is method returning a multi-reg struct? - if (varTypeIsStruct(info.compRetNativeType) && IsMultiRegReturnedType(retClsHnd, unmgdCallConv)) - { - // In case of multi-reg struct return, we force IR to be one of the following: - // GT_RETURN(lclvar) or GT_RETURN(call). If op is anything other than a - // lclvar or call, it is assigned to a temp to create: temp = op and GT_RETURN(tmp). - - if (op->gtOper == GT_LCL_VAR) - { - // Note that this is a multi-reg return. - unsigned lclNum = op->AsLclVarCommon()->GetLclNum(); - lvaTable[lclNum].lvIsMultiRegRet = true; - - // TODO-1stClassStructs: Handle constant propagation and CSE-ing of multireg returns. - op->gtFlags |= GTF_DONT_CSE; - - return op; - } - - if (op->IsCall() && (op->AsCall()->GetUnmanagedCallConv() == unmgdCallConv)) - { - return op; - } - - return impAssignMultiRegTypeToVar(op, retClsHnd DEBUGARG(unmgdCallConv)); - } -#else - assert(info.compRetNativeType != TYP_STRUCT); -#endif // defined(UNIX_AMD64_ABI) || defined(TARGET_X86) - -#elif FEATURE_MULTIREG_RET && defined(TARGET_ARM) - - if (varTypeIsStruct(info.compRetNativeType) && !info.compIsVarArgs && IsHfa(retClsHnd)) - { - if (op->gtOper == GT_LCL_VAR) - { - // This LCL_VAR is an HFA return value, it stays as a TYP_STRUCT - unsigned lclNum = op->AsLclVarCommon()->GetLclNum(); - // Make sure this struct type stays as struct so that we can return it as an HFA - lvaTable[lclNum].lvIsMultiRegRet = true; - - // TODO-1stClassStructs: Handle constant propagation and CSE-ing of multireg returns. - op->gtFlags |= GTF_DONT_CSE; - - return op; - } - - if (op->gtOper == GT_CALL) - { - if (op->AsCall()->IsVarargs()) - { - // We cannot tail call because control needs to return to fixup the calling - // convention for result return. - op->AsCall()->gtCallMoreFlags &= ~GTF_CALL_M_TAILCALL; - op->AsCall()->gtCallMoreFlags &= ~GTF_CALL_M_EXPLICIT_TAILCALL; - } - else - { - return op; - } - } - return impAssignMultiRegTypeToVar(op, retClsHnd DEBUGARG(unmgdCallConv)); - } - -#elif FEATURE_MULTIREG_RET && (defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64)) - - // Is method returning a multi-reg struct? - if (IsMultiRegReturnedType(retClsHnd, unmgdCallConv)) - { - if (op->gtOper == GT_LCL_VAR) - { - // This LCL_VAR stays as a TYP_STRUCT - unsigned lclNum = op->AsLclVarCommon()->GetLclNum(); - - if (!lvaIsImplicitByRefLocal(lclNum)) - { - // Make sure this struct type is not struct promoted - lvaTable[lclNum].lvIsMultiRegRet = true; - - // TODO-1stClassStructs: Handle constant propagation and CSE-ing of multireg returns. - op->gtFlags |= GTF_DONT_CSE; - - return op; - } - } - - if (op->gtOper == GT_CALL) - { - if (op->AsCall()->IsVarargs()) - { - // We cannot tail call because control needs to return to fixup the calling - // convention for result return. - op->AsCall()->gtCallMoreFlags &= ~GTF_CALL_M_TAILCALL; - op->AsCall()->gtCallMoreFlags &= ~GTF_CALL_M_EXPLICIT_TAILCALL; - } - else - { - return op; - } - } - return impAssignMultiRegTypeToVar(op, retClsHnd DEBUGARG(unmgdCallConv)); - } - -#endif // FEATURE_MULTIREG_RET && (TARGET_ARM64 || TARGET_LOONGARCH64) - - if (!op->IsCall() || !op->AsCall()->TreatAsShouldHaveRetBufArg(this)) - { - // Don't retype `struct` as a primitive type in `ret` instruction. - return op; - } - - // This must be one of those 'special' helpers that don't - // really have a return buffer, but instead use it as a way - // to keep the trees cleaner with fewer address-taken temps. - // - // Well now we have to materialize the return buffer as - // an address-taken temp. Then we can return the temp. - // - // NOTE: this code assumes that since the call directly - // feeds the return, then the call must be returning the - // same structure/class/type. - // - unsigned tmpNum = lvaGrabTemp(true DEBUGARG("pseudo return buffer")); - - // No need to spill anything as we're about to return. - impAssignTempGen(tmpNum, op, info.compMethodInfo->args.retTypeClass, (unsigned)CHECK_SPILL_NONE); - - op = gtNewLclvNode(tmpNum, info.compRetType); - JITDUMP("\nimpFixupStructReturnType: created a pseudo-return buffer for a special helper\n"); - DISPTREE(op); - return op; -} - -/***************************************************************************** - CEE_LEAVE may be jumping out of a protected block, viz, a catch or a - finally-protected try. We find the finally blocks protecting the current - offset (in order) by walking over the complete exception table and - finding enclosing clauses. This assumes that the table is sorted. - This will create a series of BBJ_CALLFINALLY -> BBJ_CALLFINALLY ... -> BBJ_ALWAYS. - - If we are leaving a catch handler, we need to attach the - CPX_ENDCATCHes to the correct BBJ_CALLFINALLY blocks. - - After this function, the BBJ_LEAVE block has been converted to a different type. - */ - -#if !defined(FEATURE_EH_FUNCLETS) - -void Compiler::impImportLeave(BasicBlock* block) -{ -#ifdef DEBUG - if (verbose) - { - printf("\nBefore import CEE_LEAVE:\n"); - fgDispBasicBlocks(); - fgDispHandlerTab(); - } -#endif // DEBUG - - bool invalidatePreds = false; // If we create new blocks, invalidate the predecessor lists (if created) - unsigned blkAddr = block->bbCodeOffs; - BasicBlock* leaveTarget = block->bbJumpDest; - unsigned jmpAddr = leaveTarget->bbCodeOffs; - - // LEAVE clears the stack, spill side effects, and set stack to 0 - - impSpillSideEffects(true, (unsigned)CHECK_SPILL_ALL DEBUGARG("impImportLeave")); - verCurrentState.esStackDepth = 0; - - assert(block->bbJumpKind == BBJ_LEAVE); - assert(fgBBs == (BasicBlock**)0xCDCD || fgLookupBB(jmpAddr) != NULL); // should be a BB boundary - - BasicBlock* step = DUMMY_INIT(NULL); - unsigned encFinallies = 0; // Number of enclosing finallies. - GenTree* endCatches = NULL; - Statement* endLFinStmt = NULL; // The statement tree to indicate the end of locally-invoked finally. - - unsigned XTnum; - EHblkDsc* HBtab; - - for (XTnum = 0, HBtab = compHndBBtab; XTnum < compHndBBtabCount; XTnum++, HBtab++) - { - // Grab the handler offsets - - IL_OFFSET tryBeg = HBtab->ebdTryBegOffs(); - IL_OFFSET tryEnd = HBtab->ebdTryEndOffs(); - IL_OFFSET hndBeg = HBtab->ebdHndBegOffs(); - IL_OFFSET hndEnd = HBtab->ebdHndEndOffs(); - - /* Is this a catch-handler we are CEE_LEAVEing out of? - * If so, we need to call CORINFO_HELP_ENDCATCH. - */ - - if (jitIsBetween(blkAddr, hndBeg, hndEnd) && !jitIsBetween(jmpAddr, hndBeg, hndEnd)) - { - // Can't CEE_LEAVE out of a finally/fault handler - if (HBtab->HasFinallyOrFaultHandler()) - BADCODE("leave out of fault/finally block"); - - // Create the call to CORINFO_HELP_ENDCATCH - GenTree* endCatch = gtNewHelperCallNode(CORINFO_HELP_ENDCATCH, TYP_VOID); - - // Make a list of all the currently pending endCatches - if (endCatches) - endCatches = gtNewOperNode(GT_COMMA, TYP_VOID, endCatches, endCatch); - else - endCatches = endCatch; - -#ifdef DEBUG - if (verbose) - { - printf("impImportLeave - " FMT_BB " jumping out of catch handler EH#%u, adding call to " - "CORINFO_HELP_ENDCATCH\n", - block->bbNum, XTnum); - } -#endif - } - else if (HBtab->HasFinallyHandler() && jitIsBetween(blkAddr, tryBeg, tryEnd) && - !jitIsBetween(jmpAddr, tryBeg, tryEnd)) - { - /* This is a finally-protected try we are jumping out of */ - - /* If there are any pending endCatches, and we have already - jumped out of a finally-protected try, then the endCatches - have to be put in a block in an outer try for async - exceptions to work correctly. - Else, just use append to the original block */ - - BasicBlock* callBlock; - - assert(!encFinallies == - !endLFinStmt); // if we have finallies, we better have an endLFin tree, and vice-versa - - if (encFinallies == 0) - { - assert(step == DUMMY_INIT(NULL)); - callBlock = block; - callBlock->bbJumpKind = BBJ_CALLFINALLY; // convert the BBJ_LEAVE to BBJ_CALLFINALLY - - if (endCatches) - impAppendTree(endCatches, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); - -#ifdef DEBUG - if (verbose) - { - printf("impImportLeave - jumping out of a finally-protected try, convert block to BBJ_CALLFINALLY " - "block %s\n", - callBlock->dspToString()); - } -#endif - } - else - { - assert(step != DUMMY_INIT(NULL)); - - /* Calling the finally block */ - callBlock = fgNewBBinRegion(BBJ_CALLFINALLY, XTnum + 1, 0, step); - assert(step->bbJumpKind == BBJ_ALWAYS); - step->bbJumpDest = callBlock; // the previous call to a finally returns to this call (to the next - // finally in the chain) - step->bbJumpDest->bbRefs++; - - /* The new block will inherit this block's weight */ - callBlock->inheritWeight(block); - -#ifdef DEBUG - if (verbose) - { - printf("impImportLeave - jumping out of a finally-protected try, new BBJ_CALLFINALLY block %s\n", - callBlock->dspToString()); - } -#endif - - Statement* lastStmt; - - if (endCatches) - { - lastStmt = gtNewStmt(endCatches); - endLFinStmt->SetNextStmt(lastStmt); - lastStmt->SetPrevStmt(endLFinStmt); - } - else - { - lastStmt = endLFinStmt; - } - - // note that this sets BBF_IMPORTED on the block - impEndTreeList(callBlock, endLFinStmt, lastStmt); - } - - step = fgNewBBafter(BBJ_ALWAYS, callBlock, true); - /* The new block will inherit this block's weight */ - step->inheritWeight(block); - step->bbFlags |= BBF_IMPORTED | BBF_KEEP_BBJ_ALWAYS; - -#ifdef DEBUG - if (verbose) - { - printf("impImportLeave - jumping out of a finally-protected try, created step (BBJ_ALWAYS) block %s\n", - step->dspToString()); - } -#endif - - unsigned finallyNesting = compHndBBtab[XTnum].ebdHandlerNestingLevel; - assert(finallyNesting <= compHndBBtabCount); - - callBlock->bbJumpDest = HBtab->ebdHndBeg; // This callBlock will call the "finally" handler. - GenTree* endLFin = new (this, GT_END_LFIN) GenTreeVal(GT_END_LFIN, TYP_VOID, finallyNesting); - endLFinStmt = gtNewStmt(endLFin); - endCatches = NULL; - - encFinallies++; - - invalidatePreds = true; - } - } - - /* Append any remaining endCatches, if any */ - - assert(!encFinallies == !endLFinStmt); - - if (encFinallies == 0) - { - assert(step == DUMMY_INIT(NULL)); - block->bbJumpKind = BBJ_ALWAYS; // convert the BBJ_LEAVE to a BBJ_ALWAYS - - if (endCatches) - impAppendTree(endCatches, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); - -#ifdef DEBUG - if (verbose) - { - printf("impImportLeave - no enclosing finally-protected try blocks; convert CEE_LEAVE block to BBJ_ALWAYS " - "block %s\n", - block->dspToString()); - } -#endif - } - else - { - // If leaveTarget is the start of another try block, we want to make sure that - // we do not insert finalStep into that try block. Hence, we find the enclosing - // try block. - unsigned tryIndex = bbFindInnermostCommonTryRegion(step, leaveTarget); - - // Insert a new BB either in the try region indicated by tryIndex or - // the handler region indicated by leaveTarget->bbHndIndex, - // depending on which is the inner region. - BasicBlock* finalStep = fgNewBBinRegion(BBJ_ALWAYS, tryIndex, leaveTarget->bbHndIndex, step); - finalStep->bbFlags |= BBF_KEEP_BBJ_ALWAYS; - step->bbJumpDest = finalStep; - - /* The new block will inherit this block's weight */ - finalStep->inheritWeight(block); - -#ifdef DEBUG - if (verbose) - { - printf("impImportLeave - finalStep block required (encFinallies(%d) > 0), new block %s\n", encFinallies, - finalStep->dspToString()); - } -#endif - - Statement* lastStmt; - - if (endCatches) - { - lastStmt = gtNewStmt(endCatches); - endLFinStmt->SetNextStmt(lastStmt); - lastStmt->SetPrevStmt(endLFinStmt); - } - else - { - lastStmt = endLFinStmt; - } - - impEndTreeList(finalStep, endLFinStmt, lastStmt); - - finalStep->bbJumpDest = leaveTarget; // this is the ultimate destination of the LEAVE - - // Queue up the jump target for importing - - impImportBlockPending(leaveTarget); - - invalidatePreds = true; - } - - if (invalidatePreds && fgComputePredsDone) - { - JITDUMP("\n**** impImportLeave - Removing preds after creating new blocks\n"); - fgRemovePreds(); - } - -#ifdef DEBUG - fgVerifyHandlerTab(); - - if (verbose) - { - printf("\nAfter import CEE_LEAVE:\n"); - fgDispBasicBlocks(); - fgDispHandlerTab(); - } -#endif // DEBUG -} - -#else // FEATURE_EH_FUNCLETS - -void Compiler::impImportLeave(BasicBlock* block) -{ -#ifdef DEBUG - if (verbose) - { - printf("\nBefore import CEE_LEAVE in " FMT_BB " (targeting " FMT_BB "):\n", block->bbNum, - block->bbJumpDest->bbNum); - fgDispBasicBlocks(); - fgDispHandlerTab(); - } -#endif // DEBUG - - bool invalidatePreds = false; // If we create new blocks, invalidate the predecessor lists (if created) - unsigned blkAddr = block->bbCodeOffs; - BasicBlock* leaveTarget = block->bbJumpDest; - unsigned jmpAddr = leaveTarget->bbCodeOffs; - - // LEAVE clears the stack, spill side effects, and set stack to 0 - - impSpillSideEffects(true, (unsigned)CHECK_SPILL_ALL DEBUGARG("impImportLeave")); - verCurrentState.esStackDepth = 0; - - assert(block->bbJumpKind == BBJ_LEAVE); - assert(fgBBs == (BasicBlock**)0xCDCD || fgLookupBB(jmpAddr) != nullptr); // should be a BB boundary - - BasicBlock* step = nullptr; - - enum StepType - { - // No step type; step == NULL. - ST_None, - - // Is the step block the BBJ_ALWAYS block of a BBJ_CALLFINALLY/BBJ_ALWAYS pair? - // That is, is step->bbJumpDest where a finally will return to? - ST_FinallyReturn, - - // The step block is a catch return. - ST_Catch, - - // The step block is in a "try", created as the target for a finally return or the target for a catch return. - ST_Try - }; - StepType stepType = ST_None; - - unsigned XTnum; - EHblkDsc* HBtab; - - for (XTnum = 0, HBtab = compHndBBtab; XTnum < compHndBBtabCount; XTnum++, HBtab++) - { - // Grab the handler offsets - - IL_OFFSET tryBeg = HBtab->ebdTryBegOffs(); - IL_OFFSET tryEnd = HBtab->ebdTryEndOffs(); - IL_OFFSET hndBeg = HBtab->ebdHndBegOffs(); - IL_OFFSET hndEnd = HBtab->ebdHndEndOffs(); - - /* Is this a catch-handler we are CEE_LEAVEing out of? - */ - - if (jitIsBetween(blkAddr, hndBeg, hndEnd) && !jitIsBetween(jmpAddr, hndBeg, hndEnd)) - { - // Can't CEE_LEAVE out of a finally/fault handler - if (HBtab->HasFinallyOrFaultHandler()) - { - BADCODE("leave out of fault/finally block"); - } - - /* We are jumping out of a catch */ - - if (step == nullptr) - { - step = block; - step->bbJumpKind = BBJ_EHCATCHRET; // convert the BBJ_LEAVE to BBJ_EHCATCHRET - stepType = ST_Catch; - -#ifdef DEBUG - if (verbose) - { - printf("impImportLeave - jumping out of a catch (EH#%u), convert block " FMT_BB - " to BBJ_EHCATCHRET " - "block\n", - XTnum, step->bbNum); - } -#endif - } - else - { - BasicBlock* exitBlock; - - /* Create a new catch exit block in the catch region for the existing step block to jump to in this - * scope */ - exitBlock = fgNewBBinRegion(BBJ_EHCATCHRET, 0, XTnum + 1, step); - - assert(step->KindIs(BBJ_ALWAYS, BBJ_EHCATCHRET)); - step->bbJumpDest = exitBlock; // the previous step (maybe a call to a nested finally, or a nested catch - // exit) returns to this block - step->bbJumpDest->bbRefs++; - -#if defined(TARGET_ARM) - if (stepType == ST_FinallyReturn) - { - assert(step->bbJumpKind == BBJ_ALWAYS); - // Mark the target of a finally return - step->bbJumpDest->bbFlags |= BBF_FINALLY_TARGET; - } -#endif // defined(TARGET_ARM) - - /* The new block will inherit this block's weight */ - exitBlock->inheritWeight(block); - exitBlock->bbFlags |= BBF_IMPORTED; - - /* This exit block is the new step */ - step = exitBlock; - stepType = ST_Catch; - - invalidatePreds = true; - -#ifdef DEBUG - if (verbose) - { - printf("impImportLeave - jumping out of a catch (EH#%u), new BBJ_EHCATCHRET block " FMT_BB "\n", - XTnum, exitBlock->bbNum); - } -#endif - } - } - else if (HBtab->HasFinallyHandler() && jitIsBetween(blkAddr, tryBeg, tryEnd) && - !jitIsBetween(jmpAddr, tryBeg, tryEnd)) - { - /* We are jumping out of a finally-protected try */ - - BasicBlock* callBlock; - - if (step == nullptr) - { -#if FEATURE_EH_CALLFINALLY_THUNKS - - // Put the call to the finally in the enclosing region. - unsigned callFinallyTryIndex = - (HBtab->ebdEnclosingTryIndex == EHblkDsc::NO_ENCLOSING_INDEX) ? 0 : HBtab->ebdEnclosingTryIndex + 1; - unsigned callFinallyHndIndex = - (HBtab->ebdEnclosingHndIndex == EHblkDsc::NO_ENCLOSING_INDEX) ? 0 : HBtab->ebdEnclosingHndIndex + 1; - callBlock = fgNewBBinRegion(BBJ_CALLFINALLY, callFinallyTryIndex, callFinallyHndIndex, block); - - // Convert the BBJ_LEAVE to BBJ_ALWAYS, jumping to the new BBJ_CALLFINALLY. This is because - // the new BBJ_CALLFINALLY is in a different EH region, thus it can't just replace the BBJ_LEAVE, - // which might be in the middle of the "try". In most cases, the BBJ_ALWAYS will jump to the - // next block, and flow optimizations will remove it. - block->bbJumpKind = BBJ_ALWAYS; - block->bbJumpDest = callBlock; - block->bbJumpDest->bbRefs++; - - /* The new block will inherit this block's weight */ - callBlock->inheritWeight(block); - callBlock->bbFlags |= BBF_IMPORTED; - -#ifdef DEBUG - if (verbose) - { - printf("impImportLeave - jumping out of a finally-protected try (EH#%u), convert block " FMT_BB - " to " - "BBJ_ALWAYS, add BBJ_CALLFINALLY block " FMT_BB "\n", - XTnum, block->bbNum, callBlock->bbNum); - } -#endif - -#else // !FEATURE_EH_CALLFINALLY_THUNKS - - callBlock = block; - callBlock->bbJumpKind = BBJ_CALLFINALLY; // convert the BBJ_LEAVE to BBJ_CALLFINALLY - -#ifdef DEBUG - if (verbose) - { - printf("impImportLeave - jumping out of a finally-protected try (EH#%u), convert block " FMT_BB - " to " - "BBJ_CALLFINALLY block\n", - XTnum, callBlock->bbNum); - } -#endif - -#endif // !FEATURE_EH_CALLFINALLY_THUNKS - } - else - { - // Calling the finally block. We already have a step block that is either the call-to-finally from a - // more nested try/finally (thus we are jumping out of multiple nested 'try' blocks, each protected by - // a 'finally'), or the step block is the return from a catch. - // - // Due to ThreadAbortException, we can't have the catch return target the call-to-finally block - // directly. Note that if a 'catch' ends without resetting the ThreadAbortException, the VM will - // automatically re-raise the exception, using the return address of the catch (that is, the target - // block of the BBJ_EHCATCHRET) as the re-raise address. If this address is in a finally, the VM will - // refuse to do the re-raise, and the ThreadAbortException will get eaten (and lost). On AMD64/ARM64, - // we put the call-to-finally thunk in a special "cloned finally" EH region that does look like a - // finally clause to the VM. Thus, on these platforms, we can't have BBJ_EHCATCHRET target a - // BBJ_CALLFINALLY directly. (Note that on ARM32, we don't mark the thunk specially -- it lives directly - // within the 'try' region protected by the finally, since we generate code in such a way that execution - // never returns to the call-to-finally call, and the finally-protected 'try' region doesn't appear on - // stack walks.) - - assert(step->KindIs(BBJ_ALWAYS, BBJ_EHCATCHRET)); - -#if FEATURE_EH_CALLFINALLY_THUNKS - if (step->bbJumpKind == BBJ_EHCATCHRET) - { - // Need to create another step block in the 'try' region that will actually branch to the - // call-to-finally thunk. - BasicBlock* step2 = fgNewBBinRegion(BBJ_ALWAYS, XTnum + 1, 0, step); - step->bbJumpDest = step2; - step->bbJumpDest->bbRefs++; - step2->inheritWeight(block); - step2->bbFlags |= (block->bbFlags & BBF_RUN_RARELY) | BBF_IMPORTED; - -#ifdef DEBUG - if (verbose) - { - printf("impImportLeave - jumping out of a finally-protected try (EH#%u), step block is " - "BBJ_EHCATCHRET (" FMT_BB "), new BBJ_ALWAYS step-step block " FMT_BB "\n", - XTnum, step->bbNum, step2->bbNum); - } -#endif - - step = step2; - assert(stepType == ST_Catch); // Leave it as catch type for now. - } -#endif // FEATURE_EH_CALLFINALLY_THUNKS - -#if FEATURE_EH_CALLFINALLY_THUNKS - unsigned callFinallyTryIndex = - (HBtab->ebdEnclosingTryIndex == EHblkDsc::NO_ENCLOSING_INDEX) ? 0 : HBtab->ebdEnclosingTryIndex + 1; - unsigned callFinallyHndIndex = - (HBtab->ebdEnclosingHndIndex == EHblkDsc::NO_ENCLOSING_INDEX) ? 0 : HBtab->ebdEnclosingHndIndex + 1; -#else // !FEATURE_EH_CALLFINALLY_THUNKS - unsigned callFinallyTryIndex = XTnum + 1; - unsigned callFinallyHndIndex = 0; // don't care -#endif // !FEATURE_EH_CALLFINALLY_THUNKS - - callBlock = fgNewBBinRegion(BBJ_CALLFINALLY, callFinallyTryIndex, callFinallyHndIndex, step); - step->bbJumpDest = callBlock; // the previous call to a finally returns to this call (to the next - // finally in the chain) - step->bbJumpDest->bbRefs++; - -#if defined(TARGET_ARM) - if (stepType == ST_FinallyReturn) - { - assert(step->bbJumpKind == BBJ_ALWAYS); - // Mark the target of a finally return - step->bbJumpDest->bbFlags |= BBF_FINALLY_TARGET; - } -#endif // defined(TARGET_ARM) - - /* The new block will inherit this block's weight */ - callBlock->inheritWeight(block); - callBlock->bbFlags |= BBF_IMPORTED; - -#ifdef DEBUG - if (verbose) - { - printf("impImportLeave - jumping out of a finally-protected try (EH#%u), new BBJ_CALLFINALLY " - "block " FMT_BB "\n", - XTnum, callBlock->bbNum); - } -#endif - } - - step = fgNewBBafter(BBJ_ALWAYS, callBlock, true); - stepType = ST_FinallyReturn; - - /* The new block will inherit this block's weight */ - step->inheritWeight(block); - step->bbFlags |= BBF_IMPORTED | BBF_KEEP_BBJ_ALWAYS; - -#ifdef DEBUG - if (verbose) - { - printf("impImportLeave - jumping out of a finally-protected try (EH#%u), created step (BBJ_ALWAYS) " - "block " FMT_BB "\n", - XTnum, step->bbNum); - } -#endif - - callBlock->bbJumpDest = HBtab->ebdHndBeg; // This callBlock will call the "finally" handler. - - invalidatePreds = true; - } - else if (HBtab->HasCatchHandler() && jitIsBetween(blkAddr, tryBeg, tryEnd) && - !jitIsBetween(jmpAddr, tryBeg, tryEnd)) - { - // We are jumping out of a catch-protected try. - // - // If we are returning from a call to a finally, then we must have a step block within a try - // that is protected by a catch. This is so when unwinding from that finally (e.g., if code within the - // finally raises an exception), the VM will find this step block, notice that it is in a protected region, - // and invoke the appropriate catch. - // - // We also need to handle a special case with the handling of ThreadAbortException. If a try/catch - // catches a ThreadAbortException (which might be because it catches a parent, e.g. System.Exception), - // and the catch doesn't call System.Threading.Thread::ResetAbort(), then when the catch returns to the VM, - // the VM will automatically re-raise the ThreadAbortException. When it does this, it uses the target - // address of the catch return as the new exception address. That is, the re-raised exception appears to - // occur at the catch return address. If this exception return address skips an enclosing try/catch that - // catches ThreadAbortException, then the enclosing try/catch will not catch the exception, as it should. - // For example: - // - // try { - // try { - // // something here raises ThreadAbortException - // LEAVE LABEL_1; // no need to stop at LABEL_2 - // } catch (Exception) { - // // This catches ThreadAbortException, but doesn't call System.Threading.Thread::ResetAbort(), so - // // ThreadAbortException is re-raised by the VM at the address specified by the LEAVE opcode. - // // This is bad, since it means the outer try/catch won't get a chance to catch the re-raised - // // ThreadAbortException. So, instead, create step block LABEL_2 and LEAVE to that. We only - // // need to do this transformation if the current EH block is a try/catch that catches - // // ThreadAbortException (or one of its parents), however we might not be able to find that - // // information, so currently we do it for all catch types. - // LEAVE LABEL_1; // Convert this to LEAVE LABEL2; - // } - // LABEL_2: LEAVE LABEL_1; // inserted by this step creation code - // } catch (ThreadAbortException) { - // } - // LABEL_1: - // - // Note that this pattern isn't theoretical: it occurs in ASP.NET, in IL code generated by the Roslyn C# - // compiler. - - if ((stepType == ST_FinallyReturn) || (stepType == ST_Catch)) - { - BasicBlock* catchStep; - - assert(step); - - if (stepType == ST_FinallyReturn) - { - assert(step->bbJumpKind == BBJ_ALWAYS); - } - else - { - assert(stepType == ST_Catch); - assert(step->bbJumpKind == BBJ_EHCATCHRET); - } - - /* Create a new exit block in the try region for the existing step block to jump to in this scope */ - catchStep = fgNewBBinRegion(BBJ_ALWAYS, XTnum + 1, 0, step); - step->bbJumpDest = catchStep; - step->bbJumpDest->bbRefs++; - -#if defined(TARGET_ARM) - if (stepType == ST_FinallyReturn) - { - // Mark the target of a finally return - step->bbJumpDest->bbFlags |= BBF_FINALLY_TARGET; - } -#endif // defined(TARGET_ARM) - - /* The new block will inherit this block's weight */ - catchStep->inheritWeight(block); - catchStep->bbFlags |= BBF_IMPORTED; - -#ifdef DEBUG - if (verbose) - { - if (stepType == ST_FinallyReturn) - { - printf("impImportLeave - return from finally jumping out of a catch-protected try (EH#%u), new " - "BBJ_ALWAYS block " FMT_BB "\n", - XTnum, catchStep->bbNum); - } - else - { - assert(stepType == ST_Catch); - printf("impImportLeave - return from catch jumping out of a catch-protected try (EH#%u), new " - "BBJ_ALWAYS block " FMT_BB "\n", - XTnum, catchStep->bbNum); - } - } -#endif // DEBUG - - /* This block is the new step */ - step = catchStep; - stepType = ST_Try; - - invalidatePreds = true; - } - } - } - - if (step == nullptr) - { - block->bbJumpKind = BBJ_ALWAYS; // convert the BBJ_LEAVE to a BBJ_ALWAYS - -#ifdef DEBUG - if (verbose) - { - printf("impImportLeave - no enclosing finally-protected try blocks or catch handlers; convert CEE_LEAVE " - "block " FMT_BB " to BBJ_ALWAYS\n", - block->bbNum); - } -#endif - } - else - { - step->bbJumpDest = leaveTarget; // this is the ultimate destination of the LEAVE - -#if defined(TARGET_ARM) - if (stepType == ST_FinallyReturn) - { - assert(step->bbJumpKind == BBJ_ALWAYS); - // Mark the target of a finally return - step->bbJumpDest->bbFlags |= BBF_FINALLY_TARGET; - } -#endif // defined(TARGET_ARM) - -#ifdef DEBUG - if (verbose) - { - printf("impImportLeave - final destination of step blocks set to " FMT_BB "\n", leaveTarget->bbNum); - } -#endif - - // Queue up the jump target for importing - - impImportBlockPending(leaveTarget); - } - - if (invalidatePreds && fgComputePredsDone) - { - JITDUMP("\n**** impImportLeave - Removing preds after creating new blocks\n"); - fgRemovePreds(); - } - -#ifdef DEBUG - fgVerifyHandlerTab(); - - if (verbose) - { - printf("\nAfter import CEE_LEAVE:\n"); - fgDispBasicBlocks(); - fgDispHandlerTab(); - } -#endif // DEBUG -} - -#endif // FEATURE_EH_FUNCLETS - -/*****************************************************************************/ -// This is called when reimporting a leave block. It resets the JumpKind, -// JumpDest, and bbNext to the original values - -void Compiler::impResetLeaveBlock(BasicBlock* block, unsigned jmpAddr) -{ -#if defined(FEATURE_EH_FUNCLETS) - // With EH Funclets, while importing leave opcode we create another block ending with BBJ_ALWAYS (call it B1) - // and the block containing leave (say B0) is marked as BBJ_CALLFINALLY. Say for some reason we reimport B0, - // it is reset (in this routine) by marking as ending with BBJ_LEAVE and further down when B0 is reimported, we - // create another BBJ_ALWAYS (call it B2). In this process B1 gets orphaned and any blocks to which B1 is the - // only predecessor are also considered orphans and attempted to be deleted. - // - // try { - // .... - // try - // { - // .... - // leave OUTSIDE; // B0 is the block containing this leave, following this would be B1 - // } finally { } - // } finally { } - // OUTSIDE: - // - // In the above nested try-finally example, we create a step block (call it Bstep) which in branches to a block - // where a finally would branch to (and such block is marked as finally target). Block B1 branches to step block. - // Because of re-import of B0, Bstep is also orphaned. Since Bstep is a finally target it cannot be removed. To - // work around this we will duplicate B0 (call it B0Dup) before resetting. B0Dup is marked as BBJ_CALLFINALLY and - // only serves to pair up with B1 (BBJ_ALWAYS) that got orphaned. Now during orphan block deletion B0Dup and B1 - // will be treated as pair and handled correctly. - if (block->bbJumpKind == BBJ_CALLFINALLY) - { - BasicBlock* dupBlock = bbNewBasicBlock(block->bbJumpKind); - dupBlock->bbFlags = block->bbFlags; - dupBlock->bbJumpDest = block->bbJumpDest; - dupBlock->copyEHRegion(block); - dupBlock->bbCatchTyp = block->bbCatchTyp; - - // Mark this block as - // a) not referenced by any other block to make sure that it gets deleted - // b) weight zero - // c) prevent from being imported - // d) as internal - // e) as rarely run - dupBlock->bbRefs = 0; - dupBlock->bbWeight = BB_ZERO_WEIGHT; - dupBlock->bbFlags |= BBF_IMPORTED | BBF_INTERNAL | BBF_RUN_RARELY; - - // Insert the block right after the block which is getting reset so that BBJ_CALLFINALLY and BBJ_ALWAYS - // will be next to each other. - fgInsertBBafter(block, dupBlock); - -#ifdef DEBUG - if (verbose) - { - printf("New Basic Block " FMT_BB " duplicate of " FMT_BB " created.\n", dupBlock->bbNum, block->bbNum); - } -#endif - } -#endif // FEATURE_EH_FUNCLETS - - block->bbJumpKind = BBJ_LEAVE; - fgInitBBLookup(); - block->bbJumpDest = fgLookupBB(jmpAddr); - - // We will leave the BBJ_ALWAYS block we introduced. When it's reimported - // the BBJ_ALWAYS block will be unreachable, and will be removed after. The - // reason we don't want to remove the block at this point is that if we call - // fgInitBBLookup() again we will do it wrong as the BBJ_ALWAYS block won't be - // added and the linked list length will be different than fgBBcount. -} - -/*****************************************************************************/ -// Get the first non-prefix opcode. Used for verification of valid combinations -// of prefixes and actual opcodes. - -OPCODE Compiler::impGetNonPrefixOpcode(const BYTE* codeAddr, const BYTE* codeEndp) -{ - while (codeAddr < codeEndp) - { - OPCODE opcode = (OPCODE)getU1LittleEndian(codeAddr); - codeAddr += sizeof(__int8); - - if (opcode == CEE_PREFIX1) - { - if (codeAddr >= codeEndp) - { - break; - } - opcode = (OPCODE)(getU1LittleEndian(codeAddr) + 256); - codeAddr += sizeof(__int8); - } - - switch (opcode) - { - case CEE_UNALIGNED: - case CEE_VOLATILE: - case CEE_TAILCALL: - case CEE_CONSTRAINED: - case CEE_READONLY: - break; - default: - return opcode; - } - - codeAddr += opcodeSizes[opcode]; - } - - return CEE_ILLEGAL; -} - -/*****************************************************************************/ -// Checks whether the opcode is a valid opcode for volatile. and unaligned. prefixes - -void Compiler::impValidateMemoryAccessOpcode(const BYTE* codeAddr, const BYTE* codeEndp, bool volatilePrefix) -{ - OPCODE opcode = impGetNonPrefixOpcode(codeAddr, codeEndp); - - if (!( - // Opcode of all ldind and stdind happen to be in continuous, except stind.i. - ((CEE_LDIND_I1 <= opcode) && (opcode <= CEE_STIND_R8)) || (opcode == CEE_STIND_I) || - (opcode == CEE_LDFLD) || (opcode == CEE_STFLD) || (opcode == CEE_LDOBJ) || (opcode == CEE_STOBJ) || - (opcode == CEE_INITBLK) || (opcode == CEE_CPBLK) || - // volatile. prefix is allowed with the ldsfld and stsfld - (volatilePrefix && ((opcode == CEE_LDSFLD) || (opcode == CEE_STSFLD))))) - { - BADCODE("Invalid opcode for unaligned. or volatile. prefix"); - } -} - -/*****************************************************************************/ - -#ifdef DEBUG - -#undef RETURN // undef contracts RETURN macro - -enum controlFlow_t -{ - NEXT, - CALL, - RETURN, - THROW, - BRANCH, - COND_BRANCH, - BREAK, - PHI, - META, -}; - -const static controlFlow_t controlFlow[] = { -#define OPDEF(c, s, pop, push, args, type, l, s1, s2, flow) flow, -#include "opcode.def" -#undef OPDEF -}; - -#endif // DEBUG - -/***************************************************************************** - * Determine the result type of an arithemetic operation - * On 64-bit inserts upcasts when native int is mixed with int32 - */ -var_types Compiler::impGetByRefResultType(genTreeOps oper, bool fUnsigned, GenTree** pOp1, GenTree** pOp2) -{ - var_types type = TYP_UNDEF; - GenTree* op1 = *pOp1; - GenTree* op2 = *pOp2; - - // Arithemetic operations are generally only allowed with - // primitive types, but certain operations are allowed - // with byrefs - - if ((oper == GT_SUB) && (genActualType(op1->TypeGet()) == TYP_BYREF || genActualType(op2->TypeGet()) == TYP_BYREF)) - { - if ((genActualType(op1->TypeGet()) == TYP_BYREF) && (genActualType(op2->TypeGet()) == TYP_BYREF)) - { - // byref1-byref2 => gives a native int - type = TYP_I_IMPL; - } - else if (genActualTypeIsIntOrI(op1->TypeGet()) && (genActualType(op2->TypeGet()) == TYP_BYREF)) - { - // [native] int - byref => gives a native int - - // - // The reason is that it is possible, in managed C++, - // to have a tree like this: - // - // - - // / \. - // / \. - // / \. - // / \. - // const(h) int addr byref - // - // VSW 318822 - // - // So here we decide to make the resulting type to be a native int. - CLANG_FORMAT_COMMENT_ANCHOR; - -#ifdef TARGET_64BIT - if (genActualType(op1->TypeGet()) != TYP_I_IMPL) - { - // insert an explicit upcast - op1 = *pOp1 = gtNewCastNode(TYP_I_IMPL, op1, fUnsigned, fUnsigned ? TYP_U_IMPL : TYP_I_IMPL); - } -#endif // TARGET_64BIT - - type = TYP_I_IMPL; - } - else - { - // byref - [native] int => gives a byref - assert(genActualType(op1->TypeGet()) == TYP_BYREF && genActualTypeIsIntOrI(op2->TypeGet())); - -#ifdef TARGET_64BIT - if ((genActualType(op2->TypeGet()) != TYP_I_IMPL)) - { - // insert an explicit upcast - op2 = *pOp2 = gtNewCastNode(TYP_I_IMPL, op2, fUnsigned, fUnsigned ? TYP_U_IMPL : TYP_I_IMPL); - } -#endif // TARGET_64BIT - - type = TYP_BYREF; - } - } - else if ((oper == GT_ADD) && - (genActualType(op1->TypeGet()) == TYP_BYREF || genActualType(op2->TypeGet()) == TYP_BYREF)) - { - // byref + [native] int => gives a byref - // (or) - // [native] int + byref => gives a byref - - // only one can be a byref : byref op byref not allowed - assert(genActualType(op1->TypeGet()) != TYP_BYREF || genActualType(op2->TypeGet()) != TYP_BYREF); - assert(genActualTypeIsIntOrI(op1->TypeGet()) || genActualTypeIsIntOrI(op2->TypeGet())); - -#ifdef TARGET_64BIT - if (genActualType(op2->TypeGet()) == TYP_BYREF) - { - if (genActualType(op1->TypeGet()) != TYP_I_IMPL) - { - // insert an explicit upcast - op1 = *pOp1 = gtNewCastNode(TYP_I_IMPL, op1, fUnsigned, fUnsigned ? TYP_U_IMPL : TYP_I_IMPL); - } - } - else if (genActualType(op2->TypeGet()) != TYP_I_IMPL) - { - // insert an explicit upcast - op2 = *pOp2 = gtNewCastNode(TYP_I_IMPL, op2, fUnsigned, fUnsigned ? TYP_U_IMPL : TYP_I_IMPL); - } -#endif // TARGET_64BIT - - type = TYP_BYREF; - } -#ifdef TARGET_64BIT - else if (genActualType(op1->TypeGet()) == TYP_I_IMPL || genActualType(op2->TypeGet()) == TYP_I_IMPL) - { - assert(!varTypeIsFloating(op1->gtType) && !varTypeIsFloating(op2->gtType)); - - // int + long => gives long - // long + int => gives long - // we get this because in the IL the long isn't Int64, it's just IntPtr - - if (genActualType(op1->TypeGet()) != TYP_I_IMPL) - { - // insert an explicit upcast - op1 = *pOp1 = gtNewCastNode(TYP_I_IMPL, op1, fUnsigned, fUnsigned ? TYP_U_IMPL : TYP_I_IMPL); - } - else if (genActualType(op2->TypeGet()) != TYP_I_IMPL) - { - // insert an explicit upcast - op2 = *pOp2 = gtNewCastNode(TYP_I_IMPL, op2, fUnsigned, fUnsigned ? TYP_U_IMPL : TYP_I_IMPL); - } - - type = TYP_I_IMPL; - } -#else // 32-bit TARGET - else if (genActualType(op1->TypeGet()) == TYP_LONG || genActualType(op2->TypeGet()) == TYP_LONG) - { - assert(!varTypeIsFloating(op1->gtType) && !varTypeIsFloating(op2->gtType)); - - // int + long => gives long - // long + int => gives long - - type = TYP_LONG; - } -#endif // TARGET_64BIT - else - { - // int + int => gives an int - assert(genActualType(op1->TypeGet()) != TYP_BYREF && genActualType(op2->TypeGet()) != TYP_BYREF); - - assert(genActualType(op1->TypeGet()) == genActualType(op2->TypeGet()) || - (varTypeIsFloating(op1->gtType) && varTypeIsFloating(op2->gtType))); - - type = genActualType(op1->gtType); - - // If both operands are TYP_FLOAT, then leave it as TYP_FLOAT. - // Otherwise, turn floats into doubles - if ((type == TYP_FLOAT) && (genActualType(op2->gtType) != TYP_FLOAT)) - { - assert(genActualType(op2->gtType) == TYP_DOUBLE); - type = TYP_DOUBLE; - } - } - - assert(type == TYP_BYREF || type == TYP_DOUBLE || type == TYP_FLOAT || type == TYP_LONG || type == TYP_INT); - return type; -} - -//------------------------------------------------------------------------ -// impOptimizeCastClassOrIsInst: attempt to resolve a cast when jitting -// -// Arguments: -// op1 - value to cast -// pResolvedToken - resolved token for type to cast to -// isCastClass - true if this is a castclass, false if isinst -// -// Return Value: -// tree representing optimized cast, or null if no optimization possible - -GenTree* Compiler::impOptimizeCastClassOrIsInst(GenTree* op1, CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isCastClass) -{ - assert(op1->TypeGet() == TYP_REF); - - // Don't optimize for minopts or debug codegen. - if (opts.OptimizationDisabled()) - { - return nullptr; - } - - // See what we know about the type of the object being cast. - bool isExact = false; - bool isNonNull = false; - CORINFO_CLASS_HANDLE fromClass = gtGetClassHandle(op1, &isExact, &isNonNull); - - if (fromClass != nullptr) - { - CORINFO_CLASS_HANDLE toClass = pResolvedToken->hClass; - JITDUMP("\nConsidering optimization of %s from %s%p (%s) to %p (%s)\n", isCastClass ? "castclass" : "isinst", - isExact ? "exact " : "", dspPtr(fromClass), eeGetClassName(fromClass), dspPtr(toClass), - eeGetClassName(toClass)); - - // Perhaps we know if the cast will succeed or fail. - TypeCompareState castResult = info.compCompHnd->compareTypesForCast(fromClass, toClass); - - if (castResult == TypeCompareState::Must) - { - // Cast will succeed, result is simply op1. - JITDUMP("Cast will succeed, optimizing to simply return input\n"); - return op1; - } - else if (castResult == TypeCompareState::MustNot) - { - // See if we can sharpen exactness by looking for final classes - if (!isExact) - { - isExact = impIsClassExact(fromClass); - } - - // Cast to exact type will fail. Handle case where we have - // an exact type (that is, fromClass is not a subtype) - // and we're not going to throw on failure. - if (isExact && !isCastClass) - { - JITDUMP("Cast will fail, optimizing to return null\n"); - GenTree* result = gtNewIconNode(0, TYP_REF); - - // If the cast was fed by a box, we can remove that too. - if (op1->IsBoxedValue()) - { - JITDUMP("Also removing upstream box\n"); - gtTryRemoveBoxUpstreamEffects(op1); - } - - return result; - } - else if (isExact) - { - JITDUMP("Not optimizing failing castclass (yet)\n"); - } - else - { - JITDUMP("Can't optimize since fromClass is inexact\n"); - } - } - else - { - JITDUMP("Result of cast unknown, must generate runtime test\n"); - } - } - else - { - JITDUMP("\nCan't optimize since fromClass is unknown\n"); - } - - return nullptr; -} - -//------------------------------------------------------------------------ -// impCastClassOrIsInstToTree: build and import castclass/isinst -// -// Arguments: -// op1 - value to cast -// op2 - type handle for type to cast to -// pResolvedToken - resolved token from the cast operation -// isCastClass - true if this is castclass, false means isinst -// -// Return Value: -// Tree representing the cast -// -// Notes: -// May expand into a series of runtime checks or a helper call. - -GenTree* Compiler::impCastClassOrIsInstToTree( - GenTree* op1, GenTree* op2, CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isCastClass, IL_OFFSET ilOffset) -{ - assert(op1->TypeGet() == TYP_REF); - - // Optimistically assume the jit should expand this as an inline test - bool shouldExpandInline = true; - bool isClassExact = impIsClassExact(pResolvedToken->hClass); - - // Profitability check. - // - // Don't bother with inline expansion when jit is trying to generate code quickly - if (opts.OptimizationDisabled()) - { - // not worth the code expansion if jitting fast or in a rarely run block - shouldExpandInline = false; - } - else if ((op1->gtFlags & GTF_GLOB_EFFECT) && lvaHaveManyLocals()) - { - // not worth creating an untracked local variable - shouldExpandInline = false; - } - else if (opts.jitFlags->IsSet(JitFlags::JIT_FLAG_BBINSTR) && (JitConfig.JitProfileCasts() == 1)) - { - // Optimizations are enabled but we're still instrumenting (including casts) - if (isCastClass && !isClassExact) - { - // Usually, we make a speculative assumption that it makes sense to expand castclass - // even for non-sealed classes, but let's rely on PGO in this specific case - shouldExpandInline = false; - } - } - - if (shouldExpandInline && compCurBB->isRunRarely()) - { - // For cold blocks we only expand castclass against exact classes becauses because it's cheap - shouldExpandInline = isCastClass && isClassExact; - } - - // Pessimistically assume the jit cannot expand this as an inline test - bool canExpandInline = false; - bool partialExpand = false; - const CorInfoHelpFunc helper = info.compCompHnd->getCastingHelper(pResolvedToken, isCastClass); - - GenTree* exactCls = nullptr; - - // Legality check. - // - // Not all classclass/isinst operations can be inline expanded. - // Check legality only if an inline expansion is desirable. - if (shouldExpandInline) - { - if (isCastClass) - { - // Jit can only inline expand the normal CHKCASTCLASS helper. - canExpandInline = (helper == CORINFO_HELP_CHKCASTCLASS); - } - else - { - if (helper == CORINFO_HELP_ISINSTANCEOFCLASS) - { - // If the class is exact, the jit can expand the IsInst check inline. - canExpandInline = isClassExact; - } - } - - // Check if this cast helper have some profile data - if (impIsCastHelperMayHaveProfileData(helper)) - { - const int maxLikelyClasses = 32; - LikelyClassMethodRecord likelyClasses[maxLikelyClasses]; - unsigned likelyClassCount = - getLikelyClasses(likelyClasses, maxLikelyClasses, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset); - - if (likelyClassCount > 0) - { -#ifdef DEBUG - // Optional stress mode to pick a random known class, rather than - // the most likely known class. - if (JitConfig.JitRandomGuardedDevirtualization() != 0) - { - // Reuse the random inliner's random state. - CLRRandom* const random = - impInlineRoot()->m_inlineStrategy->GetRandom(JitConfig.JitRandomGuardedDevirtualization()); - - unsigned index = static_cast(random->Next(static_cast(likelyClassCount))); - likelyClasses[0].handle = likelyClasses[index].handle; - likelyClasses[0].likelihood = 100; - likelyClassCount = 1; - } -#endif - - LikelyClassMethodRecord likelyClass = likelyClasses[0]; - CORINFO_CLASS_HANDLE likelyCls = (CORINFO_CLASS_HANDLE)likelyClass.handle; - - if ((likelyCls != NO_CLASS_HANDLE) && - (likelyClass.likelihood > (UINT32)JitConfig.JitGuardedDevirtualizationChainLikelihood())) - { - if ((info.compCompHnd->compareTypesForCast(likelyCls, pResolvedToken->hClass) == - TypeCompareState::Must)) - { - assert((info.compCompHnd->getClassAttribs(likelyCls) & - (CORINFO_FLG_INTERFACE | CORINFO_FLG_ABSTRACT)) == 0); - JITDUMP("Adding \"is %s (%X)\" check as a fast path for %s using PGO data.\n", - eeGetClassName(likelyCls), likelyCls, isCastClass ? "castclass" : "isinst"); - - canExpandInline = true; - partialExpand = true; - exactCls = gtNewIconEmbClsHndNode(likelyCls); - } - } - } - } - } - - const bool expandInline = canExpandInline && shouldExpandInline; - - if (!expandInline) - { - JITDUMP("\nExpanding %s as call because %s\n", isCastClass ? "castclass" : "isinst", - canExpandInline ? "want smaller code or faster jitting" : "inline expansion not legal"); - - // If we CSE this class handle we prevent assertionProp from making SubType assertions - // so instead we force the CSE logic to not consider CSE-ing this class handle. - // - op2->gtFlags |= GTF_DONT_CSE; - - GenTreeCall* call = gtNewHelperCallNode(helper, TYP_REF, op2, op1); - if ((JitConfig.JitClassProfiling() > 0) && impIsCastHelperEligibleForClassProbe(call) && !isClassExact) - { - HandleHistogramProfileCandidateInfo* pInfo = new (this, CMK_Inlining) HandleHistogramProfileCandidateInfo; - pInfo->ilOffset = ilOffset; - pInfo->probeIndex = info.compHandleHistogramProbeCount++; - call->gtHandleHistogramProfileCandidateInfo = pInfo; - compCurBB->bbFlags |= BBF_HAS_HISTOGRAM_PROFILE; - } - return call; - } - - JITDUMP("\nExpanding %s inline\n", isCastClass ? "castclass" : "isinst"); - - impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("bubbling QMark2")); - - GenTree* temp; - GenTree* condMT; - // - // expand the methodtable match: - // - // condMT ==> GT_NE - // / \. - // GT_IND op2 (typically CNS_INT) - // | - // op1Copy - // - - // This can replace op1 with a GT_COMMA that evaluates op1 into a local - // - op1 = impCloneExpr(op1, &temp, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, nullptr DEBUGARG("CASTCLASS eval op1")); - // - // op1 is now known to be a non-complex tree - // thus we can use gtClone(op1) from now on - // - - GenTree* op2Var = op2; - if (isCastClass && !partialExpand) - { - op2Var = fgInsertCommaFormTemp(&op2); - lvaTable[op2Var->AsLclVarCommon()->GetLclNum()].lvIsCSE = true; - } - temp = gtNewMethodTableLookup(temp); - condMT = gtNewOperNode(GT_NE, TYP_INT, temp, exactCls != nullptr ? exactCls : op2); - - GenTree* condNull; - // - // expand the null check: - // - // condNull ==> GT_EQ - // / \. - // op1Copy CNS_INT - // null - // - condNull = gtNewOperNode(GT_EQ, TYP_INT, gtClone(op1), gtNewIconNode(0, TYP_REF)); - - // - // expand the true and false trees for the condMT - // - GenTree* condFalse = gtClone(op1); - GenTree* condTrue; - if (isCastClass) - { - // - // use the special helper that skips the cases checked by our inlined cast - // - const CorInfoHelpFunc specialHelper = CORINFO_HELP_CHKCASTCLASS_SPECIAL; - - condTrue = gtNewHelperCallNode(specialHelper, TYP_REF, partialExpand ? op2 : op2Var, gtClone(op1)); - } - else if (partialExpand) - { - condTrue = gtNewHelperCallNode(helper, TYP_REF, op2, gtClone(op1)); - } - else - { - condTrue = gtNewIconNode(0, TYP_REF); - } - - GenTree* qmarkMT; - // - // Generate first QMARK - COLON tree - // - // qmarkMT ==> GT_QMARK - // / \. - // condMT GT_COLON - // / \. - // condFalse condTrue - // - temp = new (this, GT_COLON) GenTreeColon(TYP_REF, condTrue, condFalse); - qmarkMT = gtNewQmarkNode(TYP_REF, condMT, temp->AsColon()); - - if (isCastClass && isClassExact && condTrue->OperIs(GT_CALL)) - { - // condTrue is used only for throwing InvalidCastException in case of casting to an exact class. - condTrue->AsCall()->gtCallMoreFlags |= GTF_CALL_M_DOES_NOT_RETURN; - } - - GenTree* qmarkNull; - // - // Generate second QMARK - COLON tree - // - // qmarkNull ==> GT_QMARK - // / \. - // condNull GT_COLON - // / \. - // qmarkMT op1Copy - // - temp = new (this, GT_COLON) GenTreeColon(TYP_REF, gtClone(op1), qmarkMT); - qmarkNull = gtNewQmarkNode(TYP_REF, condNull, temp->AsColon()); - qmarkNull->gtFlags |= GTF_QMARK_CAST_INSTOF; - - // Make QMark node a top level node by spilling it. - unsigned tmp = lvaGrabTemp(true DEBUGARG("spilling QMark2")); - impAssignTempGen(tmp, qmarkNull, (unsigned)CHECK_SPILL_NONE); - - // TODO-CQ: Is it possible op1 has a better type? - // - // See also gtGetHelperCallClassHandle where we make the same - // determination for the helper call variants. - LclVarDsc* lclDsc = lvaGetDesc(tmp); - assert(lclDsc->lvSingleDef == 0); - lclDsc->lvSingleDef = 1; - JITDUMP("Marked V%02u as a single def temp\n", tmp); - lvaSetClass(tmp, pResolvedToken->hClass); - return gtNewLclvNode(tmp, TYP_REF); -} - -#ifndef DEBUG -#define assertImp(cond) ((void)0) -#else -#define assertImp(cond) \ - do \ - { \ - if (!(cond)) \ - { \ - const int cchAssertImpBuf = 600; \ - char* assertImpBuf = (char*)_alloca(cchAssertImpBuf); \ - _snprintf_s(assertImpBuf, cchAssertImpBuf, cchAssertImpBuf - 1, \ - "%s : Possibly bad IL with CEE_%s at offset %04Xh (op1=%s op2=%s stkDepth=%d)", #cond, \ - impCurOpcName, impCurOpcOffs, op1 ? varTypeName(op1->TypeGet()) : "NULL", \ - op2 ? varTypeName(op2->TypeGet()) : "NULL", verCurrentState.esStackDepth); \ - assertAbort(assertImpBuf, __FILE__, __LINE__); \ - } \ - } while (0) -#endif // DEBUG - -//------------------------------------------------------------------------ -// impBlockIsInALoop: check if a block might be in a loop -// -// Arguments: -// block - block to check -// -// Returns: -// true if the block might be in a loop. -// -// Notes: -// Conservatively correct; may return true for some blocks that are -// not actually in loops. -// -bool Compiler::impBlockIsInALoop(BasicBlock* block) -{ - return (compIsForInlining() && ((impInlineInfo->iciBlock->bbFlags & BBF_BACKWARD_JUMP) != 0)) || - ((block->bbFlags & BBF_BACKWARD_JUMP) != 0); -} - -#ifdef _PREFAST_ -#pragma warning(push) -#pragma warning(disable : 21000) // Suppress PREFast warning about overly large function -#endif -/***************************************************************************** - * Import the instr for the given basic block - */ -void Compiler::impImportBlockCode(BasicBlock* block) -{ -#define _impResolveToken(kind) impResolveToken(codeAddr, &resolvedToken, kind) - -#ifdef DEBUG - - if (verbose) - { - printf("\nImporting " FMT_BB " (PC=%03u) of '%s'", block->bbNum, block->bbCodeOffs, info.compFullName); - } -#endif - - unsigned nxtStmtIndex = impInitBlockLineInfo(); - IL_OFFSET nxtStmtOffs; - CorInfoHelpFunc helper; - CorInfoIsAccessAllowedResult accessAllowedResult; - CORINFO_HELPER_DESC calloutHelper; - const BYTE* lastLoadToken = nullptr; - - /* Get the tree list started */ - - impBeginTreeList(); - -#ifdef FEATURE_ON_STACK_REPLACEMENT - - bool enablePatchpoints = opts.jitFlags->IsSet(JitFlags::JIT_FLAG_TIER0) && (JitConfig.TC_OnStackReplacement() > 0); - -#ifdef DEBUG - - // Optionally suppress patchpoints by method hash - // - static ConfigMethodRange JitEnablePatchpointRange; - JitEnablePatchpointRange.EnsureInit(JitConfig.JitEnablePatchpointRange()); - const unsigned hash = impInlineRoot()->info.compMethodHash(); - const bool inRange = JitEnablePatchpointRange.Contains(hash); - enablePatchpoints &= inRange; - -#endif // DEBUG - - if (enablePatchpoints) - { - // We don't inline at Tier0, if we do, we may need rethink our approach. - // Could probably support inlines that don't introduce flow. - // - assert(!compIsForInlining()); - - // OSR is not yet supported for methods with explicit tail calls. - // - // But we also do not have to switch these methods to be optimized, as we should be - // able to avoid getting trapped in Tier0 code by normal call counting. - // So instead, just suppress adding patchpoints. - // - if (!compTailPrefixSeen) - { - // We only need to add patchpoints if the method can loop. - // - if (compHasBackwardJump) - { - assert(compCanHavePatchpoints()); - - // By default we use the "adaptive" strategy. - // - // This can create both source and target patchpoints within a given - // loop structure, which isn't ideal, but is not incorrect. We will - // just have some extra Tier0 overhead. - // - // Todo: implement support for mid-block patchpoints. If `block` - // is truly a backedge source (and not in a handler) then we should be - // able to find a stack empty point somewhere in the block. - // - const int patchpointStrategy = JitConfig.TC_PatchpointStrategy(); - bool addPatchpoint = false; - bool mustUseTargetPatchpoint = false; - - switch (patchpointStrategy) - { - default: - { - // Patchpoints at backedge sources, if possible, otherwise targets. - // - addPatchpoint = ((block->bbFlags & BBF_BACKWARD_JUMP_SOURCE) == BBF_BACKWARD_JUMP_SOURCE); - mustUseTargetPatchpoint = (verCurrentState.esStackDepth != 0) || block->hasHndIndex(); - break; - } - - case 1: - { - // Patchpoints at stackempty backedge targets. - // Note if we have loops where the IL stack is not empty on the backedge we can't patchpoint - // them. - // - // We should not have allowed OSR if there were backedges in handlers. - // - assert(!block->hasHndIndex()); - addPatchpoint = ((block->bbFlags & BBF_BACKWARD_JUMP_TARGET) == BBF_BACKWARD_JUMP_TARGET) && - (verCurrentState.esStackDepth == 0); - break; - } - - case 2: - { - // Adaptive strategy. - // - // Patchpoints at backedge targets if there are multiple backedges, - // otherwise at backedge sources, if possible. Note a block can be both; if so we - // just need one patchpoint. - // - if ((block->bbFlags & BBF_BACKWARD_JUMP_TARGET) == BBF_BACKWARD_JUMP_TARGET) - { - // We don't know backedge count, so just use ref count. - // - addPatchpoint = (block->bbRefs > 1) && (verCurrentState.esStackDepth == 0); - } - - if (!addPatchpoint && ((block->bbFlags & BBF_BACKWARD_JUMP_SOURCE) == BBF_BACKWARD_JUMP_SOURCE)) - { - addPatchpoint = true; - mustUseTargetPatchpoint = (verCurrentState.esStackDepth != 0) || block->hasHndIndex(); - - // Also force target patchpoint if target block has multiple (backedge) preds. - // - if (!mustUseTargetPatchpoint) - { - for (BasicBlock* const succBlock : block->Succs(this)) - { - if ((succBlock->bbNum <= block->bbNum) && (succBlock->bbRefs > 1)) - { - mustUseTargetPatchpoint = true; - break; - } - } - } - } - break; - } - } - - if (addPatchpoint) - { - if (mustUseTargetPatchpoint) - { - // We wanted a source patchpoint, but could not have one. - // So, add patchpoints to the backedge targets. - // - for (BasicBlock* const succBlock : block->Succs(this)) - { - if (succBlock->bbNum <= block->bbNum) - { - // The succBlock had better agree it's a target. - // - assert((succBlock->bbFlags & BBF_BACKWARD_JUMP_TARGET) == BBF_BACKWARD_JUMP_TARGET); - - // We may already have decided to put a patchpoint in succBlock. If not, add one. - // - if ((succBlock->bbFlags & BBF_PATCHPOINT) != 0) - { - // In some cases the target may not be stack-empty at entry. - // If so, we will bypass patchpoints for this backedge. - // - if (succBlock->bbStackDepthOnEntry() > 0) - { - JITDUMP("\nCan't set source patchpoint at " FMT_BB ", can't use target " FMT_BB - " as it has non-empty stack on entry.\n", - block->bbNum, succBlock->bbNum); - } - else - { - JITDUMP("\nCan't set source patchpoint at " FMT_BB ", using target " FMT_BB - " instead\n", - block->bbNum, succBlock->bbNum); - - assert(!succBlock->hasHndIndex()); - succBlock->bbFlags |= BBF_PATCHPOINT; - } - } - } - } - } - else - { - assert(!block->hasHndIndex()); - block->bbFlags |= BBF_PATCHPOINT; - } - - setMethodHasPatchpoint(); - } - } - else - { - // Should not see backward branch targets w/o backwards branches. - // So if !compHasBackwardsBranch, these flags should never be set. - // - assert((block->bbFlags & (BBF_BACKWARD_JUMP_TARGET | BBF_BACKWARD_JUMP_SOURCE)) == 0); - } - } - -#ifdef DEBUG - // As a stress test, we can place patchpoints at the start of any block - // that is a stack empty point and is not within a handler. - // - // Todo: enable for mid-block stack empty points too. - // - const int offsetOSR = JitConfig.JitOffsetOnStackReplacement(); - const int randomOSR = JitConfig.JitRandomOnStackReplacement(); - const bool tryOffsetOSR = offsetOSR >= 0; - const bool tryRandomOSR = randomOSR > 0; - - if (compCanHavePatchpoints() && (tryOffsetOSR || tryRandomOSR) && (verCurrentState.esStackDepth == 0) && - !block->hasHndIndex() && ((block->bbFlags & BBF_PATCHPOINT) == 0)) - { - // Block start can have a patchpoint. See if we should add one. - // - bool addPatchpoint = false; - - // Specific offset? - // - if (tryOffsetOSR) - { - if (impCurOpcOffs == (unsigned)offsetOSR) - { - addPatchpoint = true; - } - } - // Random? - // - else - { - // Reuse the random inliner's random state. - // Note m_inlineStrategy is always created, even if we're not inlining. - // - CLRRandom* const random = impInlineRoot()->m_inlineStrategy->GetRandom(randomOSR); - const int randomValue = (int)random->Next(100); - - addPatchpoint = (randomValue < randomOSR); - } - - if (addPatchpoint) - { - block->bbFlags |= BBF_PATCHPOINT; - setMethodHasPatchpoint(); - } - - JITDUMP("\n** %s patchpoint%s added to " FMT_BB " (il offset %u)\n", tryOffsetOSR ? "offset" : "random", - addPatchpoint ? "" : " not", block->bbNum, impCurOpcOffs); - } - -#endif // DEBUG - } - - // Mark stack-empty rare blocks to be considered for partial compilation. - // - // Ideally these are conditionally executed blocks -- if the method is going - // to unconditionally throw, there's not as much to be gained by deferring jitting. - // For now, we just screen out the entry bb. - // - // In general we might want track all the IL stack empty points so we can - // propagate rareness back through flow and place the partial compilation patchpoints "earlier" - // so there are fewer overall. - // - // Note unlike OSR, it's ok to forgo these. - // - // Todo: stress mode... - // - if (opts.jitFlags->IsSet(JitFlags::JIT_FLAG_TIER0) && (JitConfig.TC_PartialCompilation() > 0) && - compCanHavePatchpoints() && !compTailPrefixSeen) - { - // Is this block a good place for partial compilation? - // - if ((block != fgFirstBB) && block->isRunRarely() && (verCurrentState.esStackDepth == 0) && - ((block->bbFlags & BBF_PATCHPOINT) == 0) && !block->hasHndIndex()) - { - JITDUMP("\nBlock " FMT_BB " will be a partial compilation patchpoint -- not importing\n", block->bbNum); - block->bbFlags |= BBF_PARTIAL_COMPILATION_PATCHPOINT; - setMethodHasPartialCompilationPatchpoint(); - - // Change block to BBJ_THROW so we won't trigger importation of successors. - // - block->bbJumpKind = BBJ_THROW; - - // If this method has a explicit generic context, the only uses of it may be in - // the IL for this block. So assume it's used. - // - if (info.compMethodInfo->options & - (CORINFO_GENERICS_CTXT_FROM_METHODDESC | CORINFO_GENERICS_CTXT_FROM_METHODTABLE)) - { - lvaGenericsContextInUse = true; - } - - return; - } - } - -#endif // FEATURE_ON_STACK_REPLACEMENT - - /* Walk the opcodes that comprise the basic block */ - - const BYTE* codeAddr = info.compCode + block->bbCodeOffs; - const BYTE* codeEndp = info.compCode + block->bbCodeOffsEnd; - - IL_OFFSET opcodeOffs = block->bbCodeOffs; - IL_OFFSET lastSpillOffs = opcodeOffs; - - signed jmpDist; - - /* remember the start of the delegate creation sequence (used for verification) */ - const BYTE* delegateCreateStart = nullptr; - - int prefixFlags = 0; - bool explicitTailCall, constraintCall, readonlyCall; - - typeInfo tiRetVal; - - unsigned numArgs = info.compArgsCount; - - /* Now process all the opcodes in the block */ - - var_types callTyp = TYP_COUNT; - OPCODE prevOpcode = CEE_ILLEGAL; - - if (block->bbCatchTyp) - { - if (info.compStmtOffsetsImplicit & ICorDebugInfo::CALL_SITE_BOUNDARIES) - { - impCurStmtOffsSet(block->bbCodeOffs); - } - - // We will spill the GT_CATCH_ARG and the input of the BB_QMARK block - // to a temp. This is a trade off for code simplicity - impSpillSpecialSideEff(); - } - - while (codeAddr < codeEndp) - { -#ifdef FEATURE_READYTORUN - bool usingReadyToRunHelper = false; -#endif - CORINFO_RESOLVED_TOKEN resolvedToken; - CORINFO_RESOLVED_TOKEN constrainedResolvedToken = {}; - CORINFO_CALL_INFO callInfo; - CORINFO_FIELD_INFO fieldInfo; - - tiRetVal = typeInfo(); // Default type info - - //--------------------------------------------------------------------- - - /* We need to restrict the max tree depth as many of the Compiler - functions are recursive. We do this by spilling the stack */ - - if (verCurrentState.esStackDepth) - { - /* Has it been a while since we last saw a non-empty stack (which - guarantees that the tree depth isnt accumulating. */ - - if ((opcodeOffs - lastSpillOffs) > MAX_TREE_SIZE && impCanSpillNow(prevOpcode)) - { - impSpillStackEnsure(); - lastSpillOffs = opcodeOffs; - } - } - else - { - lastSpillOffs = opcodeOffs; - impBoxTempInUse = false; // nothing on the stack, box temp OK to use again - } - - /* Compute the current instr offset */ - - opcodeOffs = (IL_OFFSET)(codeAddr - info.compCode); - -#ifndef DEBUG - if (opts.compDbgInfo) -#endif - { - nxtStmtOffs = - (nxtStmtIndex < info.compStmtOffsetsCount) ? info.compStmtOffsets[nxtStmtIndex] : BAD_IL_OFFSET; - - /* Have we reached the next stmt boundary ? */ - - if (nxtStmtOffs != BAD_IL_OFFSET && opcodeOffs >= nxtStmtOffs) - { - assert(nxtStmtOffs == info.compStmtOffsets[nxtStmtIndex]); - - if (verCurrentState.esStackDepth != 0 && opts.compDbgCode) - { - /* We need to provide accurate IP-mapping at this point. - So spill anything on the stack so that it will form - gtStmts with the correct stmt offset noted */ - - impSpillStackEnsure(true); - } - - // Have we reported debug info for any tree? - - if (impCurStmtDI.IsValid() && opts.compDbgCode) - { - GenTree* placeHolder = new (this, GT_NO_OP) GenTree(GT_NO_OP, TYP_VOID); - impAppendTree(placeHolder, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); - - assert(!impCurStmtDI.IsValid()); - } - - if (!impCurStmtDI.IsValid()) - { - /* Make sure that nxtStmtIndex is in sync with opcodeOffs. - If opcodeOffs has gone past nxtStmtIndex, catch up */ - - while ((nxtStmtIndex + 1) < info.compStmtOffsetsCount && - info.compStmtOffsets[nxtStmtIndex + 1] <= opcodeOffs) - { - nxtStmtIndex++; - } - - /* Go to the new stmt */ - - impCurStmtOffsSet(info.compStmtOffsets[nxtStmtIndex]); - - /* Update the stmt boundary index */ - - nxtStmtIndex++; - assert(nxtStmtIndex <= info.compStmtOffsetsCount); - - /* Are there any more line# entries after this one? */ - - if (nxtStmtIndex < info.compStmtOffsetsCount) - { - /* Remember where the next line# starts */ - - nxtStmtOffs = info.compStmtOffsets[nxtStmtIndex]; - } - else - { - /* No more line# entries */ - - nxtStmtOffs = BAD_IL_OFFSET; - } - } - } - else if ((info.compStmtOffsetsImplicit & ICorDebugInfo::STACK_EMPTY_BOUNDARIES) && - (verCurrentState.esStackDepth == 0)) - { - /* At stack-empty locations, we have already added the tree to - the stmt list with the last offset. We just need to update - impCurStmtDI - */ - - impCurStmtOffsSet(opcodeOffs); - } - else if ((info.compStmtOffsetsImplicit & ICorDebugInfo::CALL_SITE_BOUNDARIES) && - impOpcodeIsCallSiteBoundary(prevOpcode)) - { - /* Make sure we have a type cached */ - assert(callTyp != TYP_COUNT); - - if (callTyp == TYP_VOID) - { - impCurStmtOffsSet(opcodeOffs); - } - else if (opts.compDbgCode) - { - impSpillStackEnsure(true); - impCurStmtOffsSet(opcodeOffs); - } - } - else if ((info.compStmtOffsetsImplicit & ICorDebugInfo::NOP_BOUNDARIES) && (prevOpcode == CEE_NOP)) - { - if (opts.compDbgCode) - { - impSpillStackEnsure(true); - } - - impCurStmtOffsSet(opcodeOffs); - } - - assert(!impCurStmtDI.IsValid() || (nxtStmtOffs == BAD_IL_OFFSET) || - (impCurStmtDI.GetLocation().GetOffset() <= nxtStmtOffs)); - } - - CORINFO_CLASS_HANDLE clsHnd = DUMMY_INIT(NULL); - CORINFO_CLASS_HANDLE ldelemClsHnd = NO_CLASS_HANDLE; - CORINFO_CLASS_HANDLE stelemClsHnd = DUMMY_INIT(NULL); - - var_types lclTyp, ovflType = TYP_UNKNOWN; - GenTree* op1 = DUMMY_INIT(NULL); - GenTree* op2 = DUMMY_INIT(NULL); - GenTree* newObjThisPtr = DUMMY_INIT(NULL); - bool uns = DUMMY_INIT(false); - bool isLocal = false; - - /* Get the next opcode and the size of its parameters */ - OPCODE opcode = (OPCODE)getU1LittleEndian(codeAddr); - codeAddr += sizeof(__int8); + // The backend does not support other struct-producing nodes (e. g. OBJs) as sources of multi-reg returns. + // It also does not support assembling a multi-reg node into one register (for RETURN nodes at least). + return impAssignMultiRegTypeToVar(op, info.compMethodInfo->args.retTypeClass DEBUGARG(info.compCallConv)); + } -#ifdef DEBUG - impCurOpcOffs = (IL_OFFSET)(codeAddr - info.compCode - 1); - JITDUMP("\n [%2u] %3u (0x%03x) ", verCurrentState.esStackDepth, impCurOpcOffs, impCurOpcOffs); -#endif + // Not a multi-reg return or value, we can simply use it directly. + return op; +} - DECODE_OPCODE: +/***************************************************************************** + CEE_LEAVE may be jumping out of a protected block, viz, a catch or a + finally-protected try. We find the finally blocks protecting the current + offset (in order) by walking over the complete exception table and + finding enclosing clauses. This assumes that the table is sorted. + This will create a series of BBJ_CALLFINALLY -> BBJ_CALLFINALLY ... -> BBJ_ALWAYS. - // Return if any previous code has caused inline to fail. - if (compDonotInline()) - { - return; - } + If we are leaving a catch handler, we need to attach the + CPX_ENDCATCHes to the correct BBJ_CALLFINALLY blocks. - /* Get the size of additional parameters */ + After this function, the BBJ_LEAVE block has been converted to a different type. + */ - signed int sz = opcodeSizes[opcode]; +#if !defined(FEATURE_EH_FUNCLETS) +void Compiler::impImportLeave(BasicBlock* block) +{ #ifdef DEBUG - clsHnd = NO_CLASS_HANDLE; - lclTyp = TYP_COUNT; - callTyp = TYP_COUNT; - - impCurOpcOffs = (IL_OFFSET)(codeAddr - info.compCode - 1); - impCurOpcName = opcodeNames[opcode]; - - if (verbose && (opcode != CEE_PREFIX1)) - { - printf("%s", impCurOpcName); - } - - /* Use assertImp() to display the opcode */ - - op1 = op2 = nullptr; -#endif - - /* See what kind of an opcode we have, then */ - - unsigned mflags = 0; - unsigned clsFlags = 0; - - switch (opcode) - { - unsigned lclNum; - var_types type; - - GenTree* op3; - genTreeOps oper; - unsigned size; - - int val; + if (verbose) + { + printf("\nBefore import CEE_LEAVE:\n"); + fgDispBasicBlocks(); + fgDispHandlerTab(); + } +#endif // DEBUG - CORINFO_SIG_INFO sig; - IL_OFFSET jmpAddr; - bool ovfl, unordered, callNode; - CORINFO_CLASS_HANDLE tokenType; + bool invalidatePreds = false; // If we create new blocks, invalidate the predecessor lists (if created) + unsigned blkAddr = block->bbCodeOffs; + BasicBlock* leaveTarget = block->bbJumpDest; + unsigned jmpAddr = leaveTarget->bbCodeOffs; - union { - int intVal; - float fltVal; - __int64 lngVal; - double dblVal; - } cval; + // LEAVE clears the stack, spill side effects, and set stack to 0 - case CEE_PREFIX1: - opcode = (OPCODE)(getU1LittleEndian(codeAddr) + 256); - opcodeOffs = (IL_OFFSET)(codeAddr - info.compCode); - codeAddr += sizeof(__int8); - goto DECODE_OPCODE; + impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("impImportLeave")); + verCurrentState.esStackDepth = 0; - SPILL_APPEND: + assert(block->bbJumpKind == BBJ_LEAVE); + assert(fgBBs == (BasicBlock**)0xCDCD || fgLookupBB(jmpAddr) != NULL); // should be a BB boundary - // We need to call impSpillLclRefs() for a struct type lclVar. - // This is because there may be loads of that lclVar on the evaluation stack, and - // we need to ensure that those loads are completed before we modify it. - if ((op1->OperGet() == GT_ASG) && varTypeIsStruct(op1->gtGetOp1())) - { - GenTree* lhs = op1->gtGetOp1(); - GenTreeLclVarCommon* lclVar = nullptr; - if (lhs->gtOper == GT_LCL_VAR) - { - lclVar = lhs->AsLclVarCommon(); - } - else if (lhs->OperIsBlk()) - { - // Check if LHS address is within some struct local, to catch - // cases where we're updating the struct by something other than a stfld - GenTree* addr = lhs->AsBlk()->Addr(); + BasicBlock* step = DUMMY_INIT(NULL); + unsigned encFinallies = 0; // Number of enclosing finallies. + GenTree* endCatches = NULL; + Statement* endLFinStmt = NULL; // The statement tree to indicate the end of locally-invoked finally. - // Catches ADDR(LCL_VAR), or ADD(ADDR(LCL_VAR),CNS_INT)) - lclVar = addr->IsLocalAddrExpr(); + unsigned XTnum; + EHblkDsc* HBtab; - // Catches ADDR(FIELD(... ADDR(LCL_VAR))) - if (lclVar == nullptr) - { - GenTree* lclTree = nullptr; - if (impIsAddressInLocal(addr, &lclTree)) - { - lclVar = lclTree->AsLclVarCommon(); - } - } - } - if (lclVar != nullptr) - { - impSpillLclRefs(lclVar->GetLclNum()); - } - } + for (XTnum = 0, HBtab = compHndBBtab; XTnum < compHndBBtabCount; XTnum++, HBtab++) + { + // Grab the handler offsets - /* Append 'op1' to the list of statements */ - impAppendTree(op1, (unsigned)CHECK_SPILL_ALL, impCurStmtDI); - goto DONE_APPEND; + IL_OFFSET tryBeg = HBtab->ebdTryBegOffs(); + IL_OFFSET tryEnd = HBtab->ebdTryEndOffs(); + IL_OFFSET hndBeg = HBtab->ebdHndBegOffs(); + IL_OFFSET hndEnd = HBtab->ebdHndEndOffs(); - APPEND: + /* Is this a catch-handler we are CEE_LEAVEing out of? + * If so, we need to call CORINFO_HELP_ENDCATCH. + */ - /* Append 'op1' to the list of statements */ + if (jitIsBetween(blkAddr, hndBeg, hndEnd) && !jitIsBetween(jmpAddr, hndBeg, hndEnd)) + { + // Can't CEE_LEAVE out of a finally/fault handler + if (HBtab->HasFinallyOrFaultHandler()) + BADCODE("leave out of fault/finally block"); - impAppendTree(op1, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); - goto DONE_APPEND; + // Create the call to CORINFO_HELP_ENDCATCH + GenTree* endCatch = gtNewHelperCallNode(CORINFO_HELP_ENDCATCH, TYP_VOID); - DONE_APPEND: + // Make a list of all the currently pending endCatches + if (endCatches) + endCatches = gtNewOperNode(GT_COMMA, TYP_VOID, endCatches, endCatch); + else + endCatches = endCatch; #ifdef DEBUG - // Remember at which BC offset the tree was finished - impNoteLastILoffs(); + if (verbose) + { + printf("impImportLeave - " FMT_BB " jumping out of catch handler EH#%u, adding call to " + "CORINFO_HELP_ENDCATCH\n", + block->bbNum, XTnum); + } #endif - break; - - case CEE_LDNULL: - impPushNullObjRefOnStack(); - break; - - case CEE_LDC_I4_M1: - case CEE_LDC_I4_0: - case CEE_LDC_I4_1: - case CEE_LDC_I4_2: - case CEE_LDC_I4_3: - case CEE_LDC_I4_4: - case CEE_LDC_I4_5: - case CEE_LDC_I4_6: - case CEE_LDC_I4_7: - case CEE_LDC_I4_8: - cval.intVal = (opcode - CEE_LDC_I4_0); - assert(-1 <= cval.intVal && cval.intVal <= 8); - goto PUSH_I4CON; - - case CEE_LDC_I4_S: - cval.intVal = getI1LittleEndian(codeAddr); - goto PUSH_I4CON; - case CEE_LDC_I4: - cval.intVal = getI4LittleEndian(codeAddr); - goto PUSH_I4CON; - PUSH_I4CON: - JITDUMP(" %d", cval.intVal); - impPushOnStack(gtNewIconNode(cval.intVal), typeInfo(TI_INT)); - break; - - case CEE_LDC_I8: - cval.lngVal = getI8LittleEndian(codeAddr); - JITDUMP(" 0x%016llx", cval.lngVal); - impPushOnStack(gtNewLconNode(cval.lngVal), typeInfo(TI_LONG)); - break; - - case CEE_LDC_R8: - cval.dblVal = getR8LittleEndian(codeAddr); - JITDUMP(" %#.17g", cval.dblVal); - impPushOnStack(gtNewDconNode(cval.dblVal), typeInfo(TI_DOUBLE)); - break; - - case CEE_LDC_R4: - cval.dblVal = getR4LittleEndian(codeAddr); - JITDUMP(" %#.17g", cval.dblVal); - impPushOnStack(gtNewDconNode(cval.dblVal, TYP_FLOAT), typeInfo(TI_DOUBLE)); - break; - - case CEE_LDSTR: - val = getU4LittleEndian(codeAddr); - JITDUMP(" %08X", val); - impPushOnStack(gtNewSconNode(val, info.compScopeHnd), tiRetVal); - break; - - case CEE_LDARG: - lclNum = getU2LittleEndian(codeAddr); - JITDUMP(" %u", lclNum); - impLoadArg(lclNum, opcodeOffs + sz + 1); - break; - - case CEE_LDARG_S: - lclNum = getU1LittleEndian(codeAddr); - JITDUMP(" %u", lclNum); - impLoadArg(lclNum, opcodeOffs + sz + 1); - break; - - case CEE_LDARG_0: - case CEE_LDARG_1: - case CEE_LDARG_2: - case CEE_LDARG_3: - lclNum = (opcode - CEE_LDARG_0); - assert(lclNum >= 0 && lclNum < 4); - impLoadArg(lclNum, opcodeOffs + sz + 1); - break; + } + else if (HBtab->HasFinallyHandler() && jitIsBetween(blkAddr, tryBeg, tryEnd) && + !jitIsBetween(jmpAddr, tryBeg, tryEnd)) + { + /* This is a finally-protected try we are jumping out of */ - case CEE_LDLOC: - lclNum = getU2LittleEndian(codeAddr); - JITDUMP(" %u", lclNum); - impLoadLoc(lclNum, opcodeOffs + sz + 1); - break; + /* If there are any pending endCatches, and we have already + jumped out of a finally-protected try, then the endCatches + have to be put in a block in an outer try for async + exceptions to work correctly. + Else, just use append to the original block */ - case CEE_LDLOC_S: - lclNum = getU1LittleEndian(codeAddr); - JITDUMP(" %u", lclNum); - impLoadLoc(lclNum, opcodeOffs + sz + 1); - break; + BasicBlock* callBlock; - case CEE_LDLOC_0: - case CEE_LDLOC_1: - case CEE_LDLOC_2: - case CEE_LDLOC_3: - lclNum = (opcode - CEE_LDLOC_0); - assert(lclNum >= 0 && lclNum < 4); - impLoadLoc(lclNum, opcodeOffs + sz + 1); - break; + assert(!encFinallies == + !endLFinStmt); // if we have finallies, we better have an endLFin tree, and vice-versa - case CEE_STARG: - lclNum = getU2LittleEndian(codeAddr); - goto STARG; + if (encFinallies == 0) + { + assert(step == DUMMY_INIT(NULL)); + callBlock = block; + callBlock->bbJumpKind = BBJ_CALLFINALLY; // convert the BBJ_LEAVE to BBJ_CALLFINALLY - case CEE_STARG_S: - lclNum = getU1LittleEndian(codeAddr); - STARG: - JITDUMP(" %u", lclNum); + if (endCatches) + impAppendTree(endCatches, CHECK_SPILL_NONE, impCurStmtDI); - if (compIsForInlining()) +#ifdef DEBUG + if (verbose) { - op1 = impInlineFetchArg(lclNum, impInlineInfo->inlArgInfo, impInlineInfo->lclVarInfo); - noway_assert(op1->gtOper == GT_LCL_VAR); - lclNum = op1->AsLclVar()->GetLclNum(); + printf("impImportLeave - jumping out of a finally-protected try, convert block to BBJ_CALLFINALLY " + "block %s\n", + callBlock->dspToString()); + } +#endif + } + else + { + assert(step != DUMMY_INIT(NULL)); - goto VAR_ST_VALID; + /* Calling the finally block */ + callBlock = fgNewBBinRegion(BBJ_CALLFINALLY, XTnum + 1, 0, step); + assert(step->bbJumpKind == BBJ_ALWAYS); + step->bbJumpDest = callBlock; // the previous call to a finally returns to this call (to the next + // finally in the chain) + step->bbJumpDest->bbRefs++; + + /* The new block will inherit this block's weight */ + callBlock->inheritWeight(block); + +#ifdef DEBUG + if (verbose) + { + printf("impImportLeave - jumping out of a finally-protected try, new BBJ_CALLFINALLY block %s\n", + callBlock->dspToString()); } +#endif - lclNum = compMapILargNum(lclNum); // account for possible hidden param - assertImp(lclNum < numArgs); + Statement* lastStmt; - if (lclNum == info.compThisArg) + if (endCatches) { - lclNum = lvaArg0Var; + lastStmt = gtNewStmt(endCatches); + endLFinStmt->SetNextStmt(lastStmt); + lastStmt->SetPrevStmt(endLFinStmt); + } + else + { + lastStmt = endLFinStmt; } - // We should have seen this arg write in the prescan - assert(lvaTable[lclNum].lvHasILStoreOp); + // note that this sets BBF_IMPORTED on the block + impEndTreeList(callBlock, endLFinStmt, lastStmt); + } - goto VAR_ST; + step = fgNewBBafter(BBJ_ALWAYS, callBlock, true); + /* The new block will inherit this block's weight */ + step->inheritWeight(block); + step->bbFlags |= BBF_IMPORTED | BBF_KEEP_BBJ_ALWAYS; - case CEE_STLOC: - lclNum = getU2LittleEndian(codeAddr); - isLocal = true; - JITDUMP(" %u", lclNum); - goto LOC_ST; +#ifdef DEBUG + if (verbose) + { + printf("impImportLeave - jumping out of a finally-protected try, created step (BBJ_ALWAYS) block %s\n", + step->dspToString()); + } +#endif - case CEE_STLOC_S: - lclNum = getU1LittleEndian(codeAddr); - isLocal = true; - JITDUMP(" %u", lclNum); - goto LOC_ST; + unsigned finallyNesting = compHndBBtab[XTnum].ebdHandlerNestingLevel; + assert(finallyNesting <= compHndBBtabCount); - case CEE_STLOC_0: - case CEE_STLOC_1: - case CEE_STLOC_2: - case CEE_STLOC_3: - isLocal = true; - lclNum = (opcode - CEE_STLOC_0); - assert(lclNum >= 0 && lclNum < 4); + callBlock->bbJumpDest = HBtab->ebdHndBeg; // This callBlock will call the "finally" handler. + GenTree* endLFin = new (this, GT_END_LFIN) GenTreeVal(GT_END_LFIN, TYP_VOID, finallyNesting); + endLFinStmt = gtNewStmt(endLFin); + endCatches = NULL; - LOC_ST: - if (compIsForInlining()) - { - lclTyp = impInlineInfo->lclVarInfo[lclNum + impInlineInfo->argCnt].lclTypeInfo; + encFinallies++; - /* Have we allocated a temp for this local? */ + invalidatePreds = true; + } + } - lclNum = impInlineFetchLocal(lclNum DEBUGARG("Inline stloc first use temp")); + /* Append any remaining endCatches, if any */ - goto _PopValue; - } + assert(!encFinallies == !endLFinStmt); - lclNum += numArgs; + if (encFinallies == 0) + { + assert(step == DUMMY_INIT(NULL)); + block->bbJumpKind = BBJ_ALWAYS; // convert the BBJ_LEAVE to a BBJ_ALWAYS - VAR_ST: + if (endCatches) + impAppendTree(endCatches, CHECK_SPILL_NONE, impCurStmtDI); - if (lclNum >= info.compLocalsCount && lclNum != lvaArg0Var) - { - BADCODE("Bad IL"); - } +#ifdef DEBUG + if (verbose) + { + printf("impImportLeave - no enclosing finally-protected try blocks; convert CEE_LEAVE block to BBJ_ALWAYS " + "block %s\n", + block->dspToString()); + } +#endif + } + else + { + // If leaveTarget is the start of another try block, we want to make sure that + // we do not insert finalStep into that try block. Hence, we find the enclosing + // try block. + unsigned tryIndex = bbFindInnermostCommonTryRegion(step, leaveTarget); - VAR_ST_VALID: + // Insert a new BB either in the try region indicated by tryIndex or + // the handler region indicated by leaveTarget->bbHndIndex, + // depending on which is the inner region. + BasicBlock* finalStep = fgNewBBinRegion(BBJ_ALWAYS, tryIndex, leaveTarget->bbHndIndex, step); + finalStep->bbFlags |= BBF_KEEP_BBJ_ALWAYS; + step->bbJumpDest = finalStep; - /* if it is a struct assignment, make certain we don't overflow the buffer */ - assert(lclTyp != TYP_STRUCT || lvaLclSize(lclNum) >= info.compCompHnd->getClassSize(clsHnd)); + /* The new block will inherit this block's weight */ + finalStep->inheritWeight(block); - if (lvaTable[lclNum].lvNormalizeOnLoad()) - { - lclTyp = lvaGetRealType(lclNum); - } - else - { - lclTyp = lvaGetActualType(lclNum); - } +#ifdef DEBUG + if (verbose) + { + printf("impImportLeave - finalStep block required (encFinallies(%d) > 0), new block %s\n", encFinallies, + finalStep->dspToString()); + } +#endif - _PopValue: - /* Pop the value being assigned */ + Statement* lastStmt; - { - StackEntry se = impPopStack(); - clsHnd = se.seTypeInfo.GetClassHandle(); - op1 = se.val; - tiRetVal = se.seTypeInfo; - } + if (endCatches) + { + lastStmt = gtNewStmt(endCatches); + endLFinStmt->SetNextStmt(lastStmt); + lastStmt->SetPrevStmt(endLFinStmt); + } + else + { + lastStmt = endLFinStmt; + } -#ifdef FEATURE_SIMD - if (varTypeIsSIMD(lclTyp) && (lclTyp != op1->TypeGet())) - { - assert(op1->TypeGet() == TYP_STRUCT); - op1->gtType = lclTyp; - } -#endif // FEATURE_SIMD + impEndTreeList(finalStep, endLFinStmt, lastStmt); - op1 = impImplicitIorI4Cast(op1, lclTyp); + finalStep->bbJumpDest = leaveTarget; // this is the ultimate destination of the LEAVE -#ifdef TARGET_64BIT - // Downcast the TYP_I_IMPL into a 32-bit Int for x86 JIT compatiblity - if (varTypeIsI(op1->TypeGet()) && (genActualType(lclTyp) == TYP_INT)) - { - op1 = gtNewCastNode(TYP_INT, op1, false, TYP_INT); - } -#endif // TARGET_64BIT + // Queue up the jump target for importing - // We had better assign it a value of the correct type - assertImp( - genActualType(lclTyp) == genActualType(op1->gtType) || - (genActualType(lclTyp) == TYP_I_IMPL && op1->IsLocalAddrExpr() != nullptr) || - (genActualType(lclTyp) == TYP_I_IMPL && (op1->gtType == TYP_BYREF || op1->gtType == TYP_REF)) || - (genActualType(op1->gtType) == TYP_I_IMPL && lclTyp == TYP_BYREF) || - (varTypeIsFloating(lclTyp) && varTypeIsFloating(op1->TypeGet())) || - ((genActualType(lclTyp) == TYP_BYREF) && genActualType(op1->TypeGet()) == TYP_REF)); + impImportBlockPending(leaveTarget); + + invalidatePreds = true; + } + + if (invalidatePreds && fgComputePredsDone) + { + JITDUMP("\n**** impImportLeave - Removing preds after creating new blocks\n"); + fgRemovePreds(); + } + +#ifdef DEBUG + fgVerifyHandlerTab(); + + if (verbose) + { + printf("\nAfter import CEE_LEAVE:\n"); + fgDispBasicBlocks(); + fgDispHandlerTab(); + } +#endif // DEBUG +} + +#else // FEATURE_EH_FUNCLETS + +void Compiler::impImportLeave(BasicBlock* block) +{ +#ifdef DEBUG + if (verbose) + { + printf("\nBefore import CEE_LEAVE in " FMT_BB " (targeting " FMT_BB "):\n", block->bbNum, + block->bbJumpDest->bbNum); + fgDispBasicBlocks(); + fgDispHandlerTab(); + } +#endif // DEBUG + + bool invalidatePreds = false; // If we create new blocks, invalidate the predecessor lists (if created) + unsigned blkAddr = block->bbCodeOffs; + BasicBlock* leaveTarget = block->bbJumpDest; + unsigned jmpAddr = leaveTarget->bbCodeOffs; - /* If op1 is "&var" then its type is the transient "*" and it can - be used either as TYP_BYREF or TYP_I_IMPL */ + // LEAVE clears the stack, spill side effects, and set stack to 0 - if (op1->IsLocalAddrExpr() != nullptr) - { - assertImp(genActualType(lclTyp) == TYP_I_IMPL || lclTyp == TYP_BYREF); + impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("impImportLeave")); + verCurrentState.esStackDepth = 0; - /* When "&var" is created, we assume it is a byref. If it is - being assigned to a TYP_I_IMPL var, change the type to - prevent unnecessary GC info */ + assert(block->bbJumpKind == BBJ_LEAVE); + assert(fgBBs == (BasicBlock**)0xCDCD || fgLookupBB(jmpAddr) != nullptr); // should be a BB boundary - if (genActualType(lclTyp) == TYP_I_IMPL) - { - op1->gtType = TYP_I_IMPL; - } - } + BasicBlock* step = nullptr; - // If this is a local and the local is a ref type, see - // if we can improve type information based on the - // value being assigned. - if (isLocal && (lclTyp == TYP_REF)) - { - // We should have seen a stloc in our IL prescan. - assert(lvaTable[lclNum].lvHasILStoreOp); + enum StepType + { + // No step type; step == NULL. + ST_None, - // Is there just one place this local is defined? - const bool isSingleDefLocal = lvaTable[lclNum].lvSingleDef; + // Is the step block the BBJ_ALWAYS block of a BBJ_CALLFINALLY/BBJ_ALWAYS pair? + // That is, is step->bbJumpDest where a finally will return to? + ST_FinallyReturn, - // Conservative check that there is just one - // definition that reaches this store. - const bool hasSingleReachingDef = (block->bbStackDepthOnEntry() == 0); + // The step block is a catch return. + ST_Catch, - if (isSingleDefLocal && hasSingleReachingDef) - { - lvaUpdateClass(lclNum, op1, clsHnd); - } - } + // The step block is in a "try", created as the target for a finally return or the target for a catch return. + ST_Try + }; + StepType stepType = ST_None; - /* Filter out simple assignments to itself */ + unsigned XTnum; + EHblkDsc* HBtab; - if (op1->gtOper == GT_LCL_VAR && lclNum == op1->AsLclVarCommon()->GetLclNum()) - { - if (opts.compDbgCode) - { - op1 = gtNewNothingNode(); - goto SPILL_APPEND; - } - else - { - break; - } - } + for (XTnum = 0, HBtab = compHndBBtab; XTnum < compHndBBtabCount; XTnum++, HBtab++) + { + // Grab the handler offsets + + IL_OFFSET tryBeg = HBtab->ebdTryBegOffs(); + IL_OFFSET tryEnd = HBtab->ebdTryEndOffs(); + IL_OFFSET hndBeg = HBtab->ebdHndBegOffs(); + IL_OFFSET hndEnd = HBtab->ebdHndEndOffs(); - /* Create the assignment node */ + /* Is this a catch-handler we are CEE_LEAVEing out of? + */ - op2 = gtNewLclvNode(lclNum, lclTyp DEBUGARG(opcodeOffs + sz + 1)); + if (jitIsBetween(blkAddr, hndBeg, hndEnd) && !jitIsBetween(jmpAddr, hndBeg, hndEnd)) + { + // Can't CEE_LEAVE out of a finally/fault handler + if (HBtab->HasFinallyOrFaultHandler()) + { + BADCODE("leave out of fault/finally block"); + } + + /* We are jumping out of a catch */ - /* If the local is aliased or pinned, we need to spill calls and - indirections from the stack. */ + if (step == nullptr) + { + step = block; + step->bbJumpKind = BBJ_EHCATCHRET; // convert the BBJ_LEAVE to BBJ_EHCATCHRET + stepType = ST_Catch; - if ((lvaTable[lclNum].IsAddressExposed() || lvaTable[lclNum].lvHasLdAddrOp || - lvaTable[lclNum].lvPinned) && - (verCurrentState.esStackDepth > 0)) +#ifdef DEBUG + if (verbose) { - impSpillSideEffects(false, - (unsigned)CHECK_SPILL_ALL DEBUGARG("Local could be aliased or is pinned")); + printf("impImportLeave - jumping out of a catch (EH#%u), convert block " FMT_BB + " to BBJ_EHCATCHRET " + "block\n", + XTnum, step->bbNum); } +#endif + } + else + { + BasicBlock* exitBlock; - /* Spill any refs to the local from the stack */ + /* Create a new catch exit block in the catch region for the existing step block to jump to in this + * scope */ + exitBlock = fgNewBBinRegion(BBJ_EHCATCHRET, 0, XTnum + 1, step); - impSpillLclRefs(lclNum); + assert(step->KindIs(BBJ_ALWAYS, BBJ_EHCATCHRET)); + step->bbJumpDest = exitBlock; // the previous step (maybe a call to a nested finally, or a nested catch + // exit) returns to this block + step->bbJumpDest->bbRefs++; - // We can generate an assignment to a TYP_FLOAT from a TYP_DOUBLE - // We insert a cast to the dest 'op2' type - // - if ((op1->TypeGet() != op2->TypeGet()) && varTypeIsFloating(op1->gtType) && - varTypeIsFloating(op2->gtType)) +#if defined(TARGET_ARM) + if (stepType == ST_FinallyReturn) { - op1 = gtNewCastNode(op2->TypeGet(), op1, false, op2->TypeGet()); + assert(step->bbJumpKind == BBJ_ALWAYS); + // Mark the target of a finally return + step->bbJumpDest->bbFlags |= BBF_FINALLY_TARGET; } +#endif // defined(TARGET_ARM) - if (varTypeIsStruct(lclTyp)) - { - op1 = impAssignStruct(op2, op1, clsHnd, (unsigned)CHECK_SPILL_ALL); - } - else + /* The new block will inherit this block's weight */ + exitBlock->inheritWeight(block); + exitBlock->bbFlags |= BBF_IMPORTED; + + /* This exit block is the new step */ + step = exitBlock; + stepType = ST_Catch; + + invalidatePreds = true; + +#ifdef DEBUG + if (verbose) { - op1 = gtNewAssignNode(op2, op1); + printf("impImportLeave - jumping out of a catch (EH#%u), new BBJ_EHCATCHRET block " FMT_BB "\n", + XTnum, exitBlock->bbNum); } +#endif + } + } + else if (HBtab->HasFinallyHandler() && jitIsBetween(blkAddr, tryBeg, tryEnd) && + !jitIsBetween(jmpAddr, tryBeg, tryEnd)) + { + /* We are jumping out of a finally-protected try */ - goto SPILL_APPEND; + BasicBlock* callBlock; - case CEE_LDLOCA: - lclNum = getU2LittleEndian(codeAddr); - goto LDLOCA; + if (step == nullptr) + { +#if FEATURE_EH_CALLFINALLY_THUNKS - case CEE_LDLOCA_S: - lclNum = getU1LittleEndian(codeAddr); - LDLOCA: - JITDUMP(" %u", lclNum); + // Put the call to the finally in the enclosing region. + unsigned callFinallyTryIndex = + (HBtab->ebdEnclosingTryIndex == EHblkDsc::NO_ENCLOSING_INDEX) ? 0 : HBtab->ebdEnclosingTryIndex + 1; + unsigned callFinallyHndIndex = + (HBtab->ebdEnclosingHndIndex == EHblkDsc::NO_ENCLOSING_INDEX) ? 0 : HBtab->ebdEnclosingHndIndex + 1; + callBlock = fgNewBBinRegion(BBJ_CALLFINALLY, callFinallyTryIndex, callFinallyHndIndex, block); - if (compIsForInlining()) - { - // Get the local type - lclTyp = impInlineInfo->lclVarInfo[lclNum + impInlineInfo->argCnt].lclTypeInfo; + // Convert the BBJ_LEAVE to BBJ_ALWAYS, jumping to the new BBJ_CALLFINALLY. This is because + // the new BBJ_CALLFINALLY is in a different EH region, thus it can't just replace the BBJ_LEAVE, + // which might be in the middle of the "try". In most cases, the BBJ_ALWAYS will jump to the + // next block, and flow optimizations will remove it. + block->bbJumpKind = BBJ_ALWAYS; + block->bbJumpDest = callBlock; + block->bbJumpDest->bbRefs++; - /* Have we allocated a temp for this local? */ + /* The new block will inherit this block's weight */ + callBlock->inheritWeight(block); + callBlock->bbFlags |= BBF_IMPORTED; - lclNum = impInlineFetchLocal(lclNum DEBUGARG("Inline ldloca(s) first use temp")); +#ifdef DEBUG + if (verbose) + { + printf("impImportLeave - jumping out of a finally-protected try (EH#%u), convert block " FMT_BB + " to " + "BBJ_ALWAYS, add BBJ_CALLFINALLY block " FMT_BB "\n", + XTnum, block->bbNum, callBlock->bbNum); + } +#endif - assert(!lvaGetDesc(lclNum)->lvNormalizeOnLoad()); - op1 = gtNewLclvNode(lclNum, lvaGetActualType(lclNum)); +#else // !FEATURE_EH_CALLFINALLY_THUNKS - goto _PUSH_ADRVAR; - } + callBlock = block; + callBlock->bbJumpKind = BBJ_CALLFINALLY; // convert the BBJ_LEAVE to BBJ_CALLFINALLY - lclNum += numArgs; - assertImp(lclNum < info.compLocalsCount); - goto ADRVAR; +#ifdef DEBUG + if (verbose) + { + printf("impImportLeave - jumping out of a finally-protected try (EH#%u), convert block " FMT_BB + " to " + "BBJ_CALLFINALLY block\n", + XTnum, callBlock->bbNum); + } +#endif - case CEE_LDARGA: - lclNum = getU2LittleEndian(codeAddr); - goto LDARGA; +#endif // !FEATURE_EH_CALLFINALLY_THUNKS + } + else + { + // Calling the finally block. We already have a step block that is either the call-to-finally from a + // more nested try/finally (thus we are jumping out of multiple nested 'try' blocks, each protected by + // a 'finally'), or the step block is the return from a catch. + // + // Due to ThreadAbortException, we can't have the catch return target the call-to-finally block + // directly. Note that if a 'catch' ends without resetting the ThreadAbortException, the VM will + // automatically re-raise the exception, using the return address of the catch (that is, the target + // block of the BBJ_EHCATCHRET) as the re-raise address. If this address is in a finally, the VM will + // refuse to do the re-raise, and the ThreadAbortException will get eaten (and lost). On AMD64/ARM64, + // we put the call-to-finally thunk in a special "cloned finally" EH region that does look like a + // finally clause to the VM. Thus, on these platforms, we can't have BBJ_EHCATCHRET target a + // BBJ_CALLFINALLY directly. (Note that on ARM32, we don't mark the thunk specially -- it lives directly + // within the 'try' region protected by the finally, since we generate code in such a way that execution + // never returns to the call-to-finally call, and the finally-protected 'try' region doesn't appear on + // stack walks.) - case CEE_LDARGA_S: - lclNum = getU1LittleEndian(codeAddr); - LDARGA: - JITDUMP(" %u", lclNum); - Verify(lclNum < info.compILargsCount, "bad arg num"); + assert(step->KindIs(BBJ_ALWAYS, BBJ_EHCATCHRET)); - if (compIsForInlining()) +#if FEATURE_EH_CALLFINALLY_THUNKS + if (step->bbJumpKind == BBJ_EHCATCHRET) { - // In IL, LDARGA(_S) is used to load the byref managed pointer of struct argument, - // followed by a ldfld to load the field. + // Need to create another step block in the 'try' region that will actually branch to the + // call-to-finally thunk. + BasicBlock* step2 = fgNewBBinRegion(BBJ_ALWAYS, XTnum + 1, 0, step); + step->bbJumpDest = step2; + step->bbJumpDest->bbRefs++; + step2->inheritWeight(block); + step2->bbFlags |= (block->bbFlags & BBF_RUN_RARELY) | BBF_IMPORTED; - op1 = impInlineFetchArg(lclNum, impInlineInfo->inlArgInfo, impInlineInfo->lclVarInfo); - if (op1->gtOper != GT_LCL_VAR) +#ifdef DEBUG + if (verbose) { - compInlineResult->NoteFatal(InlineObservation::CALLSITE_LDARGA_NOT_LOCAL_VAR); - return; + printf("impImportLeave - jumping out of a finally-protected try (EH#%u), step block is " + "BBJ_EHCATCHRET (" FMT_BB "), new BBJ_ALWAYS step-step block " FMT_BB "\n", + XTnum, step->bbNum, step2->bbNum); } +#endif - assert(op1->gtOper == GT_LCL_VAR); - - goto _PUSH_ADRVAR; + step = step2; + assert(stepType == ST_Catch); // Leave it as catch type for now. } +#endif // FEATURE_EH_CALLFINALLY_THUNKS - lclNum = compMapILargNum(lclNum); // account for possible hidden param - assertImp(lclNum < numArgs); +#if FEATURE_EH_CALLFINALLY_THUNKS + unsigned callFinallyTryIndex = + (HBtab->ebdEnclosingTryIndex == EHblkDsc::NO_ENCLOSING_INDEX) ? 0 : HBtab->ebdEnclosingTryIndex + 1; + unsigned callFinallyHndIndex = + (HBtab->ebdEnclosingHndIndex == EHblkDsc::NO_ENCLOSING_INDEX) ? 0 : HBtab->ebdEnclosingHndIndex + 1; +#else // !FEATURE_EH_CALLFINALLY_THUNKS + unsigned callFinallyTryIndex = XTnum + 1; + unsigned callFinallyHndIndex = 0; // don't care +#endif // !FEATURE_EH_CALLFINALLY_THUNKS - if (lclNum == info.compThisArg) + callBlock = fgNewBBinRegion(BBJ_CALLFINALLY, callFinallyTryIndex, callFinallyHndIndex, step); + step->bbJumpDest = callBlock; // the previous call to a finally returns to this call (to the next + // finally in the chain) + step->bbJumpDest->bbRefs++; + +#if defined(TARGET_ARM) + if (stepType == ST_FinallyReturn) { - lclNum = lvaArg0Var; + assert(step->bbJumpKind == BBJ_ALWAYS); + // Mark the target of a finally return + step->bbJumpDest->bbFlags |= BBF_FINALLY_TARGET; } +#endif // defined(TARGET_ARM) - goto ADRVAR; + /* The new block will inherit this block's weight */ + callBlock->inheritWeight(block); + callBlock->bbFlags |= BBF_IMPORTED; - ADRVAR: +#ifdef DEBUG + if (verbose) + { + printf("impImportLeave - jumping out of a finally-protected try (EH#%u), new BBJ_CALLFINALLY " + "block " FMT_BB "\n", + XTnum, callBlock->bbNum); + } +#endif + } - op1 = impCreateLocalNode(lclNum DEBUGARG(opcodeOffs + sz + 1)); + step = fgNewBBafter(BBJ_ALWAYS, callBlock, true); + stepType = ST_FinallyReturn; - _PUSH_ADRVAR: - assert(op1->gtOper == GT_LCL_VAR); + /* The new block will inherit this block's weight */ + step->inheritWeight(block); + step->bbFlags |= BBF_IMPORTED | BBF_KEEP_BBJ_ALWAYS; - /* Note that this is supposed to create the transient type "*" - which may be used as a TYP_I_IMPL. However we catch places - where it is used as a TYP_I_IMPL and change the node if needed. - Thus we are pessimistic and may report byrefs in the GC info - where it was not absolutely needed, but it is safer this way. - */ - op1 = gtNewOperNode(GT_ADDR, TYP_BYREF, op1); +#ifdef DEBUG + if (verbose) + { + printf("impImportLeave - jumping out of a finally-protected try (EH#%u), created step (BBJ_ALWAYS) " + "block " FMT_BB "\n", + XTnum, step->bbNum); + } +#endif - // &aliasedVar doesnt need GTF_GLOB_REF, though alisasedVar does - assert((op1->gtFlags & GTF_GLOB_REF) == 0); + callBlock->bbJumpDest = HBtab->ebdHndBeg; // This callBlock will call the "finally" handler. - tiRetVal = lvaTable[lclNum].lvVerTypeInfo; + invalidatePreds = true; + } + else if (HBtab->HasCatchHandler() && jitIsBetween(blkAddr, tryBeg, tryEnd) && + !jitIsBetween(jmpAddr, tryBeg, tryEnd)) + { + // We are jumping out of a catch-protected try. + // + // If we are returning from a call to a finally, then we must have a step block within a try + // that is protected by a catch. This is so when unwinding from that finally (e.g., if code within the + // finally raises an exception), the VM will find this step block, notice that it is in a protected region, + // and invoke the appropriate catch. + // + // We also need to handle a special case with the handling of ThreadAbortException. If a try/catch + // catches a ThreadAbortException (which might be because it catches a parent, e.g. System.Exception), + // and the catch doesn't call System.Threading.Thread::ResetAbort(), then when the catch returns to the VM, + // the VM will automatically re-raise the ThreadAbortException. When it does this, it uses the target + // address of the catch return as the new exception address. That is, the re-raised exception appears to + // occur at the catch return address. If this exception return address skips an enclosing try/catch that + // catches ThreadAbortException, then the enclosing try/catch will not catch the exception, as it should. + // For example: + // + // try { + // try { + // // something here raises ThreadAbortException + // LEAVE LABEL_1; // no need to stop at LABEL_2 + // } catch (Exception) { + // // This catches ThreadAbortException, but doesn't call System.Threading.Thread::ResetAbort(), so + // // ThreadAbortException is re-raised by the VM at the address specified by the LEAVE opcode. + // // This is bad, since it means the outer try/catch won't get a chance to catch the re-raised + // // ThreadAbortException. So, instead, create step block LABEL_2 and LEAVE to that. We only + // // need to do this transformation if the current EH block is a try/catch that catches + // // ThreadAbortException (or one of its parents), however we might not be able to find that + // // information, so currently we do it for all catch types. + // LEAVE LABEL_1; // Convert this to LEAVE LABEL2; + // } + // LABEL_2: LEAVE LABEL_1; // inserted by this step creation code + // } catch (ThreadAbortException) { + // } + // LABEL_1: + // + // Note that this pattern isn't theoretical: it occurs in ASP.NET, in IL code generated by the Roslyn C# + // compiler. - impPushOnStack(op1, tiRetVal); - break; + if ((stepType == ST_FinallyReturn) || (stepType == ST_Catch)) + { + BasicBlock* catchStep; - case CEE_ARGLIST: + assert(step); - if (!info.compIsVarArgs) + if (stepType == ST_FinallyReturn) { - BADCODE("arglist in non-vararg method"); + assert(step->bbJumpKind == BBJ_ALWAYS); } - - assertImp((info.compMethodInfo->args.callConv & CORINFO_CALLCONV_MASK) == CORINFO_CALLCONV_VARARG); - - /* The ARGLIST cookie is a hidden 'last' parameter, we have already - adjusted the arg count cos this is like fetching the last param */ - assertImp(0 < numArgs); - lclNum = lvaVarargsHandleArg; - op1 = gtNewLclvNode(lclNum, TYP_I_IMPL DEBUGARG(opcodeOffs + sz + 1)); - op1 = gtNewOperNode(GT_ADDR, TYP_BYREF, op1); - impPushOnStack(op1, tiRetVal); - break; - - case CEE_ENDFINALLY: - - if (compIsForInlining()) + else { - assert(!"Shouldn't have exception handlers in the inliner!"); - compInlineResult->NoteFatal(InlineObservation::CALLEE_HAS_ENDFINALLY); - return; + assert(stepType == ST_Catch); + assert(step->bbJumpKind == BBJ_EHCATCHRET); } - if (verCurrentState.esStackDepth > 0) + /* Create a new exit block in the try region for the existing step block to jump to in this scope */ + catchStep = fgNewBBinRegion(BBJ_ALWAYS, XTnum + 1, 0, step); + step->bbJumpDest = catchStep; + step->bbJumpDest->bbRefs++; + +#if defined(TARGET_ARM) + if (stepType == ST_FinallyReturn) { - impEvalSideEffects(); + // Mark the target of a finally return + step->bbJumpDest->bbFlags |= BBF_FINALLY_TARGET; } +#endif // defined(TARGET_ARM) - if (info.compXcptnsCount == 0) + /* The new block will inherit this block's weight */ + catchStep->inheritWeight(block); + catchStep->bbFlags |= BBF_IMPORTED; + +#ifdef DEBUG + if (verbose) { - BADCODE("endfinally outside finally"); + if (stepType == ST_FinallyReturn) + { + printf("impImportLeave - return from finally jumping out of a catch-protected try (EH#%u), new " + "BBJ_ALWAYS block " FMT_BB "\n", + XTnum, catchStep->bbNum); + } + else + { + assert(stepType == ST_Catch); + printf("impImportLeave - return from catch jumping out of a catch-protected try (EH#%u), new " + "BBJ_ALWAYS block " FMT_BB "\n", + XTnum, catchStep->bbNum); + } } +#endif // DEBUG - assert(verCurrentState.esStackDepth == 0); + /* This block is the new step */ + step = catchStep; + stepType = ST_Try; - op1 = gtNewOperNode(GT_RETFILT, TYP_VOID, nullptr); - goto APPEND; + invalidatePreds = true; + } + } + } - case CEE_ENDFILTER: + if (step == nullptr) + { + block->bbJumpKind = BBJ_ALWAYS; // convert the BBJ_LEAVE to a BBJ_ALWAYS - if (compIsForInlining()) - { - assert(!"Shouldn't have exception handlers in the inliner!"); - compInlineResult->NoteFatal(InlineObservation::CALLEE_HAS_ENDFILTER); - return; - } +#ifdef DEBUG + if (verbose) + { + printf("impImportLeave - no enclosing finally-protected try blocks or catch handlers; convert CEE_LEAVE " + "block " FMT_BB " to BBJ_ALWAYS\n", + block->bbNum); + } +#endif + } + else + { + step->bbJumpDest = leaveTarget; // this is the ultimate destination of the LEAVE - block->bbSetRunRarely(); // filters are rare +#if defined(TARGET_ARM) + if (stepType == ST_FinallyReturn) + { + assert(step->bbJumpKind == BBJ_ALWAYS); + // Mark the target of a finally return + step->bbJumpDest->bbFlags |= BBF_FINALLY_TARGET; + } +#endif // defined(TARGET_ARM) - if (info.compXcptnsCount == 0) - { - BADCODE("endfilter outside filter"); - } +#ifdef DEBUG + if (verbose) + { + printf("impImportLeave - final destination of step blocks set to " FMT_BB "\n", leaveTarget->bbNum); + } +#endif - op1 = impPopStack().val; - assertImp(op1->gtType == TYP_INT); - if (!bbInFilterILRange(block)) - { - BADCODE("EndFilter outside a filter handler"); - } + // Queue up the jump target for importing + + impImportBlockPending(leaveTarget); + } + + if (invalidatePreds && fgComputePredsDone) + { + JITDUMP("\n**** impImportLeave - Removing preds after creating new blocks\n"); + fgRemovePreds(); + } + +#ifdef DEBUG + fgVerifyHandlerTab(); + + if (verbose) + { + printf("\nAfter import CEE_LEAVE:\n"); + fgDispBasicBlocks(); + fgDispHandlerTab(); + } +#endif // DEBUG +} + +#endif // FEATURE_EH_FUNCLETS + +/*****************************************************************************/ +// This is called when reimporting a leave block. It resets the JumpKind, +// JumpDest, and bbNext to the original values + +void Compiler::impResetLeaveBlock(BasicBlock* block, unsigned jmpAddr) +{ +#if defined(FEATURE_EH_FUNCLETS) + // With EH Funclets, while importing leave opcode we create another block ending with BBJ_ALWAYS (call it B1) + // and the block containing leave (say B0) is marked as BBJ_CALLFINALLY. Say for some reason we reimport B0, + // it is reset (in this routine) by marking as ending with BBJ_LEAVE and further down when B0 is reimported, we + // create another BBJ_ALWAYS (call it B2). In this process B1 gets orphaned and any blocks to which B1 is the + // only predecessor are also considered orphans and attempted to be deleted. + // + // try { + // .... + // try + // { + // .... + // leave OUTSIDE; // B0 is the block containing this leave, following this would be B1 + // } finally { } + // } finally { } + // OUTSIDE: + // + // In the above nested try-finally example, we create a step block (call it Bstep) which in branches to a block + // where a finally would branch to (and such block is marked as finally target). Block B1 branches to step block. + // Because of re-import of B0, Bstep is also orphaned. Since Bstep is a finally target it cannot be removed. To + // work around this we will duplicate B0 (call it B0Dup) before resetting. B0Dup is marked as BBJ_CALLFINALLY and + // only serves to pair up with B1 (BBJ_ALWAYS) that got orphaned. Now during orphan block deletion B0Dup and B1 + // will be treated as pair and handled correctly. + if (block->bbJumpKind == BBJ_CALLFINALLY) + { + BasicBlock* dupBlock = bbNewBasicBlock(block->bbJumpKind); + dupBlock->bbFlags = block->bbFlags; + dupBlock->bbJumpDest = block->bbJumpDest; + dupBlock->copyEHRegion(block); + dupBlock->bbCatchTyp = block->bbCatchTyp; - /* Mark current bb as end of filter */ + // Mark this block as + // a) not referenced by any other block to make sure that it gets deleted + // b) weight zero + // c) prevent from being imported + // d) as internal + // e) as rarely run + dupBlock->bbRefs = 0; + dupBlock->bbWeight = BB_ZERO_WEIGHT; + dupBlock->bbFlags |= BBF_IMPORTED | BBF_INTERNAL | BBF_RUN_RARELY; - assert(compCurBB->bbFlags & BBF_DONT_REMOVE); - assert(compCurBB->bbJumpKind == BBJ_EHFILTERRET); + // Insert the block right after the block which is getting reset so that BBJ_CALLFINALLY and BBJ_ALWAYS + // will be next to each other. + fgInsertBBafter(block, dupBlock); - /* Mark catch handler as successor */ +#ifdef DEBUG + if (verbose) + { + printf("New Basic Block " FMT_BB " duplicate of " FMT_BB " created.\n", dupBlock->bbNum, block->bbNum); + } +#endif + } +#endif // FEATURE_EH_FUNCLETS - op1 = gtNewOperNode(GT_RETFILT, op1->TypeGet(), op1); - if (verCurrentState.esStackDepth != 0) - { - verRaiseVerifyException(INDEBUG("stack must be 1 on end of filter") DEBUGARG(__FILE__) - DEBUGARG(__LINE__)); - } - goto APPEND; + block->bbJumpKind = BBJ_LEAVE; + fgInitBBLookup(); + block->bbJumpDest = fgLookupBB(jmpAddr); - case CEE_RET: - prefixFlags &= ~PREFIX_TAILCALL; // ret without call before it - RET: - if (!impReturnInstruction(prefixFlags, opcode)) - { - return; // abort - } - else - { - break; - } + // We will leave the BBJ_ALWAYS block we introduced. When it's reimported + // the BBJ_ALWAYS block will be unreachable, and will be removed after. The + // reason we don't want to remove the block at this point is that if we call + // fgInitBBLookup() again we will do it wrong as the BBJ_ALWAYS block won't be + // added and the linked list length will be different than fgBBcount. +} - case CEE_JMP: +/*****************************************************************************/ +// Get the first non-prefix opcode. Used for verification of valid combinations +// of prefixes and actual opcodes. - assert(!compIsForInlining()); +OPCODE Compiler::impGetNonPrefixOpcode(const BYTE* codeAddr, const BYTE* codeEndp) +{ + while (codeAddr < codeEndp) + { + OPCODE opcode = (OPCODE)getU1LittleEndian(codeAddr); + codeAddr += sizeof(__int8); - if ((info.compFlags & CORINFO_FLG_SYNCH) || block->hasTryIndex() || block->hasHndIndex()) - { - /* CEE_JMP does not make sense in some "protected" regions. */ + if (opcode == CEE_PREFIX1) + { + if (codeAddr >= codeEndp) + { + break; + } + opcode = (OPCODE)(getU1LittleEndian(codeAddr) + 256); + codeAddr += sizeof(__int8); + } - BADCODE("Jmp not allowed in protected region"); - } + switch (opcode) + { + case CEE_UNALIGNED: + case CEE_VOLATILE: + case CEE_TAILCALL: + case CEE_CONSTRAINED: + case CEE_READONLY: + break; + default: + return opcode; + } - if (opts.IsReversePInvoke()) - { - BADCODE("Jmp not allowed in reverse P/Invoke"); - } + codeAddr += opcodeSizes[opcode]; + } - if (verCurrentState.esStackDepth != 0) - { - BADCODE("Stack must be empty after CEE_JMPs"); - } + return CEE_ILLEGAL; +} - _impResolveToken(CORINFO_TOKENKIND_Method); +/*****************************************************************************/ +// Checks whether the opcode is a valid opcode for volatile. and unaligned. prefixes - JITDUMP(" %08X", resolvedToken.token); +void Compiler::impValidateMemoryAccessOpcode(const BYTE* codeAddr, const BYTE* codeEndp, bool volatilePrefix) +{ + OPCODE opcode = impGetNonPrefixOpcode(codeAddr, codeEndp); - /* The signature of the target has to be identical to ours. - At least check that argCnt and returnType match */ + if (!( + // Opcode of all ldind and stdind happen to be in continuous, except stind.i. + ((CEE_LDIND_I1 <= opcode) && (opcode <= CEE_STIND_R8)) || (opcode == CEE_STIND_I) || + (opcode == CEE_LDFLD) || (opcode == CEE_STFLD) || (opcode == CEE_LDOBJ) || (opcode == CEE_STOBJ) || + (opcode == CEE_INITBLK) || (opcode == CEE_CPBLK) || + // volatile. prefix is allowed with the ldsfld and stsfld + (volatilePrefix && ((opcode == CEE_LDSFLD) || (opcode == CEE_STSFLD))))) + { + BADCODE("Invalid opcode for unaligned. or volatile. prefix"); + } +} - eeGetMethodSig(resolvedToken.hMethod, &sig); - if (sig.numArgs != info.compMethodInfo->args.numArgs || - sig.retType != info.compMethodInfo->args.retType || - sig.callConv != info.compMethodInfo->args.callConv) - { - BADCODE("Incompatible target for CEE_JMPs"); - } +/***************************************************************************** + * Determine the result type of an arithmetic operation + * On 64-bit inserts upcasts when native int is mixed with int32 + */ +var_types Compiler::impGetByRefResultType(genTreeOps oper, bool fUnsigned, GenTree** pOp1, GenTree** pOp2) +{ + var_types type = TYP_UNDEF; + GenTree* op1 = *pOp1; + GenTree* op2 = *pOp2; - op1 = new (this, GT_JMP) GenTreeVal(GT_JMP, TYP_VOID, (size_t)resolvedToken.hMethod); + // Arithmetic operations are generally only allowed with + // primitive types, but certain operations are allowed + // with byrefs - /* Mark the basic block as being a JUMP instead of RETURN */ + if ((oper == GT_SUB) && (genActualType(op1->TypeGet()) == TYP_BYREF || genActualType(op2->TypeGet()) == TYP_BYREF)) + { + if ((genActualType(op1->TypeGet()) == TYP_BYREF) && (genActualType(op2->TypeGet()) == TYP_BYREF)) + { + // byref1-byref2 => gives a native int + type = TYP_I_IMPL; + } + else if (genActualTypeIsIntOrI(op1->TypeGet()) && (genActualType(op2->TypeGet()) == TYP_BYREF)) + { + // [native] int - byref => gives a native int - block->bbFlags |= BBF_HAS_JMP; + // + // The reason is that it is possible, in managed C++, + // to have a tree like this: + // + // - + // / \. + // / \. + // / \. + // / \. + // const(h) int addr byref + // + // VSW 318822 + // + // So here we decide to make the resulting type to be a native int. + CLANG_FORMAT_COMMENT_ANCHOR; - /* Set this flag to make sure register arguments have a location assigned - * even if we don't use them inside the method */ +#ifdef TARGET_64BIT + if (genActualType(op1->TypeGet()) != TYP_I_IMPL) + { + // insert an explicit upcast + op1 = *pOp1 = gtNewCastNode(TYP_I_IMPL, op1, fUnsigned, fUnsigned ? TYP_U_IMPL : TYP_I_IMPL); + } +#endif // TARGET_64BIT - compJmpOpUsed = true; + type = TYP_I_IMPL; + } + else + { + // byref - [native] int => gives a byref + assert(genActualType(op1->TypeGet()) == TYP_BYREF && genActualTypeIsIntOrI(op2->TypeGet())); - fgNoStructPromotion = true; +#ifdef TARGET_64BIT + if ((genActualType(op2->TypeGet()) != TYP_I_IMPL)) + { + // insert an explicit upcast + op2 = *pOp2 = gtNewCastNode(TYP_I_IMPL, op2, fUnsigned, fUnsigned ? TYP_U_IMPL : TYP_I_IMPL); + } +#endif // TARGET_64BIT - goto APPEND; + type = TYP_BYREF; + } + } + else if ((oper == GT_ADD) && + (genActualType(op1->TypeGet()) == TYP_BYREF || genActualType(op2->TypeGet()) == TYP_BYREF)) + { + // byref + [native] int => gives a byref + // (or) + // [native] int + byref => gives a byref - case CEE_LDELEMA: - assertImp(sz == sizeof(unsigned)); + // only one can be a byref : byref op byref not allowed + assert(genActualType(op1->TypeGet()) != TYP_BYREF || genActualType(op2->TypeGet()) != TYP_BYREF); + assert(genActualTypeIsIntOrI(op1->TypeGet()) || genActualTypeIsIntOrI(op2->TypeGet())); - _impResolveToken(CORINFO_TOKENKIND_Class); +#ifdef TARGET_64BIT + if (genActualType(op2->TypeGet()) == TYP_BYREF) + { + if (genActualType(op1->TypeGet()) != TYP_I_IMPL) + { + // insert an explicit upcast + op1 = *pOp1 = gtNewCastNode(TYP_I_IMPL, op1, fUnsigned, fUnsigned ? TYP_U_IMPL : TYP_I_IMPL); + } + } + else if (genActualType(op2->TypeGet()) != TYP_I_IMPL) + { + // insert an explicit upcast + op2 = *pOp2 = gtNewCastNode(TYP_I_IMPL, op2, fUnsigned, fUnsigned ? TYP_U_IMPL : TYP_I_IMPL); + } +#endif // TARGET_64BIT - JITDUMP(" %08X", resolvedToken.token); + type = TYP_BYREF; + } +#ifdef TARGET_64BIT + else if (genActualType(op1->TypeGet()) == TYP_I_IMPL || genActualType(op2->TypeGet()) == TYP_I_IMPL) + { + assert(!varTypeIsFloating(op1->gtType) && !varTypeIsFloating(op2->gtType)); - ldelemClsHnd = resolvedToken.hClass; + // int + long => gives long + // long + int => gives long + // we get this because in the IL the long isn't Int64, it's just IntPtr - // If it's a value class array we just do a simple address-of - if (eeIsValueClass(ldelemClsHnd)) - { - CorInfoType cit = info.compCompHnd->getTypeForPrimitiveValueClass(ldelemClsHnd); - if (cit == CORINFO_TYPE_UNDEF) - { - lclTyp = TYP_STRUCT; - } - else - { - lclTyp = JITtype2varType(cit); - } - goto ARR_LD; - } + if (genActualType(op1->TypeGet()) != TYP_I_IMPL) + { + // insert an explicit upcast + op1 = *pOp1 = gtNewCastNode(TYP_I_IMPL, op1, fUnsigned, fUnsigned ? TYP_U_IMPL : TYP_I_IMPL); + } + else if (genActualType(op2->TypeGet()) != TYP_I_IMPL) + { + // insert an explicit upcast + op2 = *pOp2 = gtNewCastNode(TYP_I_IMPL, op2, fUnsigned, fUnsigned ? TYP_U_IMPL : TYP_I_IMPL); + } - // Similarly, if its a readonly access, we can do a simple address-of - // without doing a runtime type-check - if (prefixFlags & PREFIX_READONLY) - { - lclTyp = TYP_REF; - goto ARR_LD; - } + type = TYP_I_IMPL; + } +#else // 32-bit TARGET + else if (genActualType(op1->TypeGet()) == TYP_LONG || genActualType(op2->TypeGet()) == TYP_LONG) + { + assert(!varTypeIsFloating(op1->gtType) && !varTypeIsFloating(op2->gtType)); - // Otherwise we need the full helper function with run-time type check - op1 = impTokenToHandle(&resolvedToken); - if (op1 == nullptr) - { // compDonotInline() - return; - } + // int + long => gives long + // long + int => gives long - { - GenTree* type = op1; - GenTree* index = impPopStack().val; - GenTree* arr = impPopStack().val; -#ifdef TARGET_64BIT - // The CLI Spec allows an array to be indexed by either an int32 or a native int. - // The array helper takes a native int for array length. - // So if we have an int, explicitly extend it to be a native int. - if (genActualType(index->TypeGet()) != TYP_I_IMPL) - { - if (index->IsIntegralConst()) - { - index->gtType = TYP_I_IMPL; - } - else - { - bool isUnsigned = false; - index = gtNewCastNode(TYP_I_IMPL, index, isUnsigned, TYP_I_IMPL); - } - } + type = TYP_LONG; + } #endif // TARGET_64BIT - op1 = gtNewHelperCallNode(CORINFO_HELP_LDELEMA_REF, TYP_BYREF, arr, index, type); - } + else + { + // int + int => gives an int + assert(genActualType(op1->TypeGet()) != TYP_BYREF && genActualType(op2->TypeGet()) != TYP_BYREF); - impPushOnStack(op1, tiRetVal); - break; + assert(genActualType(op1->TypeGet()) == genActualType(op2->TypeGet()) || + (varTypeIsFloating(op1->gtType) && varTypeIsFloating(op2->gtType))); - // ldelem for reference and value types - case CEE_LDELEM: - assertImp(sz == sizeof(unsigned)); + type = genActualType(op1->gtType); - _impResolveToken(CORINFO_TOKENKIND_Class); + // If both operands are TYP_FLOAT, then leave it as TYP_FLOAT. + // Otherwise, turn floats into doubles + if ((type == TYP_FLOAT) && (genActualType(op2->gtType) != TYP_FLOAT)) + { + assert(genActualType(op2->gtType) == TYP_DOUBLE); + type = TYP_DOUBLE; + } + } - JITDUMP(" %08X", resolvedToken.token); + assert(type == TYP_BYREF || type == TYP_DOUBLE || type == TYP_FLOAT || type == TYP_LONG || type == TYP_INT); + return type; +} - ldelemClsHnd = resolvedToken.hClass; +//------------------------------------------------------------------------ +// impOptimizeCastClassOrIsInst: attempt to resolve a cast when jitting +// +// Arguments: +// op1 - value to cast +// pResolvedToken - resolved token for type to cast to +// isCastClass - true if this is a castclass, false if isinst +// +// Return Value: +// tree representing optimized cast, or null if no optimization possible - // If it's a reference type or generic variable type - // then just generate code as though it's a ldelem.ref instruction - if (!eeIsValueClass(ldelemClsHnd)) - { - lclTyp = TYP_REF; - opcode = CEE_LDELEM_REF; - } - else - { - CorInfoType jitTyp = info.compCompHnd->asCorInfoType(ldelemClsHnd); - lclTyp = JITtype2varType(jitTyp); - tiRetVal = verMakeTypeInfo(ldelemClsHnd); // precise type always needed for struct - tiRetVal.NormaliseForStack(); - } - goto ARR_LD; +GenTree* Compiler::impOptimizeCastClassOrIsInst(GenTree* op1, CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isCastClass) +{ + assert(op1->TypeGet() == TYP_REF); - case CEE_LDELEM_I1: - lclTyp = TYP_BYTE; - goto ARR_LD; - case CEE_LDELEM_I2: - lclTyp = TYP_SHORT; - goto ARR_LD; - case CEE_LDELEM_I: - lclTyp = TYP_I_IMPL; - goto ARR_LD; - case CEE_LDELEM_U4: - lclTyp = TYP_INT; - goto ARR_LD; - case CEE_LDELEM_I4: - lclTyp = TYP_INT; - goto ARR_LD; - case CEE_LDELEM_I8: - lclTyp = TYP_LONG; - goto ARR_LD; - case CEE_LDELEM_REF: - lclTyp = TYP_REF; - goto ARR_LD; - case CEE_LDELEM_R4: - lclTyp = TYP_FLOAT; - goto ARR_LD; - case CEE_LDELEM_R8: - lclTyp = TYP_DOUBLE; - goto ARR_LD; - case CEE_LDELEM_U1: - lclTyp = TYP_UBYTE; - goto ARR_LD; - case CEE_LDELEM_U2: - lclTyp = TYP_USHORT; - goto ARR_LD; + // Don't optimize for minopts or debug codegen. + if (opts.OptimizationDisabled()) + { + return nullptr; + } - ARR_LD: + // See what we know about the type of the object being cast. + bool isExact = false; + bool isNonNull = false; + CORINFO_CLASS_HANDLE fromClass = gtGetClassHandle(op1, &isExact, &isNonNull); - op2 = impPopStack().val; // index - op1 = impPopStack().val; // array - assertImp(op1->TypeIs(TYP_REF)); + if (fromClass != nullptr) + { + CORINFO_CLASS_HANDLE toClass = pResolvedToken->hClass; + JITDUMP("\nConsidering optimization of %s from %s%p (%s) to %p (%s)\n", isCastClass ? "castclass" : "isinst", + isExact ? "exact " : "", dspPtr(fromClass), eeGetClassName(fromClass), dspPtr(toClass), + eeGetClassName(toClass)); - // Check for null pointer - in the inliner case we simply abort. - if (compIsForInlining() && op1->IsCnsIntOrI()) - { - compInlineResult->NoteFatal(InlineObservation::CALLEE_HAS_NULL_FOR_LDELEM); - return; - } + // Perhaps we know if the cast will succeed or fail. + TypeCompareState castResult = info.compCompHnd->compareTypesForCast(fromClass, toClass); - // Mark the block as containing an index expression. + if (castResult == TypeCompareState::Must) + { + // Cast will succeed, result is simply op1. + JITDUMP("Cast will succeed, optimizing to simply return input\n"); + return op1; + } + else if (castResult == TypeCompareState::MustNot) + { + // See if we can sharpen exactness by looking for final classes + if (!isExact) + { + isExact = impIsClassExact(fromClass); + } - if (op1->OperIs(GT_LCL_VAR) && op2->OperIs(GT_LCL_VAR, GT_CNS_INT, GT_ADD)) + // Cast to exact type will fail. Handle case where we have + // an exact type (that is, fromClass is not a subtype) + // and we're not going to throw on failure. + if (isExact && !isCastClass) + { + JITDUMP("Cast will fail, optimizing to return null\n"); + GenTree* result = gtNewIconNode(0, TYP_REF); + + // If the cast was fed by a box, we can remove that too. + if (op1->IsBoxedValue()) { - block->bbFlags |= BBF_HAS_IDX_LEN; - optMethodFlags |= OMF_HAS_ARRAYREF; + JITDUMP("Also removing upstream box\n"); + gtTryRemoveBoxUpstreamEffects(op1); } - op1 = gtNewArrayIndexAddr(op1, op2, lclTyp, ldelemClsHnd); + return result; + } + else if (isExact) + { + JITDUMP("Not optimizing failing castclass (yet)\n"); + } + else + { + JITDUMP("Can't optimize since fromClass is inexact\n"); + } + } + else + { + JITDUMP("Result of cast unknown, must generate runtime test\n"); + } + } + else + { + JITDUMP("\nCan't optimize since fromClass is unknown\n"); + } - if (opcode != CEE_LDELEMA) - { - op1 = gtNewIndexIndir(op1->AsIndexAddr()); - } + return nullptr; +} - impPushOnStack(op1, tiRetVal); - break; +//------------------------------------------------------------------------ +// impCastClassOrIsInstToTree: build and import castclass/isinst +// +// Arguments: +// op1 - value to cast +// op2 - type handle for type to cast to +// pResolvedToken - resolved token from the cast operation +// isCastClass - true if this is castclass, false means isinst +// +// Return Value: +// Tree representing the cast +// +// Notes: +// May expand into a series of runtime checks or a helper call. - // stelem for reference and value types - case CEE_STELEM: +GenTree* Compiler::impCastClassOrIsInstToTree( + GenTree* op1, GenTree* op2, CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isCastClass, IL_OFFSET ilOffset) +{ + assert(op1->TypeGet() == TYP_REF); - assertImp(sz == sizeof(unsigned)); + // Optimistically assume the jit should expand this as an inline test + bool shouldExpandInline = true; + bool isClassExact = impIsClassExact(pResolvedToken->hClass); - _impResolveToken(CORINFO_TOKENKIND_Class); + // Profitability check. + // + // Don't bother with inline expansion when jit is trying to generate code quickly + if (opts.OptimizationDisabled()) + { + // not worth the code expansion if jitting fast or in a rarely run block + shouldExpandInline = false; + } + else if ((op1->gtFlags & GTF_GLOB_EFFECT) && lvaHaveManyLocals()) + { + // not worth creating an untracked local variable + shouldExpandInline = false; + } + else if (opts.jitFlags->IsSet(JitFlags::JIT_FLAG_BBINSTR) && (JitConfig.JitProfileCasts() == 1)) + { + // Optimizations are enabled but we're still instrumenting (including casts) + if (isCastClass && !isClassExact) + { + // Usually, we make a speculative assumption that it makes sense to expand castclass + // even for non-sealed classes, but let's rely on PGO in this specific case + shouldExpandInline = false; + } + } - JITDUMP(" %08X", resolvedToken.token); + if (shouldExpandInline && compCurBB->isRunRarely()) + { + // For cold blocks we only expand castclass against exact classes because it's cheap + shouldExpandInline = isCastClass && isClassExact; + } - stelemClsHnd = resolvedToken.hClass; + // Pessimistically assume the jit cannot expand this as an inline test + bool canExpandInline = false; + bool partialExpand = false; + const CorInfoHelpFunc helper = info.compCompHnd->getCastingHelper(pResolvedToken, isCastClass); - // If it's a reference type just behave as though it's a stelem.ref instruction - if (!eeIsValueClass(stelemClsHnd)) - { - goto STELEM_REF_POST_VERIFY; - } + CORINFO_CLASS_HANDLE exactCls = NO_CLASS_HANDLE; - // Otherwise extract the type - { - CorInfoType jitTyp = info.compCompHnd->asCorInfoType(stelemClsHnd); - lclTyp = JITtype2varType(jitTyp); - goto ARR_ST; - } + // Legality check. + // + // Not all classclass/isinst operations can be inline expanded. + // Check legality only if an inline expansion is desirable. + if (shouldExpandInline) + { + if (isCastClass) + { + // Jit can only inline expand CHKCASTCLASS and CHKCASTARRAY helpers. + canExpandInline = (helper == CORINFO_HELP_CHKCASTCLASS) || (helper == CORINFO_HELP_CHKCASTARRAY); + } + else if ((helper == CORINFO_HELP_ISINSTANCEOFCLASS) || (helper == CORINFO_HELP_ISINSTANCEOFARRAY)) + { + // If the class is exact, the jit can expand the IsInst check inline. + canExpandInline = isClassExact; + } - case CEE_STELEM_REF: - STELEM_REF_POST_VERIFY: + // Check if this cast helper have some profile data + if (impIsCastHelperMayHaveProfileData(helper)) + { + const int maxLikelyClasses = 32; + LikelyClassMethodRecord likelyClasses[maxLikelyClasses]; + unsigned likelyClassCount = + getLikelyClasses(likelyClasses, maxLikelyClasses, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset); + if (likelyClassCount > 0) { - GenTree* value = impStackTop(0).val; - GenTree* index = impStackTop(1).val; - GenTree* array = impStackTop(2).val; - - if (opts.OptimizationEnabled()) +#ifdef DEBUG + // Optional stress mode to pick a random known class, rather than + // the most likely known class. + if (JitConfig.JitRandomGuardedDevirtualization() != 0) { - // Is this a case where we can skip the covariant store check? - if (impCanSkipCovariantStoreCheck(value, array)) - { - lclTyp = TYP_REF; - goto ARR_ST; - } + // Reuse the random inliner's random state. + CLRRandom* const random = + impInlineRoot()->m_inlineStrategy->GetRandom(JitConfig.JitRandomGuardedDevirtualization()); + + unsigned index = static_cast(random->Next(static_cast(likelyClassCount))); + likelyClasses[0].handle = likelyClasses[index].handle; + likelyClasses[0].likelihood = 100; + likelyClassCount = 1; } +#endif - impPopStack(3); + LikelyClassMethodRecord likelyClass = likelyClasses[0]; + CORINFO_CLASS_HANDLE likelyCls = (CORINFO_CLASS_HANDLE)likelyClass.handle; -// Else call a helper function to do the assignment -#ifdef TARGET_64BIT - // The CLI Spec allows an array to be indexed by either an int32 or a native int. - // The array helper takes a native int for array length. - // So if we have an int, explicitly extend it to be a native int. - if (genActualType(index->TypeGet()) != TYP_I_IMPL) + if ((likelyCls != NO_CLASS_HANDLE) && + (likelyClass.likelihood > (UINT32)JitConfig.JitGuardedDevirtualizationChainLikelihood())) { - if (index->IsIntegralConst()) - { - index->gtType = TYP_I_IMPL; - } - else + if ((info.compCompHnd->compareTypesForCast(likelyCls, pResolvedToken->hClass) == + TypeCompareState::Must)) { - bool isUnsigned = false; - index = gtNewCastNode(TYP_I_IMPL, index, isUnsigned, TYP_I_IMPL); + assert((info.compCompHnd->getClassAttribs(likelyCls) & + (CORINFO_FLG_INTERFACE | CORINFO_FLG_ABSTRACT)) == 0); + JITDUMP("Adding \"is %s (%X)\" check as a fast path for %s using PGO data.\n", + eeGetClassName(likelyCls), likelyCls, isCastClass ? "castclass" : "isinst"); + + canExpandInline = true; + partialExpand = true; + exactCls = likelyCls; } } -#endif // TARGET_64BIT - op1 = gtNewHelperCallNode(CORINFO_HELP_ARRADDR_ST, TYP_VOID, array, index, value); - goto SPILL_APPEND; } + } + } - case CEE_STELEM_I1: - lclTyp = TYP_BYTE; - goto ARR_ST; - case CEE_STELEM_I2: - lclTyp = TYP_SHORT; - goto ARR_ST; - case CEE_STELEM_I: - lclTyp = TYP_I_IMPL; - goto ARR_ST; - case CEE_STELEM_I4: - lclTyp = TYP_INT; - goto ARR_ST; - case CEE_STELEM_I8: - lclTyp = TYP_LONG; - goto ARR_ST; - case CEE_STELEM_R4: - lclTyp = TYP_FLOAT; - goto ARR_ST; - case CEE_STELEM_R8: - lclTyp = TYP_DOUBLE; - goto ARR_ST; - - ARR_ST: - // TODO-Review: this comment is no longer correct. - /* The strict order of evaluation is LHS-operands, RHS-operands, - range-check, and then assignment. However, codegen currently - does the range-check before evaluation the RHS-operands. So to - maintain strict ordering, we spill the stack. */ - - if (impStackTop().val->gtFlags & GTF_SIDE_EFFECT) - { - impSpillSideEffects(false, (unsigned)CHECK_SPILL_ALL DEBUGARG( - "Strict ordering of exceptions for Array store")); - } - - // Pull the new value from the stack. - op2 = impPopStack().val; - impBashVarAddrsToI(op2); + const bool expandInline = canExpandInline && shouldExpandInline; - // Pull the index value. - op1 = impPopStack().val; + if (!expandInline) + { + JITDUMP("\nExpanding %s as call because %s\n", isCastClass ? "castclass" : "isinst", + canExpandInline ? "want smaller code or faster jitting" : "inline expansion not legal"); - // Pull the array address. - op3 = impPopStack().val; - assertImp(op3->TypeIs(TYP_REF)); + // If we CSE this class handle we prevent assertionProp from making SubType assertions + // so instead we force the CSE logic to not consider CSE-ing this class handle. + // + op2->gtFlags |= GTF_DONT_CSE; - // Mark the block as containing an index expression - if (op3->OperIs(GT_LCL_VAR) && op1->OperIs(GT_LCL_VAR, GT_CNS_INT, GT_ADD)) - { - block->bbFlags |= BBF_HAS_IDX_LEN; - optMethodFlags |= OMF_HAS_ARRAYREF; - } + GenTreeCall* call = gtNewHelperCallNode(helper, TYP_REF, op2, op1); + if ((JitConfig.JitClassProfiling() > 0) && impIsCastHelperEligibleForClassProbe(call) && !isClassExact) + { + HandleHistogramProfileCandidateInfo* pInfo = new (this, CMK_Inlining) HandleHistogramProfileCandidateInfo; + pInfo->ilOffset = ilOffset; + pInfo->probeIndex = info.compHandleHistogramProbeCount++; + call->gtHandleHistogramProfileCandidateInfo = pInfo; + compCurBB->bbFlags |= BBF_HAS_HISTOGRAM_PROFILE; + } + return call; + } - // Create the index node. - op1 = gtNewArrayIndexAddr(op3, op1, lclTyp, stelemClsHnd); - op1 = gtNewIndexIndir(op1->AsIndexAddr()); + JITDUMP("\nExpanding %s inline\n", isCastClass ? "castclass" : "isinst"); - // Create the assignment node and append it. - if (varTypeIsStruct(op1)) - { - op1 = impAssignStruct(op1, op2, stelemClsHnd, (unsigned)CHECK_SPILL_ALL); - } - else - { - op2 = impImplicitR4orR8Cast(op2, op1->TypeGet()); - op1 = gtNewAssignNode(op1, op2); - } - goto SPILL_APPEND; + impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("bubbling QMark2")); - case CEE_ADD: - oper = GT_ADD; - goto MATH_OP2; + GenTree* temp; + GenTree* condMT; + // + // expand the methodtable match: + // + // condMT ==> GT_NE + // / \. + // GT_IND op2 (typically CNS_INT) + // | + // op1Copy + // - case CEE_ADD_OVF: - uns = false; - goto ADD_OVF; - case CEE_ADD_OVF_UN: - uns = true; - goto ADD_OVF; + // This can replace op1 with a GT_COMMA that evaluates op1 into a local + // + op1 = impCloneExpr(op1, &temp, NO_CLASS_HANDLE, CHECK_SPILL_ALL, nullptr DEBUGARG("CASTCLASS eval op1")); + // + // op1 is now known to be a non-complex tree + // thus we can use gtClone(op1) from now on + // - ADD_OVF: - ovfl = true; - callNode = false; - oper = GT_ADD; - goto MATH_OP2_FLAGS; + GenTree* op2Var = op2; + if (isCastClass && !partialExpand) + { + op2Var = fgInsertCommaFormTemp(&op2); + lvaTable[op2Var->AsLclVarCommon()->GetLclNum()].lvIsCSE = true; + } + temp = gtNewMethodTableLookup(temp); + condMT = + gtNewOperNode(GT_NE, TYP_INT, temp, (exactCls != NO_CLASS_HANDLE) ? gtNewIconEmbClsHndNode(exactCls) : op2); - case CEE_SUB: - oper = GT_SUB; - goto MATH_OP2; + GenTree* condNull; + // + // expand the null check: + // + // condNull ==> GT_EQ + // / \. + // op1Copy CNS_INT + // null + // + condNull = gtNewOperNode(GT_EQ, TYP_INT, gtClone(op1), gtNewIconNode(0, TYP_REF)); - case CEE_SUB_OVF: - uns = false; - goto SUB_OVF; - case CEE_SUB_OVF_UN: - uns = true; - goto SUB_OVF; + // + // expand the true and false trees for the condMT + // + GenTree* condFalse = gtClone(op1); + GenTree* condTrue; + if (isCastClass) + { + assert((helper == CORINFO_HELP_CHKCASTCLASS) || (helper == CORINFO_HELP_CHKCASTARRAY) || + (helper == CORINFO_HELP_CHKCASTINTERFACE)); - SUB_OVF: - ovfl = true; - callNode = false; - oper = GT_SUB; - goto MATH_OP2_FLAGS; + CorInfoHelpFunc specialHelper = helper; + if ((helper == CORINFO_HELP_CHKCASTCLASS) && + ((exactCls == nullptr) || (exactCls == gtGetHelperArgClassHandle(op2)))) + { + // use the special helper that skips the cases checked by our inlined cast + specialHelper = CORINFO_HELP_CHKCASTCLASS_SPECIAL; + } + condTrue = gtNewHelperCallNode(specialHelper, TYP_REF, partialExpand ? op2 : op2Var, gtClone(op1)); + } + else if (partialExpand) + { + condTrue = gtNewHelperCallNode(helper, TYP_REF, op2, gtClone(op1)); + } + else + { + condTrue = gtNewIconNode(0, TYP_REF); + } - case CEE_MUL: - oper = GT_MUL; - goto MATH_MAYBE_CALL_NO_OVF; + GenTree* qmarkMT; + // + // Generate first QMARK - COLON tree + // + // qmarkMT ==> GT_QMARK + // / \. + // condMT GT_COLON + // / \. + // condFalse condTrue + // + temp = new (this, GT_COLON) GenTreeColon(TYP_REF, condTrue, condFalse); + qmarkMT = gtNewQmarkNode(TYP_REF, condMT, temp->AsColon()); - case CEE_MUL_OVF: - uns = false; - goto MUL_OVF; - case CEE_MUL_OVF_UN: - uns = true; - goto MUL_OVF; + if (isCastClass && isClassExact && condTrue->OperIs(GT_CALL)) + { + if (helper == CORINFO_HELP_CHKCASTCLASS) + { + // condTrue is used only for throwing InvalidCastException in case of casting to an exact class. + condTrue->AsCall()->gtCallMoreFlags |= GTF_CALL_M_DOES_NOT_RETURN; + } + } - MUL_OVF: - ovfl = true; - oper = GT_MUL; - goto MATH_MAYBE_CALL_OVF; + GenTree* qmarkNull; + // + // Generate second QMARK - COLON tree + // + // qmarkNull ==> GT_QMARK + // / \. + // condNull GT_COLON + // / \. + // qmarkMT op1Copy + // + temp = new (this, GT_COLON) GenTreeColon(TYP_REF, gtClone(op1), qmarkMT); + qmarkNull = gtNewQmarkNode(TYP_REF, condNull, temp->AsColon()); + qmarkNull->gtFlags |= GTF_QMARK_CAST_INSTOF; - // Other binary math operations + // Make QMark node a top level node by spilling it. + unsigned tmp = lvaGrabTemp(true DEBUGARG("spilling QMark2")); + impAssignTempGen(tmp, qmarkNull, CHECK_SPILL_NONE); - case CEE_DIV: - oper = GT_DIV; - goto MATH_MAYBE_CALL_NO_OVF; + // TODO-CQ: Is it possible op1 has a better type? + // + // See also gtGetHelperCallClassHandle where we make the same + // determination for the helper call variants. + LclVarDsc* lclDsc = lvaGetDesc(tmp); + assert(lclDsc->lvSingleDef == 0); + lclDsc->lvSingleDef = 1; + JITDUMP("Marked V%02u as a single def temp\n", tmp); + lvaSetClass(tmp, pResolvedToken->hClass); + return gtNewLclvNode(tmp, TYP_REF); +} - case CEE_DIV_UN: - oper = GT_UDIV; - goto MATH_MAYBE_CALL_NO_OVF; +#ifndef DEBUG +#define assertImp(cond) ((void)0) +#else +#define assertImp(cond) \ + do \ + { \ + if (!(cond)) \ + { \ + const int cchAssertImpBuf = 600; \ + char* assertImpBuf = (char*)_alloca(cchAssertImpBuf); \ + _snprintf_s(assertImpBuf, cchAssertImpBuf, cchAssertImpBuf - 1, \ + "%s : Possibly bad IL with CEE_%s at offset %04Xh (op1=%s op2=%s stkDepth=%d)", #cond, \ + impCurOpcName, impCurOpcOffs, op1 ? varTypeName(op1->TypeGet()) : "NULL", \ + op2 ? varTypeName(op2->TypeGet()) : "NULL", verCurrentState.esStackDepth); \ + assertAbort(assertImpBuf, __FILE__, __LINE__); \ + } \ + } while (0) +#endif // DEBUG - case CEE_REM: - oper = GT_MOD; - goto MATH_MAYBE_CALL_NO_OVF; +//------------------------------------------------------------------------ +// impBlockIsInALoop: check if a block might be in a loop +// +// Arguments: +// block - block to check +// +// Returns: +// true if the block might be in a loop. +// +// Notes: +// Conservatively correct; may return true for some blocks that are +// not actually in loops. +// +bool Compiler::impBlockIsInALoop(BasicBlock* block) +{ + return (compIsForInlining() && ((impInlineInfo->iciBlock->bbFlags & BBF_BACKWARD_JUMP) != 0)) || + ((block->bbFlags & BBF_BACKWARD_JUMP) != 0); +} - case CEE_REM_UN: - oper = GT_UMOD; - goto MATH_MAYBE_CALL_NO_OVF; +#ifdef _PREFAST_ +#pragma warning(push) +#pragma warning(disable : 21000) // Suppress PREFast warning about overly large function +#endif +/***************************************************************************** + * Import the instr for the given basic block + */ +void Compiler::impImportBlockCode(BasicBlock* block) +{ +#define _impResolveToken(kind) impResolveToken(codeAddr, &resolvedToken, kind) - MATH_MAYBE_CALL_NO_OVF: - ovfl = false; - MATH_MAYBE_CALL_OVF: - // Morpher has some complex logic about when to turn different - // typed nodes on different platforms into helper calls. We - // need to either duplicate that logic here, or just - // pessimistically make all the nodes large enough to become - // call nodes. Since call nodes aren't that much larger and - // these opcodes are infrequent enough I chose the latter. - callNode = true; - goto MATH_OP2_FLAGS; +#ifdef DEBUG - case CEE_AND: - oper = GT_AND; - goto MATH_OP2; - case CEE_OR: - oper = GT_OR; - goto MATH_OP2; - case CEE_XOR: - oper = GT_XOR; - goto MATH_OP2; + if (verbose) + { + printf("\nImporting " FMT_BB " (PC=%03u) of '%s'", block->bbNum, block->bbCodeOffs, info.compFullName); + } +#endif - MATH_OP2: // For default values of 'ovfl' and 'callNode' + unsigned nxtStmtIndex = impInitBlockLineInfo(); + IL_OFFSET nxtStmtOffs; + CorInfoHelpFunc helper; + CorInfoIsAccessAllowedResult accessAllowedResult; + CORINFO_HELPER_DESC calloutHelper; + const BYTE* lastLoadToken = nullptr; - ovfl = false; - callNode = false; + /* Get the tree list started */ - MATH_OP2_FLAGS: // If 'ovfl' and 'callNode' have already been set + impBeginTreeList(); - /* Pull two values and push back the result */ +#ifdef FEATURE_ON_STACK_REPLACEMENT - op2 = impPopStack().val; - op1 = impPopStack().val; + bool enablePatchpoints = opts.jitFlags->IsSet(JitFlags::JIT_FLAG_TIER0) && (JitConfig.TC_OnStackReplacement() > 0); - /* Can't do arithmetic with references */ - assertImp(genActualType(op1->TypeGet()) != TYP_REF && genActualType(op2->TypeGet()) != TYP_REF); +#ifdef DEBUG - // Change both to TYP_I_IMPL (impBashVarAddrsToI won't change if its a true byref, only - // if it is in the stack) - impBashVarAddrsToI(op1, op2); + // Optionally suppress patchpoints by method hash + // + static ConfigMethodRange JitEnablePatchpointRange; + JitEnablePatchpointRange.EnsureInit(JitConfig.JitEnablePatchpointRange()); + const unsigned hash = impInlineRoot()->info.compMethodHash(); + const bool inRange = JitEnablePatchpointRange.Contains(hash); + enablePatchpoints &= inRange; - type = impGetByRefResultType(oper, uns, &op1, &op2); +#endif // DEBUG - assert(!ovfl || !varTypeIsFloating(op1->gtType)); + if (enablePatchpoints) + { + // We don't inline at Tier0, if we do, we may need rethink our approach. + // Could probably support inlines that don't introduce flow. + // + assert(!compIsForInlining()); - /* Special case: "int+0", "int-0", "int*1", "int/1" */ + // OSR is not yet supported for methods with explicit tail calls. + // + // But we also do not have to switch these methods to be optimized, as we should be + // able to avoid getting trapped in Tier0 code by normal call counting. + // So instead, just suppress adding patchpoints. + // + if (!compTailPrefixSeen) + { + // We only need to add patchpoints if the method can loop. + // + if (compHasBackwardJump) + { + assert(compCanHavePatchpoints()); - if (op2->gtOper == GT_CNS_INT) + // By default we use the "adaptive" strategy. + // + // This can create both source and target patchpoints within a given + // loop structure, which isn't ideal, but is not incorrect. We will + // just have some extra Tier0 overhead. + // + // Todo: implement support for mid-block patchpoints. If `block` + // is truly a backedge source (and not in a handler) then we should be + // able to find a stack empty point somewhere in the block. + // + const int patchpointStrategy = JitConfig.TC_PatchpointStrategy(); + bool addPatchpoint = false; + bool mustUseTargetPatchpoint = false; + + switch (patchpointStrategy) { - if ((op2->IsIntegralConst(0) && (oper == GT_ADD || oper == GT_SUB)) || - (op2->IsIntegralConst(1) && (oper == GT_MUL || oper == GT_DIV))) + default: + { + // Patchpoints at backedge sources, if possible, otherwise targets. + // + addPatchpoint = ((block->bbFlags & BBF_BACKWARD_JUMP_SOURCE) == BBF_BACKWARD_JUMP_SOURCE); + mustUseTargetPatchpoint = (verCurrentState.esStackDepth != 0) || block->hasHndIndex(); + break; + } + case 1: { - impPushOnStack(op1, tiRetVal); + // Patchpoints at stackempty backedge targets. + // Note if we have loops where the IL stack is not empty on the backedge we can't patchpoint + // them. + // + // We should not have allowed OSR if there were backedges in handlers. + // + assert(!block->hasHndIndex()); + addPatchpoint = ((block->bbFlags & BBF_BACKWARD_JUMP_TARGET) == BBF_BACKWARD_JUMP_TARGET) && + (verCurrentState.esStackDepth == 0); + break; + } + + case 2: + { + // Adaptive strategy. + // + // Patchpoints at backedge targets if there are multiple backedges, + // otherwise at backedge sources, if possible. Note a block can be both; if so we + // just need one patchpoint. + // + if ((block->bbFlags & BBF_BACKWARD_JUMP_TARGET) == BBF_BACKWARD_JUMP_TARGET) + { + // We don't know backedge count, so just use ref count. + // + addPatchpoint = (block->bbRefs > 1) && (verCurrentState.esStackDepth == 0); + } + + if (!addPatchpoint && ((block->bbFlags & BBF_BACKWARD_JUMP_SOURCE) == BBF_BACKWARD_JUMP_SOURCE)) + { + addPatchpoint = true; + mustUseTargetPatchpoint = (verCurrentState.esStackDepth != 0) || block->hasHndIndex(); + + // Also force target patchpoint if target block has multiple (backedge) preds. + // + if (!mustUseTargetPatchpoint) + { + for (BasicBlock* const succBlock : block->Succs(this)) + { + if ((succBlock->bbNum <= block->bbNum) && (succBlock->bbRefs > 1)) + { + mustUseTargetPatchpoint = true; + break; + } + } + } + } break; } } - // We can generate a TYP_FLOAT operation that has a TYP_DOUBLE operand - // - if (varTypeIsFloating(type) && varTypeIsFloating(op1->gtType) && varTypeIsFloating(op2->gtType)) + if (addPatchpoint) { - if (op1->TypeGet() != type) + if (mustUseTargetPatchpoint) { - // We insert a cast of op1 to 'type' - op1 = gtNewCastNode(type, op1, false, type); + // We wanted a source patchpoint, but could not have one. + // So, add patchpoints to the backedge targets. + // + for (BasicBlock* const succBlock : block->Succs(this)) + { + if (succBlock->bbNum <= block->bbNum) + { + // The succBlock had better agree it's a target. + // + assert((succBlock->bbFlags & BBF_BACKWARD_JUMP_TARGET) == BBF_BACKWARD_JUMP_TARGET); + + // We may already have decided to put a patchpoint in succBlock. If not, add one. + // + if ((succBlock->bbFlags & BBF_PATCHPOINT) != 0) + { + // In some cases the target may not be stack-empty at entry. + // If so, we will bypass patchpoints for this backedge. + // + if (succBlock->bbStackDepthOnEntry() > 0) + { + JITDUMP("\nCan't set source patchpoint at " FMT_BB ", can't use target " FMT_BB + " as it has non-empty stack on entry.\n", + block->bbNum, succBlock->bbNum); + } + else + { + JITDUMP("\nCan't set source patchpoint at " FMT_BB ", using target " FMT_BB + " instead\n", + block->bbNum, succBlock->bbNum); + + assert(!succBlock->hasHndIndex()); + succBlock->bbFlags |= BBF_PATCHPOINT; + } + } + } + } } - if (op2->TypeGet() != type) + else { - // We insert a cast of op2 to 'type' - op2 = gtNewCastNode(type, op2, false, type); + assert(!block->hasHndIndex()); + block->bbFlags |= BBF_PATCHPOINT; } - } - - if (callNode) - { - /* These operators can later be transformed into 'GT_CALL' */ - assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_MUL]); -#ifndef TARGET_ARM - assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_DIV]); - assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_UDIV]); - assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_MOD]); - assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_UMOD]); -#endif - // It's tempting to use LargeOpOpcode() here, but this logic is *not* saying - // that we'll need to transform into a general large node, but rather specifically - // to a call: by doing it this way, things keep working if there are multiple sizes, - // and a CALL is no longer the largest. - // That said, as of now it *is* a large node, so we'll do this with an assert rather - // than an "if". - assert(GenTree::s_gtNodeSizes[GT_CALL] == TREE_NODE_SZ_LARGE); - op1 = new (this, GT_CALL) GenTreeOp(oper, type, op1, op2 DEBUGARG(/*largeNode*/ true)); - } - else - { - op1 = gtNewOperNode(oper, type, op1, op2); + setMethodHasPatchpoint(); } + } + else + { + // Should not see backward branch targets w/o backwards branches. + // So if !compHasBackwardsBranch, these flags should never be set. + // + assert((block->bbFlags & (BBF_BACKWARD_JUMP_TARGET | BBF_BACKWARD_JUMP_SOURCE)) == 0); + } + } - /* Special case: integer/long division may throw an exception */ +#ifdef DEBUG + // As a stress test, we can place patchpoints at the start of any block + // that is a stack empty point and is not within a handler. + // + // Todo: enable for mid-block stack empty points too. + // + const int offsetOSR = JitConfig.JitOffsetOnStackReplacement(); + const int randomOSR = JitConfig.JitRandomOnStackReplacement(); + const bool tryOffsetOSR = offsetOSR >= 0; + const bool tryRandomOSR = randomOSR > 0; - if (varTypeIsIntegral(op1->TypeGet()) && op1->OperMayThrow(this)) - { - op1->gtFlags |= GTF_EXCEPT; - } + if (compCanHavePatchpoints() && (tryOffsetOSR || tryRandomOSR) && (verCurrentState.esStackDepth == 0) && + !block->hasHndIndex() && ((block->bbFlags & BBF_PATCHPOINT) == 0)) + { + // Block start can have a patchpoint. See if we should add one. + // + bool addPatchpoint = false; - if (ovfl) + // Specific offset? + // + if (tryOffsetOSR) + { + if (impCurOpcOffs == (unsigned)offsetOSR) { - assert(oper == GT_ADD || oper == GT_SUB || oper == GT_MUL); - if (ovflType != TYP_UNKNOWN) - { - op1->gtType = ovflType; - } - op1->gtFlags |= (GTF_EXCEPT | GTF_OVERFLOW); - if (uns) - { - op1->gtFlags |= GTF_UNSIGNED; - } + addPatchpoint = true; } + } + // Random? + // + else + { + // Reuse the random inliner's random state. + // Note m_inlineStrategy is always created, even if we're not inlining. + // + CLRRandom* const random = impInlineRoot()->m_inlineStrategy->GetRandom(randomOSR); + const int randomValue = (int)random->Next(100); - impPushOnStack(op1, tiRetVal); - break; - - case CEE_SHL: - oper = GT_LSH; - goto CEE_SH_OP2; - - case CEE_SHR: - oper = GT_RSH; - goto CEE_SH_OP2; - case CEE_SHR_UN: - oper = GT_RSZ; - goto CEE_SH_OP2; - - CEE_SH_OP2: - op2 = impPopStack().val; - op1 = impPopStack().val; // operand to be shifted - impBashVarAddrsToI(op1, op2); - - type = genActualType(op1->TypeGet()); - op1 = gtNewOperNode(oper, type, op1, op2); - - impPushOnStack(op1, tiRetVal); - break; - - case CEE_NOT: - op1 = impPopStack().val; - impBashVarAddrsToI(op1, nullptr); - type = genActualType(op1->TypeGet()); - impPushOnStack(gtNewOperNode(GT_NOT, type, op1), tiRetVal); - break; - - case CEE_CKFINITE: - op1 = impPopStack().val; - type = op1->TypeGet(); - op1 = gtNewOperNode(GT_CKFINITE, type, op1); - op1->gtFlags |= GTF_EXCEPT; + addPatchpoint = (randomValue < randomOSR); + } - impPushOnStack(op1, tiRetVal); - break; + if (addPatchpoint) + { + block->bbFlags |= BBF_PATCHPOINT; + setMethodHasPatchpoint(); + } - case CEE_LEAVE: + JITDUMP("\n** %s patchpoint%s added to " FMT_BB " (il offset %u)\n", tryOffsetOSR ? "offset" : "random", + addPatchpoint ? "" : " not", block->bbNum, impCurOpcOffs); + } - val = getI4LittleEndian(codeAddr); // jump distance - jmpAddr = (IL_OFFSET)((codeAddr - info.compCode + sizeof(__int32)) + val); - goto LEAVE; +#endif // DEBUG + } - case CEE_LEAVE_S: - val = getI1LittleEndian(codeAddr); // jump distance - jmpAddr = (IL_OFFSET)((codeAddr - info.compCode + sizeof(__int8)) + val); + // Mark stack-empty rare blocks to be considered for partial compilation. + // + // Ideally these are conditionally executed blocks -- if the method is going + // to unconditionally throw, there's not as much to be gained by deferring jitting. + // For now, we just screen out the entry bb. + // + // In general we might want track all the IL stack empty points so we can + // propagate rareness back through flow and place the partial compilation patchpoints "earlier" + // so there are fewer overall. + // + // Note unlike OSR, it's ok to forgo these. + // + // Todo: stress mode... + // + if (opts.jitFlags->IsSet(JitFlags::JIT_FLAG_TIER0) && (JitConfig.TC_PartialCompilation() > 0) && + compCanHavePatchpoints() && !compTailPrefixSeen) + { + // Is this block a good place for partial compilation? + // + if ((block != fgFirstBB) && block->isRunRarely() && (verCurrentState.esStackDepth == 0) && + ((block->bbFlags & BBF_PATCHPOINT) == 0) && !block->hasHndIndex()) + { + JITDUMP("\nBlock " FMT_BB " will be a partial compilation patchpoint -- not importing\n", block->bbNum); + block->bbFlags |= BBF_PARTIAL_COMPILATION_PATCHPOINT; + setMethodHasPartialCompilationPatchpoint(); - LEAVE: + // Change block to BBJ_THROW so we won't trigger importation of successors. + // + block->bbJumpKind = BBJ_THROW; - if (compIsForInlining()) - { - compInlineResult->NoteFatal(InlineObservation::CALLEE_HAS_LEAVE); - return; - } + // If this method has a explicit generic context, the only uses of it may be in + // the IL for this block. So assume it's used. + // + if (info.compMethodInfo->options & + (CORINFO_GENERICS_CTXT_FROM_METHODDESC | CORINFO_GENERICS_CTXT_FROM_METHODTABLE)) + { + lvaGenericsContextInUse = true; + } - JITDUMP(" %04X", jmpAddr); - if (block->bbJumpKind != BBJ_LEAVE) - { - impResetLeaveBlock(block, jmpAddr); - } + return; + } + } - assert(jmpAddr == block->bbJumpDest->bbCodeOffs); - impImportLeave(block); - impNoteBranchOffs(); +#endif // FEATURE_ON_STACK_REPLACEMENT - break; + /* Walk the opcodes that comprise the basic block */ - case CEE_BR: - case CEE_BR_S: - jmpDist = (sz == 1) ? getI1LittleEndian(codeAddr) : getI4LittleEndian(codeAddr); + const BYTE* codeAddr = info.compCode + block->bbCodeOffs; + const BYTE* codeEndp = info.compCode + block->bbCodeOffsEnd; - if (compIsForInlining() && jmpDist == 0) - { - break; /* NOP */ - } + IL_OFFSET opcodeOffs = block->bbCodeOffs; + IL_OFFSET lastSpillOffs = opcodeOffs; - impNoteBranchOffs(); - break; + signed jmpDist; - case CEE_BRTRUE: - case CEE_BRTRUE_S: - case CEE_BRFALSE: - case CEE_BRFALSE_S: + /* remember the start of the delegate creation sequence (used for verification) */ + const BYTE* delegateCreateStart = nullptr; - /* Pop the comparand (now there's a neat term) from the stack */ + int prefixFlags = 0; + bool explicitTailCall, constraintCall, readonlyCall; - op1 = impPopStack().val; - type = op1->TypeGet(); + typeInfo tiRetVal; - // Per Ecma-355, brfalse and brtrue are only specified for nint, ref, and byref. - // - // We've historically been a bit more permissive, so here we allow - // any type that gtNewZeroConNode can handle. - if (!varTypeIsArithmetic(type) && !varTypeIsGC(type)) - { - BADCODE("invalid type for brtrue/brfalse"); - } + unsigned numArgs = info.compArgsCount; - if (opts.OptimizationEnabled() && (block->bbJumpDest == block->bbNext)) - { - block->bbJumpKind = BBJ_NONE; + /* Now process all the opcodes in the block */ - if (op1->gtFlags & GTF_GLOB_EFFECT) - { - op1 = gtUnusedValNode(op1); - goto SPILL_APPEND; - } - else - { - break; - } - } + var_types callTyp = TYP_COUNT; + OPCODE prevOpcode = CEE_ILLEGAL; - if (op1->OperIsCompare()) - { - if (opcode == CEE_BRFALSE || opcode == CEE_BRFALSE_S) - { - // Flip the sense of the compare + if (block->bbCatchTyp) + { + if (info.compStmtOffsetsImplicit & ICorDebugInfo::CALL_SITE_BOUNDARIES) + { + impCurStmtOffsSet(block->bbCodeOffs); + } - op1 = gtReverseCond(op1); - } - } - else - { - // We'll compare against an equally-sized integer 0 - // For small types, we always compare against int - op2 = gtNewZeroConNode(genActualType(op1->gtType)); + // We will spill the GT_CATCH_ARG and the input of the BB_QMARK block + // to a temp. This is a trade off for code simplicity + impSpillSpecialSideEff(); + } - // Create the comparison operator and try to fold it - oper = (opcode == CEE_BRTRUE || opcode == CEE_BRTRUE_S) ? GT_NE : GT_EQ; - op1 = gtNewOperNode(oper, TYP_INT, op1, op2); - } + while (codeAddr < codeEndp) + { +#ifdef FEATURE_READYTORUN + bool usingReadyToRunHelper = false; +#endif + CORINFO_RESOLVED_TOKEN resolvedToken; + CORINFO_RESOLVED_TOKEN constrainedResolvedToken = {}; + CORINFO_CALL_INFO callInfo; + CORINFO_FIELD_INFO fieldInfo; - // fall through + tiRetVal = typeInfo(); // Default type info - COND_JUMP: + //--------------------------------------------------------------------- - /* Fold comparison if we can */ + /* We need to restrict the max tree depth as many of the Compiler + functions are recursive. We do this by spilling the stack */ - op1 = gtFoldExpr(op1); + if (verCurrentState.esStackDepth) + { + /* Has it been a while since we last saw a non-empty stack (which + guarantees that the tree depth isnt accumulating. */ - /* Try to fold the really simple cases like 'iconst *, ifne/ifeq'*/ - /* Don't make any blocks unreachable in import only mode */ + if ((opcodeOffs - lastSpillOffs) > MAX_TREE_SIZE && impCanSpillNow(prevOpcode)) + { + impSpillStackEnsure(); + lastSpillOffs = opcodeOffs; + } + } + else + { + lastSpillOffs = opcodeOffs; + impBoxTempInUse = false; // nothing on the stack, box temp OK to use again + } - if ((op1->gtOper == GT_CNS_INT) && !compIsForImportOnly()) - { - /* gtFoldExpr() should prevent this as we don't want to make any blocks - unreachable under compDbgCode */ - assert(!opts.compDbgCode); + /* Compute the current instr offset */ - BBjumpKinds foldedJumpKind = (BBjumpKinds)(op1->AsIntCon()->gtIconVal ? BBJ_ALWAYS : BBJ_NONE); - assertImp((block->bbJumpKind == BBJ_COND) // normal case - || (block->bbJumpKind == foldedJumpKind)); // this can happen if we are reimporting the - // block for the second time + opcodeOffs = (IL_OFFSET)(codeAddr - info.compCode); - block->bbJumpKind = foldedJumpKind; -#ifdef DEBUG - if (verbose) - { - if (op1->AsIntCon()->gtIconVal) - { - printf("\nThe conditional jump becomes an unconditional jump to " FMT_BB "\n", - block->bbJumpDest->bbNum); - } - else - { - printf("\nThe block falls through into the next " FMT_BB "\n", block->bbNext->bbNum); - } - } +#ifndef DEBUG + if (opts.compDbgInfo) #endif - break; - } + { + nxtStmtOffs = + (nxtStmtIndex < info.compStmtOffsetsCount) ? info.compStmtOffsets[nxtStmtIndex] : BAD_IL_OFFSET; - op1 = gtNewOperNode(GT_JTRUE, TYP_VOID, op1); + /* Have we reached the next stmt boundary ? */ - /* GT_JTRUE is handled specially for non-empty stacks. See 'addStmt' - in impImportBlock(block). For correct line numbers, spill stack. */ + if (nxtStmtOffs != BAD_IL_OFFSET && opcodeOffs >= nxtStmtOffs) + { + assert(nxtStmtOffs == info.compStmtOffsets[nxtStmtIndex]); - if (opts.compDbgCode && impCurStmtDI.IsValid()) + if (verCurrentState.esStackDepth != 0 && opts.compDbgCode) { + /* We need to provide accurate IP-mapping at this point. + So spill anything on the stack so that it will form + gtStmts with the correct stmt offset noted */ + impSpillStackEnsure(true); } - goto SPILL_APPEND; - - case CEE_CEQ: - oper = GT_EQ; - uns = false; - goto CMP_2_OPs; - case CEE_CGT_UN: - oper = GT_GT; - uns = true; - goto CMP_2_OPs; - case CEE_CGT: - oper = GT_GT; - uns = false; - goto CMP_2_OPs; - case CEE_CLT_UN: - oper = GT_LT; - uns = true; - goto CMP_2_OPs; - case CEE_CLT: - oper = GT_LT; - uns = false; - goto CMP_2_OPs; - - CMP_2_OPs: - op2 = impPopStack().val; - op1 = impPopStack().val; + // Have we reported debug info for any tree? - // Recognize the IL idiom of CGT_UN(op1, 0) and normalize - // it so that downstream optimizations don't have to. - if ((opcode == CEE_CGT_UN) && op2->IsIntegralConst(0)) + if (impCurStmtDI.IsValid() && opts.compDbgCode) { - oper = GT_NE; - uns = false; - } + GenTree* placeHolder = new (this, GT_NO_OP) GenTree(GT_NO_OP, TYP_VOID); + impAppendTree(placeHolder, CHECK_SPILL_NONE, impCurStmtDI); -#ifdef TARGET_64BIT - // TODO-Casts: create a helper that upcasts int32 -> native int when necessary. - // See also identical code in impGetByRefResultType and STSFLD import. - if (varTypeIsI(op1) && (genActualType(op2) == TYP_INT)) - { - op2 = gtNewCastNode(TYP_I_IMPL, op2, uns, TYP_I_IMPL); + assert(!impCurStmtDI.IsValid()); } - else if (varTypeIsI(op2) && (genActualType(op1) == TYP_INT)) + + if (!impCurStmtDI.IsValid()) { - op1 = gtNewCastNode(TYP_I_IMPL, op1, uns, TYP_I_IMPL); - } -#endif // TARGET_64BIT + /* Make sure that nxtStmtIndex is in sync with opcodeOffs. + If opcodeOffs has gone past nxtStmtIndex, catch up */ - assertImp(genActualType(op1) == genActualType(op2) || (varTypeIsI(op1) && varTypeIsI(op2)) || - (varTypeIsFloating(op1) && varTypeIsFloating(op2))); + while ((nxtStmtIndex + 1) < info.compStmtOffsetsCount && + info.compStmtOffsets[nxtStmtIndex + 1] <= opcodeOffs) + { + nxtStmtIndex++; + } - // Create the comparison node. + /* Go to the new stmt */ - op1 = gtNewOperNode(oper, TYP_INT, op1, op2); + impCurStmtOffsSet(info.compStmtOffsets[nxtStmtIndex]); + + /* Update the stmt boundary index */ + + nxtStmtIndex++; + assert(nxtStmtIndex <= info.compStmtOffsetsCount); + + /* Are there any more line# entries after this one? */ + + if (nxtStmtIndex < info.compStmtOffsetsCount) + { + /* Remember where the next line# starts */ + + nxtStmtOffs = info.compStmtOffsets[nxtStmtIndex]; + } + else + { + /* No more line# entries */ + + nxtStmtOffs = BAD_IL_OFFSET; + } + } + } + else if ((info.compStmtOffsetsImplicit & ICorDebugInfo::STACK_EMPTY_BOUNDARIES) && + (verCurrentState.esStackDepth == 0)) + { + /* At stack-empty locations, we have already added the tree to + the stmt list with the last offset. We just need to update + impCurStmtDI + */ + + impCurStmtOffsSet(opcodeOffs); + } + else if ((info.compStmtOffsetsImplicit & ICorDebugInfo::CALL_SITE_BOUNDARIES) && + impOpcodeIsCallSiteBoundary(prevOpcode)) + { + /* Make sure we have a type cached */ + assert(callTyp != TYP_COUNT); - // TODO: setting both flags when only one is appropriate. - if (uns) + if (callTyp == TYP_VOID) { - op1->gtFlags |= GTF_RELOP_NAN_UN | GTF_UNSIGNED; + impCurStmtOffsSet(opcodeOffs); + } + else if (opts.compDbgCode) + { + impSpillStackEnsure(true); + impCurStmtOffsSet(opcodeOffs); + } + } + else if ((info.compStmtOffsetsImplicit & ICorDebugInfo::NOP_BOUNDARIES) && (prevOpcode == CEE_NOP)) + { + if (opts.compDbgCode) + { + impSpillStackEnsure(true); } - // Fold result, if possible. - op1 = gtFoldExpr(op1); - - impPushOnStack(op1, tiRetVal); - break; - - case CEE_BEQ_S: - case CEE_BEQ: - oper = GT_EQ; - goto CMP_2_OPs_AND_BR; + impCurStmtOffsSet(opcodeOffs); + } - case CEE_BGE_S: - case CEE_BGE: - oper = GT_GE; - goto CMP_2_OPs_AND_BR; + assert(!impCurStmtDI.IsValid() || (nxtStmtOffs == BAD_IL_OFFSET) || + (impCurStmtDI.GetLocation().GetOffset() <= nxtStmtOffs)); + } - case CEE_BGE_UN_S: - case CEE_BGE_UN: - oper = GT_GE; - goto CMP_2_OPs_AND_BR_UN; + CORINFO_CLASS_HANDLE clsHnd = DUMMY_INIT(NULL); + CORINFO_CLASS_HANDLE ldelemClsHnd = NO_CLASS_HANDLE; + CORINFO_CLASS_HANDLE stelemClsHnd = DUMMY_INIT(NULL); - case CEE_BGT_S: - case CEE_BGT: - oper = GT_GT; - goto CMP_2_OPs_AND_BR; + var_types lclTyp, ovflType = TYP_UNKNOWN; + GenTree* op1 = DUMMY_INIT(NULL); + GenTree* op2 = DUMMY_INIT(NULL); + GenTree* newObjThisPtr = DUMMY_INIT(NULL); + bool uns = DUMMY_INIT(false); + bool isLocal = false; - case CEE_BGT_UN_S: - case CEE_BGT_UN: - oper = GT_GT; - goto CMP_2_OPs_AND_BR_UN; + /* Get the next opcode and the size of its parameters */ - case CEE_BLE_S: - case CEE_BLE: - oper = GT_LE; - goto CMP_2_OPs_AND_BR; + OPCODE opcode = (OPCODE)getU1LittleEndian(codeAddr); + codeAddr += sizeof(__int8); - case CEE_BLE_UN_S: - case CEE_BLE_UN: - oper = GT_LE; - goto CMP_2_OPs_AND_BR_UN; +#ifdef DEBUG + impCurOpcOffs = (IL_OFFSET)(codeAddr - info.compCode - 1); + JITDUMP("\n [%2u] %3u (0x%03x) ", verCurrentState.esStackDepth, impCurOpcOffs, impCurOpcOffs); +#endif - case CEE_BLT_S: - case CEE_BLT: - oper = GT_LT; - goto CMP_2_OPs_AND_BR; + DECODE_OPCODE: - case CEE_BLT_UN_S: - case CEE_BLT_UN: - oper = GT_LT; - goto CMP_2_OPs_AND_BR_UN; + // Return if any previous code has caused inline to fail. + if (compDonotInline()) + { + return; + } - case CEE_BNE_UN_S: - case CEE_BNE_UN: - oper = GT_NE; - goto CMP_2_OPs_AND_BR_UN; + /* Get the size of additional parameters */ - CMP_2_OPs_AND_BR_UN: - uns = true; - unordered = true; - goto CMP_2_OPs_AND_BR_ALL; - CMP_2_OPs_AND_BR: - uns = false; - unordered = false; - goto CMP_2_OPs_AND_BR_ALL; - CMP_2_OPs_AND_BR_ALL: - /* Pull two values */ - op2 = impPopStack().val; - op1 = impPopStack().val; + signed int sz = opcodeSizes[opcode]; -#ifdef TARGET_64BIT - if ((op1->TypeGet() == TYP_I_IMPL) && (genActualType(op2->TypeGet()) == TYP_INT)) - { - op2 = gtNewCastNode(TYP_I_IMPL, op2, uns, uns ? TYP_U_IMPL : TYP_I_IMPL); - } - else if ((op2->TypeGet() == TYP_I_IMPL) && (genActualType(op1->TypeGet()) == TYP_INT)) - { - op1 = gtNewCastNode(TYP_I_IMPL, op1, uns, uns ? TYP_U_IMPL : TYP_I_IMPL); - } -#endif // TARGET_64BIT +#ifdef DEBUG + clsHnd = NO_CLASS_HANDLE; + lclTyp = TYP_COUNT; + callTyp = TYP_COUNT; - assertImp(genActualType(op1->TypeGet()) == genActualType(op2->TypeGet()) || - (varTypeIsI(op1->TypeGet()) && varTypeIsI(op2->TypeGet())) || - (varTypeIsFloating(op1->gtType) && varTypeIsFloating(op2->gtType))); + impCurOpcOffs = (IL_OFFSET)(codeAddr - info.compCode - 1); + impCurOpcName = opcodeNames[opcode]; - if (opts.OptimizationEnabled() && (block->bbJumpDest == block->bbNext)) - { - block->bbJumpKind = BBJ_NONE; + if (verbose && (opcode != CEE_PREFIX1)) + { + printf("%s", impCurOpcName); + } - if (op1->gtFlags & GTF_GLOB_EFFECT) - { - impSpillSideEffects(false, (unsigned)CHECK_SPILL_ALL DEBUGARG( - "Branch to next Optimization, op1 side effect")); - impAppendTree(gtUnusedValNode(op1), (unsigned)CHECK_SPILL_NONE, impCurStmtDI); - } - if (op2->gtFlags & GTF_GLOB_EFFECT) - { - impSpillSideEffects(false, (unsigned)CHECK_SPILL_ALL DEBUGARG( - "Branch to next Optimization, op2 side effect")); - impAppendTree(gtUnusedValNode(op2), (unsigned)CHECK_SPILL_NONE, impCurStmtDI); - } + /* Use assertImp() to display the opcode */ -#ifdef DEBUG - if ((op1->gtFlags | op2->gtFlags) & GTF_GLOB_EFFECT) - { - impNoteLastILoffs(); - } + op1 = op2 = nullptr; #endif - break; - } - - // We can generate an compare of different sized floating point op1 and op2 - // We insert a cast - // - if (varTypeIsFloating(op1->TypeGet())) - { - if (op1->TypeGet() != op2->TypeGet()) - { - assert(varTypeIsFloating(op2->TypeGet())); - // say op1=double, op2=float. To avoid loss of precision - // while comparing, op2 is converted to double and double - // comparison is done. - if (op1->TypeGet() == TYP_DOUBLE) - { - // We insert a cast of op2 to TYP_DOUBLE - op2 = gtNewCastNode(TYP_DOUBLE, op2, false, TYP_DOUBLE); - } - else if (op2->TypeGet() == TYP_DOUBLE) - { - // We insert a cast of op1 to TYP_DOUBLE - op1 = gtNewCastNode(TYP_DOUBLE, op1, false, TYP_DOUBLE); - } - } - } + /* See what kind of an opcode we have, then */ - /* Create and append the operator */ + unsigned mflags = 0; + unsigned clsFlags = 0; - op1 = gtNewOperNode(oper, TYP_INT, op1, op2); + switch (opcode) + { + unsigned lclNum; + var_types type; - if (uns) - { - op1->gtFlags |= GTF_UNSIGNED; - } + GenTree* op3; + genTreeOps oper; + unsigned size; - if (unordered) - { - op1->gtFlags |= GTF_RELOP_NAN_UN; - } + int val; - goto COND_JUMP; + CORINFO_SIG_INFO sig; + IL_OFFSET jmpAddr; + bool ovfl, unordered, callNode; + CORINFO_CLASS_HANDLE tokenType; - case CEE_SWITCH: - /* Pop the switch value off the stack */ - op1 = impPopStack().val; - assertImp(genActualTypeIsIntOrI(op1->TypeGet())); + union { + int intVal; + float fltVal; + __int64 lngVal; + double dblVal; + } cval; - /* We can create a switch node */ + case CEE_PREFIX1: + opcode = (OPCODE)(getU1LittleEndian(codeAddr) + 256); + opcodeOffs = (IL_OFFSET)(codeAddr - info.compCode); + codeAddr += sizeof(__int8); + goto DECODE_OPCODE; - op1 = gtNewOperNode(GT_SWITCH, TYP_VOID, op1); + SPILL_APPEND: + impAppendTree(op1, CHECK_SPILL_ALL, impCurStmtDI); + goto DONE_APPEND; - val = (int)getU4LittleEndian(codeAddr); - codeAddr += 4 + val * 4; // skip over the switch-table + APPEND: + impAppendTree(op1, CHECK_SPILL_NONE, impCurStmtDI); + goto DONE_APPEND; - goto SPILL_APPEND; + DONE_APPEND: +#ifdef DEBUG + // Remember at which BC offset the tree was finished + impNoteLastILoffs(); +#endif + break; - /************************** Casting OPCODES ***************************/ + case CEE_LDNULL: + impPushNullObjRefOnStack(); + break; - case CEE_CONV_OVF_I1: - lclTyp = TYP_BYTE; - goto CONV_OVF; - case CEE_CONV_OVF_I2: - lclTyp = TYP_SHORT; - goto CONV_OVF; - case CEE_CONV_OVF_I: - lclTyp = TYP_I_IMPL; - goto CONV_OVF; - case CEE_CONV_OVF_I4: - lclTyp = TYP_INT; - goto CONV_OVF; - case CEE_CONV_OVF_I8: - lclTyp = TYP_LONG; - goto CONV_OVF; + case CEE_LDC_I4_M1: + case CEE_LDC_I4_0: + case CEE_LDC_I4_1: + case CEE_LDC_I4_2: + case CEE_LDC_I4_3: + case CEE_LDC_I4_4: + case CEE_LDC_I4_5: + case CEE_LDC_I4_6: + case CEE_LDC_I4_7: + case CEE_LDC_I4_8: + cval.intVal = (opcode - CEE_LDC_I4_0); + assert(-1 <= cval.intVal && cval.intVal <= 8); + goto PUSH_I4CON; - case CEE_CONV_OVF_U1: - lclTyp = TYP_UBYTE; - goto CONV_OVF; - case CEE_CONV_OVF_U2: - lclTyp = TYP_USHORT; - goto CONV_OVF; - case CEE_CONV_OVF_U: - lclTyp = TYP_U_IMPL; - goto CONV_OVF; - case CEE_CONV_OVF_U4: - lclTyp = TYP_UINT; - goto CONV_OVF; - case CEE_CONV_OVF_U8: - lclTyp = TYP_ULONG; - goto CONV_OVF; + case CEE_LDC_I4_S: + cval.intVal = getI1LittleEndian(codeAddr); + goto PUSH_I4CON; + case CEE_LDC_I4: + cval.intVal = getI4LittleEndian(codeAddr); + goto PUSH_I4CON; + PUSH_I4CON: + JITDUMP(" %d", cval.intVal); + impPushOnStack(gtNewIconNode(cval.intVal), typeInfo(TI_INT)); + break; - case CEE_CONV_OVF_I1_UN: - lclTyp = TYP_BYTE; - goto CONV_OVF_UN; - case CEE_CONV_OVF_I2_UN: - lclTyp = TYP_SHORT; - goto CONV_OVF_UN; - case CEE_CONV_OVF_I_UN: - lclTyp = TYP_I_IMPL; - goto CONV_OVF_UN; - case CEE_CONV_OVF_I4_UN: - lclTyp = TYP_INT; - goto CONV_OVF_UN; - case CEE_CONV_OVF_I8_UN: - lclTyp = TYP_LONG; - goto CONV_OVF_UN; + case CEE_LDC_I8: + cval.lngVal = getI8LittleEndian(codeAddr); + JITDUMP(" 0x%016llx", cval.lngVal); + impPushOnStack(gtNewLconNode(cval.lngVal), typeInfo(TI_LONG)); + break; - case CEE_CONV_OVF_U1_UN: - lclTyp = TYP_UBYTE; - goto CONV_OVF_UN; - case CEE_CONV_OVF_U2_UN: - lclTyp = TYP_USHORT; - goto CONV_OVF_UN; - case CEE_CONV_OVF_U_UN: - lclTyp = TYP_U_IMPL; - goto CONV_OVF_UN; - case CEE_CONV_OVF_U4_UN: - lclTyp = TYP_UINT; - goto CONV_OVF_UN; - case CEE_CONV_OVF_U8_UN: - lclTyp = TYP_ULONG; - goto CONV_OVF_UN; + case CEE_LDC_R8: + cval.dblVal = getR8LittleEndian(codeAddr); + JITDUMP(" %#.17g", cval.dblVal); + impPushOnStack(gtNewDconNode(cval.dblVal), typeInfo(TI_DOUBLE)); + break; - CONV_OVF_UN: - uns = true; - goto CONV_OVF_COMMON; - CONV_OVF: - uns = false; - goto CONV_OVF_COMMON; + case CEE_LDC_R4: + cval.dblVal = getR4LittleEndian(codeAddr); + JITDUMP(" %#.17g", cval.dblVal); + impPushOnStack(gtNewDconNode(cval.dblVal, TYP_FLOAT), typeInfo(TI_DOUBLE)); + break; - CONV_OVF_COMMON: - ovfl = true; - goto _CONV; + case CEE_LDSTR: + val = getU4LittleEndian(codeAddr); + JITDUMP(" %08X", val); + impPushOnStack(gtNewSconNode(val, info.compScopeHnd), tiRetVal); + break; - case CEE_CONV_I1: - lclTyp = TYP_BYTE; - goto CONV; - case CEE_CONV_I2: - lclTyp = TYP_SHORT; - goto CONV; - case CEE_CONV_I: - lclTyp = TYP_I_IMPL; - goto CONV; - case CEE_CONV_I4: - lclTyp = TYP_INT; - goto CONV; - case CEE_CONV_I8: - lclTyp = TYP_LONG; - goto CONV; + case CEE_LDARG: + lclNum = getU2LittleEndian(codeAddr); + JITDUMP(" %u", lclNum); + impLoadArg(lclNum, opcodeOffs + sz + 1); + break; - case CEE_CONV_U1: - lclTyp = TYP_UBYTE; - goto CONV; - case CEE_CONV_U2: - lclTyp = TYP_USHORT; - goto CONV; -#if (REGSIZE_BYTES == 8) - case CEE_CONV_U: - lclTyp = TYP_U_IMPL; - goto CONV_UN; -#else - case CEE_CONV_U: - lclTyp = TYP_U_IMPL; - goto CONV; -#endif - case CEE_CONV_U4: - lclTyp = TYP_UINT; - goto CONV; - case CEE_CONV_U8: - lclTyp = TYP_ULONG; - goto CONV_UN; + case CEE_LDARG_S: + lclNum = getU1LittleEndian(codeAddr); + JITDUMP(" %u", lclNum); + impLoadArg(lclNum, opcodeOffs + sz + 1); + break; - case CEE_CONV_R4: - lclTyp = TYP_FLOAT; - goto CONV; - case CEE_CONV_R8: - lclTyp = TYP_DOUBLE; - goto CONV; + case CEE_LDARG_0: + case CEE_LDARG_1: + case CEE_LDARG_2: + case CEE_LDARG_3: + lclNum = (opcode - CEE_LDARG_0); + assert(lclNum >= 0 && lclNum < 4); + impLoadArg(lclNum, opcodeOffs + sz + 1); + break; - case CEE_CONV_R_UN: - lclTyp = TYP_DOUBLE; - goto CONV_UN; + case CEE_LDLOC: + lclNum = getU2LittleEndian(codeAddr); + JITDUMP(" %u", lclNum); + impLoadLoc(lclNum, opcodeOffs + sz + 1); + break; - CONV_UN: - uns = true; - ovfl = false; - goto _CONV; + case CEE_LDLOC_S: + lclNum = getU1LittleEndian(codeAddr); + JITDUMP(" %u", lclNum); + impLoadLoc(lclNum, opcodeOffs + sz + 1); + break; - CONV: - uns = false; - ovfl = false; - goto _CONV; + case CEE_LDLOC_0: + case CEE_LDLOC_1: + case CEE_LDLOC_2: + case CEE_LDLOC_3: + lclNum = (opcode - CEE_LDLOC_0); + assert(lclNum >= 0 && lclNum < 4); + impLoadLoc(lclNum, opcodeOffs + sz + 1); + break; - _CONV: - // only converts from FLOAT or DOUBLE to an integer type - // and converts from ULONG (or LONG on ARM) to DOUBLE are morphed to calls + case CEE_STARG: + lclNum = getU2LittleEndian(codeAddr); + goto STARG; - if (varTypeIsFloating(lclTyp)) - { - callNode = varTypeIsLong(impStackTop().val) || uns // uint->dbl gets turned into uint->long->dbl -#ifdef TARGET_64BIT - // TODO-ARM64-Bug?: This was AMD64; I enabled it for ARM64 also. OK? - // TYP_BYREF could be used as TYP_I_IMPL which is long. - // TODO-CQ: remove this when we lower casts long/ulong --> float/double - // and generate SSE2 code instead of going through helper calls. - || (impStackTop().val->TypeGet() == TYP_BYREF) -#endif - ; - } - else + case CEE_STARG_S: + lclNum = getU1LittleEndian(codeAddr); + STARG: + JITDUMP(" %u", lclNum); + + if (compIsForInlining()) { - callNode = varTypeIsFloating(impStackTop().val->TypeGet()); - } + op1 = impInlineFetchArg(lclNum, impInlineInfo->inlArgInfo, impInlineInfo->lclVarInfo); + noway_assert(op1->gtOper == GT_LCL_VAR); + lclNum = op1->AsLclVar()->GetLclNum(); - op1 = impPopStack().val; + goto VAR_ST_VALID; + } - impBashVarAddrsToI(op1); + lclNum = compMapILargNum(lclNum); // account for possible hidden param + assertImp(lclNum < numArgs); - // Casts from floating point types must not have GTF_UNSIGNED set. - if (varTypeIsFloating(op1)) + if (lclNum == info.compThisArg) { - uns = false; + lclNum = lvaArg0Var; } - // At this point uns, ovf, callNode are all set. + // We should have seen this arg write in the prescan + assert(lvaTable[lclNum].lvHasILStoreOp); - if (varTypeIsSmall(lclTyp) && !ovfl && op1->gtType == TYP_INT && op1->gtOper == GT_AND) - { - op2 = op1->AsOp()->gtOp2; + goto VAR_ST; - if (op2->gtOper == GT_CNS_INT) - { - ssize_t ival = op2->AsIntCon()->gtIconVal; - ssize_t mask, umask; + case CEE_STLOC: + lclNum = getU2LittleEndian(codeAddr); + isLocal = true; + JITDUMP(" %u", lclNum); + goto LOC_ST; - switch (lclTyp) - { - case TYP_BYTE: - case TYP_UBYTE: - mask = 0x00FF; - umask = 0x007F; - break; - case TYP_USHORT: - case TYP_SHORT: - mask = 0xFFFF; - umask = 0x7FFF; - break; + case CEE_STLOC_S: + lclNum = getU1LittleEndian(codeAddr); + isLocal = true; + JITDUMP(" %u", lclNum); + goto LOC_ST; + + case CEE_STLOC_0: + case CEE_STLOC_1: + case CEE_STLOC_2: + case CEE_STLOC_3: + isLocal = true; + lclNum = (opcode - CEE_STLOC_0); + assert(lclNum >= 0 && lclNum < 4); - default: - assert(!"unexpected type"); - return; - } + LOC_ST: + if (compIsForInlining()) + { + lclTyp = impInlineInfo->lclVarInfo[lclNum + impInlineInfo->argCnt].lclTypeInfo; - if (((ival & umask) == ival) || ((ival & mask) == ival && uns)) - { - /* Toss the cast, it's a waste of time */ + /* Have we allocated a temp for this local? */ - impPushOnStack(op1, tiRetVal); - break; - } - else if (ival == mask) - { - /* Toss the masking, it's a waste of time, since - we sign-extend from the small value anyways */ + lclNum = impInlineFetchLocal(lclNum DEBUGARG("Inline stloc first use temp")); - op1 = op1->AsOp()->gtOp1; - } - } + goto _PopValue; } - /* The 'op2' sub-operand of a cast is the 'real' type number, - since the result of a cast to one of the 'small' integer - types is an integer. - */ + lclNum += numArgs; - type = genActualType(lclTyp); + VAR_ST: - // If this is a no-op cast, just use op1. - if (!ovfl && (type == op1->TypeGet()) && (genTypeSize(type) == genTypeSize(lclTyp))) + if (lclNum >= info.compLocalsCount && lclNum != lvaArg0Var) { - // Nothing needs to change + BADCODE("Bad IL"); } - // Work is evidently required, add cast node - else - { - if (callNode) - { - op1 = gtNewCastNodeL(type, op1, uns, lclTyp); - } - else - { - op1 = gtNewCastNode(type, op1, uns, lclTyp); - } - if (ovfl) - { - op1->gtFlags |= (GTF_OVERFLOW | GTF_EXCEPT); - } + VAR_ST_VALID: - if (op1->gtGetOp1()->OperIsConst() && opts.OptimizationEnabled()) - { - // Try and fold the introduced cast - op1 = gtFoldExprConst(op1); - } + /* if it is a struct assignment, make certain we don't overflow the buffer */ + assert(lclTyp != TYP_STRUCT || lvaLclSize(lclNum) >= info.compCompHnd->getClassSize(clsHnd)); + + if (lvaTable[lclNum].lvNormalizeOnLoad()) + { + lclTyp = lvaGetRealType(lclNum); + } + else + { + lclTyp = lvaGetActualType(lclNum); } - impPushOnStack(op1, tiRetVal); - break; + _PopValue: + /* Pop the value being assigned */ - case CEE_NEG: - op1 = impPopStack().val; - impBashVarAddrsToI(op1, nullptr); - impPushOnStack(gtNewOperNode(GT_NEG, genActualType(op1->gtType), op1), tiRetVal); - break; + { + StackEntry se = impPopStack(); + clsHnd = se.seTypeInfo.GetClassHandle(); + op1 = se.val; + tiRetVal = se.seTypeInfo; + } - case CEE_POP: - { - /* Pull the top value from the stack */ +#ifdef FEATURE_SIMD + if (varTypeIsSIMD(lclTyp) && (lclTyp != op1->TypeGet())) + { + assert(op1->TypeGet() == TYP_STRUCT); + op1->gtType = lclTyp; + } +#endif // FEATURE_SIMD - StackEntry se = impPopStack(); - clsHnd = se.seTypeInfo.GetClassHandle(); - op1 = se.val; + op1 = impImplicitIorI4Cast(op1, lclTyp); - /* Get hold of the type of the value being duplicated */ +#ifdef TARGET_64BIT + // Downcast the TYP_I_IMPL into a 32-bit Int for x86 JIT compatibility + if (varTypeIsI(op1->TypeGet()) && (genActualType(lclTyp) == TYP_INT)) + { + op1 = gtNewCastNode(TYP_INT, op1, false, TYP_INT); + } +#endif // TARGET_64BIT - lclTyp = genActualType(op1->gtType); + // We had better assign it a value of the correct type + assertImp( + genActualType(lclTyp) == genActualType(op1->gtType) || + (genActualType(lclTyp) == TYP_I_IMPL && op1->IsLocalAddrExpr() != nullptr) || + (genActualType(lclTyp) == TYP_I_IMPL && (op1->gtType == TYP_BYREF || op1->gtType == TYP_REF)) || + (genActualType(op1->gtType) == TYP_I_IMPL && lclTyp == TYP_BYREF) || + (varTypeIsFloating(lclTyp) && varTypeIsFloating(op1->TypeGet())) || + ((genActualType(lclTyp) == TYP_BYREF) && genActualType(op1->TypeGet()) == TYP_REF)); - /* Does the value have any side effects? */ + /* If op1 is "&var" then its type is the transient "*" and it can + be used either as TYP_BYREF or TYP_I_IMPL */ - if ((op1->gtFlags & GTF_SIDE_EFFECT) || opts.compDbgCode) + if (op1->IsLocalAddrExpr() != nullptr) { - // Since we are throwing away the value, just normalize - // it to its address. This is more efficient. - - if (varTypeIsStruct(op1)) - { - JITDUMP("\n ... CEE_POP struct ...\n"); - DISPTREE(op1); -#ifdef UNIX_AMD64_ABI - // Non-calls, such as obj or ret_expr, have to go through this. - // Calls with large struct return value have to go through this. - // Helper calls with small struct return value also have to go - // through this since they do not follow Unix calling convention. - if (op1->gtOper != GT_CALL || - !IsMultiRegReturnedType(clsHnd, op1->AsCall()->GetUnmanagedCallConv()) || - op1->AsCall()->gtCallType == CT_HELPER) -#endif // UNIX_AMD64_ABI - { - // If the value being produced comes from loading - // via an underlying address, just null check the address. - if (op1->OperIs(GT_FIELD, GT_IND, GT_OBJ)) - { - gtChangeOperToNullCheck(op1, block); - } - else - { - op1 = impGetStructAddr(op1, clsHnd, (unsigned)CHECK_SPILL_ALL, false); - } - - JITDUMP("\n ... optimized to ...\n"); - DISPTREE(op1); - } - } + assertImp(genActualType(lclTyp) == TYP_I_IMPL || lclTyp == TYP_BYREF); - // If op1 is non-overflow cast, throw it away since it is useless. - // Another reason for throwing away the useless cast is in the context of - // implicit tail calls when the operand of pop is GT_CAST(GT_CALL(..)). - // The cast gets added as part of importing GT_CALL, which gets in the way - // of fgMorphCall() on the forms of tail call nodes that we assert. - if ((op1->gtOper == GT_CAST) && !op1->gtOverflow()) - { - op1 = op1->AsOp()->gtOp1; - } + /* When "&var" is created, we assume it is a byref. If it is + being assigned to a TYP_I_IMPL var, change the type to + prevent unnecessary GC info */ - if (op1->gtOper != GT_CALL) + if (genActualType(lclTyp) == TYP_I_IMPL) { - if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0) - { - op1 = gtUnusedValNode(op1); - } - else - { - // Can't bash to NOP here because op1 can be referenced from `currentBlock->bbEntryState`, - // if we ever need to reimport we need a valid LCL_VAR on it. - op1 = gtNewNothingNode(); - } + op1->gtType = TYP_I_IMPL; } - - /* Append the value to the tree list */ - goto SPILL_APPEND; } - /* No side effects - just throw the thing away */ - } - break; + // If this is a local and the local is a ref type, see + // if we can improve type information based on the + // value being assigned. + if (isLocal && (lclTyp == TYP_REF)) + { + // We should have seen a stloc in our IL prescan. + assert(lvaTable[lclNum].lvHasILStoreOp); - case CEE_DUP: - { - StackEntry se = impPopStack(); - GenTree* tree = se.val; - tiRetVal = se.seTypeInfo; - op1 = tree; + // Is there just one place this local is defined? + const bool isSingleDefLocal = lvaTable[lclNum].lvSingleDef; - // If the expression to dup is simple, just clone it. - // Otherwise spill it to a temp, and reload the temp twice. - bool cloneExpr = false; + // Conservative check that there is just one + // definition that reaches this store. + const bool hasSingleReachingDef = (block->bbStackDepthOnEntry() == 0); - if (!opts.compDbgCode) - { - // Duplicate 0 and +0.0 - if (op1->IsIntegralConst(0) || op1->IsFloatPositiveZero()) - { - cloneExpr = true; - } - // Duplicate locals and addresses of them - else if (op1->IsLocal()) - { - cloneExpr = true; - } - else if (op1->TypeIs(TYP_BYREF) && op1->OperIs(GT_ADDR) && op1->gtGetOp1()->IsLocal() && - (OPCODE)impGetNonPrefixOpcode(codeAddr + sz, codeEndp) != CEE_INITOBJ) + if (isSingleDefLocal && hasSingleReachingDef) { - cloneExpr = true; + lvaUpdateClass(lclNum, op1, clsHnd); } } - else - { - // Always clone for debug mode - cloneExpr = true; - } - if (!cloneExpr) - { - const unsigned tmpNum = lvaGrabTemp(true DEBUGARG("dup spill")); - impAssignTempGen(tmpNum, op1, tiRetVal.GetClassHandle(), (unsigned)CHECK_SPILL_ALL); - var_types type = genActualType(lvaTable[tmpNum].TypeGet()); - op1 = gtNewLclvNode(tmpNum, type); + /* Filter out simple assignments to itself */ - // Propagate type info to the temp from the stack and the original tree - if (type == TYP_REF) + if (op1->gtOper == GT_LCL_VAR && lclNum == op1->AsLclVarCommon()->GetLclNum()) + { + if (opts.compDbgCode) { - assert(lvaTable[tmpNum].lvSingleDef == 0); - lvaTable[tmpNum].lvSingleDef = 1; - JITDUMP("Marked V%02u as a single def local\n", tmpNum); - lvaSetClass(tmpNum, tree, tiRetVal.GetClassHandle()); - } - } - - op1 = impCloneExpr(op1, &op2, tiRetVal.GetClassHandle(), (unsigned)CHECK_SPILL_ALL, - nullptr DEBUGARG("DUP instruction")); - - assert(!(op1->gtFlags & GTF_GLOB_EFFECT) && !(op2->gtFlags & GTF_GLOB_EFFECT)); - impPushOnStack(op1, tiRetVal); - impPushOnStack(op2, tiRetVal); - } - break; - - case CEE_STIND_I1: - lclTyp = TYP_BYTE; - goto STIND; - case CEE_STIND_I2: - lclTyp = TYP_SHORT; - goto STIND; - case CEE_STIND_I4: - lclTyp = TYP_INT; - goto STIND; - case CEE_STIND_I8: - lclTyp = TYP_LONG; - goto STIND; - case CEE_STIND_I: - lclTyp = TYP_I_IMPL; - goto STIND; - case CEE_STIND_REF: - lclTyp = TYP_REF; - goto STIND; - case CEE_STIND_R4: - lclTyp = TYP_FLOAT; - goto STIND; - case CEE_STIND_R8: - lclTyp = TYP_DOUBLE; - goto STIND; - STIND: - - op2 = impPopStack().val; // value to store - op1 = impPopStack().val; // address to store to + op1 = gtNewNothingNode(); + goto SPILL_APPEND; + } + else + { + break; + } + } - // you can indirect off of a TYP_I_IMPL (if we are in C) or a BYREF - assertImp(genActualType(op1->gtType) == TYP_I_IMPL || op1->gtType == TYP_BYREF); + op2 = gtNewLclvNode(lclNum, lclTyp DEBUGARG(opcodeOffs + sz + 1)); - impBashVarAddrsToI(op1, op2); + // Stores to pinned locals can have the implicit side effect of "unpinning", so we must spill + // things that could depend on the pin. TODO-Bug: which can actually be anything, including + // unpinned unaliased locals, not just side-effecting trees. + if (lvaTable[lclNum].lvPinned) + { + impSpillSideEffects(false, CHECK_SPILL_ALL DEBUGARG("Spill before store to pinned local")); + } - op2 = impImplicitR4orR8Cast(op2, lclTyp); + // We can generate an assignment to a TYP_FLOAT from a TYP_DOUBLE + // We insert a cast to the dest 'op2' type + // + if ((op1->TypeGet() != op2->TypeGet()) && varTypeIsFloating(op1->gtType) && + varTypeIsFloating(op2->gtType)) + { + op1 = gtNewCastNode(op2->TypeGet(), op1, false, op2->TypeGet()); + } -#ifdef TARGET_64BIT - // Automatic upcast for a GT_CNS_INT into TYP_I_IMPL - if ((op2->OperGet() == GT_CNS_INT) && varTypeIsI(lclTyp) && !varTypeIsI(op2->gtType)) + if (varTypeIsStruct(lclTyp)) { - op2->gtType = TYP_I_IMPL; + op1 = impAssignStruct(op2, op1, CHECK_SPILL_ALL); } else { - // Allow a downcast of op2 from TYP_I_IMPL into a 32-bit Int for x86 JIT compatiblity - // - if (varTypeIsI(op2->gtType) && (genActualType(lclTyp) == TYP_INT)) - { - op2 = gtNewCastNode(TYP_INT, op2, false, TYP_INT); - } - // Allow an upcast of op2 from a 32-bit Int into TYP_I_IMPL for x86 JIT compatiblity - // - if (varTypeIsI(lclTyp) && (genActualType(op2->gtType) == TYP_INT)) - { - op2 = gtNewCastNode(TYP_I_IMPL, op2, false, TYP_I_IMPL); - } + op1 = gtNewAssignNode(op2, op1); } -#endif // TARGET_64BIT + goto SPILL_APPEND; - if (opcode == CEE_STIND_REF) + case CEE_LDLOCA: + lclNum = getU2LittleEndian(codeAddr); + goto LDLOCA; + + case CEE_LDLOCA_S: + lclNum = getU1LittleEndian(codeAddr); + LDLOCA: + JITDUMP(" %u", lclNum); + + if (compIsForInlining()) { - // STIND_REF can be used to store TYP_INT, TYP_I_IMPL, TYP_REF, or TYP_BYREF - assertImp(varTypeIsIntOrI(op2->gtType) || varTypeIsGC(op2->gtType)); - lclTyp = genActualType(op2->TypeGet()); + // Get the local type + lclTyp = impInlineInfo->lclVarInfo[lclNum + impInlineInfo->argCnt].lclTypeInfo; + + /* Have we allocated a temp for this local? */ + + lclNum = impInlineFetchLocal(lclNum DEBUGARG("Inline ldloca(s) first use temp")); + + assert(!lvaGetDesc(lclNum)->lvNormalizeOnLoad()); + op1 = gtNewLclVarAddrNode(lclNum, TYP_BYREF); + goto _PUSH_ADRVAR; } -// Check target type. -#ifdef DEBUG - if (op2->gtType == TYP_BYREF || lclTyp == TYP_BYREF) + lclNum += numArgs; + assertImp(lclNum < info.compLocalsCount); + goto ADRVAR; + + case CEE_LDARGA: + lclNum = getU2LittleEndian(codeAddr); + goto LDARGA; + + case CEE_LDARGA_S: + lclNum = getU1LittleEndian(codeAddr); + LDARGA: + JITDUMP(" %u", lclNum); + Verify(lclNum < info.compILargsCount, "bad arg num"); + + if (compIsForInlining()) { - if (op2->gtType == TYP_BYREF) - { - assertImp(lclTyp == TYP_BYREF || lclTyp == TYP_I_IMPL); - } - else if (lclTyp == TYP_BYREF) + // In IL, LDARGA(_S) is used to load the byref managed pointer of struct argument, + // followed by a ldfld to load the field. + + op1 = impInlineFetchArg(lclNum, impInlineInfo->inlArgInfo, impInlineInfo->lclVarInfo); + if (op1->gtOper != GT_LCL_VAR) { - assertImp(op2->gtType == TYP_BYREF || varTypeIsIntOrI(op2->gtType)); + compInlineResult->NoteFatal(InlineObservation::CALLSITE_LDARGA_NOT_LOCAL_VAR); + return; } + + op1->ChangeType(TYP_BYREF); + op1->SetOper(GT_LCL_VAR_ADDR); + goto _PUSH_ADRVAR; } - else + + lclNum = compMapILargNum(lclNum); // account for possible hidden param + assertImp(lclNum < numArgs); + + if (lclNum == info.compThisArg) { - assertImp(genActualType(op2->gtType) == genActualType(lclTyp) || - ((lclTyp == TYP_I_IMPL) && (genActualType(op2->gtType) == TYP_INT)) || - (varTypeIsFloating(op2->gtType) && varTypeIsFloating(lclTyp))); + lclNum = lvaArg0Var; } -#endif - op1 = gtNewOperNode(GT_IND, lclTyp, op1); + goto ADRVAR; - if (prefixFlags & PREFIX_VOLATILE) + ADRVAR: + // Note that this is supposed to create the transient type "*" + // which may be used as a TYP_I_IMPL. However we catch places + // where it is used as a TYP_I_IMPL and change the node if needed. + // Thus we are pessimistic and may report byrefs in the GC info + // where it was not absolutely needed, but doing otherwise would + // require careful rethinking of the importer routines which use + // the IL validity model (e. g. "impGetByRefResultType"). + op1 = gtNewLclVarAddrNode(lclNum, TYP_BYREF); + + _PUSH_ADRVAR: + assert(op1->OperIs(GT_LCL_VAR_ADDR)); + + tiRetVal = typeInfo(TI_BYTE).MakeByRef(); + impPushOnStack(op1, tiRetVal); + break; + + case CEE_ARGLIST: + + if (!info.compIsVarArgs) { - assert(op1->OperGet() == GT_IND); - op1->gtFlags |= GTF_ORDER_SIDEEFF; // Prevent this from being reordered - op1->gtFlags |= GTF_IND_VOLATILE; + BADCODE("arglist in non-vararg method"); } - if ((prefixFlags & PREFIX_UNALIGNED) && !varTypeIsByte(lclTyp)) + assertImp((info.compMethodInfo->args.callConv & CORINFO_CALLCONV_MASK) == CORINFO_CALLCONV_VARARG); + + // The ARGLIST cookie is a hidden 'last' parameter, we have already + // adjusted the arg count cos this is like fetching the last param. + assertImp(numArgs > 0); + op1 = gtNewLclVarAddrNode(lvaVarargsHandleArg, TYP_BYREF); + impPushOnStack(op1, tiRetVal); + break; + + case CEE_ENDFINALLY: + + if (compIsForInlining()) { - assert(op1->OperGet() == GT_IND); - op1->gtFlags |= GTF_IND_UNALIGNED; + assert(!"Shouldn't have exception handlers in the inliner!"); + compInlineResult->NoteFatal(InlineObservation::CALLEE_HAS_ENDFINALLY); + return; } - op1 = gtNewAssignNode(op1, op2); - op1->gtFlags |= GTF_EXCEPT | GTF_GLOB_REF; - - // Spill side-effects AND global-data-accesses if (verCurrentState.esStackDepth > 0) { - impSpillSideEffects(true, (unsigned)CHECK_SPILL_ALL DEBUGARG("spill side effects before STIND")); + impEvalSideEffects(); } - goto APPEND; + if (info.compXcptnsCount == 0) + { + BADCODE("endfinally outside finally"); + } - case CEE_LDIND_I1: - lclTyp = TYP_BYTE; - goto LDIND; - case CEE_LDIND_I2: - lclTyp = TYP_SHORT; - goto LDIND; - case CEE_LDIND_U4: - case CEE_LDIND_I4: - lclTyp = TYP_INT; - goto LDIND; - case CEE_LDIND_I8: - lclTyp = TYP_LONG; - goto LDIND; - case CEE_LDIND_REF: - lclTyp = TYP_REF; - goto LDIND; - case CEE_LDIND_I: - lclTyp = TYP_I_IMPL; - goto LDIND; - case CEE_LDIND_R4: - lclTyp = TYP_FLOAT; - goto LDIND; - case CEE_LDIND_R8: - lclTyp = TYP_DOUBLE; - goto LDIND; - case CEE_LDIND_U1: - lclTyp = TYP_UBYTE; - goto LDIND; - case CEE_LDIND_U2: - lclTyp = TYP_USHORT; - goto LDIND; - LDIND: + assert(verCurrentState.esStackDepth == 0); - op1 = impPopStack().val; // address to load from - impBashVarAddrsToI(op1); + op1 = gtNewOperNode(GT_RETFILT, TYP_VOID, nullptr); + goto APPEND; -#ifdef TARGET_64BIT - // Allow an upcast of op1 from a 32-bit Int into TYP_I_IMPL for x86 JIT compatiblity - // - if (genActualType(op1->gtType) == TYP_INT) + case CEE_ENDFILTER: + + if (compIsForInlining()) { - op1 = gtNewCastNode(TYP_I_IMPL, op1, false, TYP_I_IMPL); + assert(!"Shouldn't have exception handlers in the inliner!"); + compInlineResult->NoteFatal(InlineObservation::CALLEE_HAS_ENDFILTER); + return; } -#endif - - assertImp(genActualType(op1->gtType) == TYP_I_IMPL || op1->gtType == TYP_BYREF); - - op1 = gtNewOperNode(GT_IND, lclTyp, op1); - // ldind could point anywhere, example a boxed class static int - op1->gtFlags |= (GTF_EXCEPT | GTF_GLOB_REF); + block->bbSetRunRarely(); // filters are rare - if (prefixFlags & PREFIX_VOLATILE) + if (info.compXcptnsCount == 0) { - assert(op1->OperGet() == GT_IND); - op1->gtFlags |= GTF_ORDER_SIDEEFF; // Prevent this from being reordered - op1->gtFlags |= GTF_IND_VOLATILE; + BADCODE("endfilter outside filter"); } - if ((prefixFlags & PREFIX_UNALIGNED) && !varTypeIsByte(lclTyp)) + op1 = impPopStack().val; + assertImp(op1->gtType == TYP_INT); + if (!bbInFilterILRange(block)) { - assert(op1->OperGet() == GT_IND); - op1->gtFlags |= GTF_IND_UNALIGNED; + BADCODE("EndFilter outside a filter handler"); } - impPushOnStack(op1, tiRetVal); + /* Mark current bb as end of filter */ - break; + assert(compCurBB->bbFlags & BBF_DONT_REMOVE); + assert(compCurBB->bbJumpKind == BBJ_EHFILTERRET); - case CEE_UNALIGNED: + /* Mark catch handler as successor */ - assert(sz == 1); - val = getU1LittleEndian(codeAddr); - ++codeAddr; - JITDUMP(" %u", val); - if ((val != 1) && (val != 2) && (val != 4)) + op1 = gtNewOperNode(GT_RETFILT, op1->TypeGet(), op1); + if (verCurrentState.esStackDepth != 0) { - BADCODE("Alignment unaligned. must be 1, 2, or 4"); + verRaiseVerifyException(INDEBUG("stack must be 1 on end of filter") DEBUGARG(__FILE__) + DEBUGARG(__LINE__)); } + goto APPEND; - Verify(!(prefixFlags & PREFIX_UNALIGNED), "Multiple unaligned. prefixes"); - prefixFlags |= PREFIX_UNALIGNED; + case CEE_RET: + prefixFlags &= ~PREFIX_TAILCALL; // ret without call before it + RET: + if (!impReturnInstruction(prefixFlags, opcode)) + { + return; // abort + } + else + { + break; + } - impValidateMemoryAccessOpcode(codeAddr, codeEndp, false); + case CEE_JMP: - PREFIX: - opcode = (OPCODE)getU1LittleEndian(codeAddr); - opcodeOffs = (IL_OFFSET)(codeAddr - info.compCode); - codeAddr += sizeof(__int8); - goto DECODE_OPCODE; + assert(!compIsForInlining()); - case CEE_VOLATILE: + if ((info.compFlags & CORINFO_FLG_SYNCH) || block->hasTryIndex() || block->hasHndIndex()) + { + /* CEE_JMP does not make sense in some "protected" regions. */ - Verify(!(prefixFlags & PREFIX_VOLATILE), "Multiple volatile. prefixes"); - prefixFlags |= PREFIX_VOLATILE; + BADCODE("Jmp not allowed in protected region"); + } - impValidateMemoryAccessOpcode(codeAddr, codeEndp, true); + if (opts.IsReversePInvoke()) + { + BADCODE("Jmp not allowed in reverse P/Invoke"); + } - assert(sz == 0); - goto PREFIX; + if (verCurrentState.esStackDepth != 0) + { + BADCODE("Stack must be empty after CEE_JMPs"); + } - case CEE_LDFTN: - { - // Need to do a lookup here so that we perform an access check - // and do a NOWAY if protections are violated _impResolveToken(CORINFO_TOKENKIND_Method); JITDUMP(" %08X", resolvedToken.token); - eeGetCallInfo(&resolvedToken, (prefixFlags & PREFIX_CONSTRAINED) ? &constrainedResolvedToken : nullptr, - combine(CORINFO_CALLINFO_SECURITYCHECKS, CORINFO_CALLINFO_LDFTN), &callInfo); + /* The signature of the target has to be identical to ours. + At least check that argCnt and returnType match */ - // This check really only applies to intrinsic Array.Address methods - if (callInfo.sig.callConv & CORINFO_CALLCONV_PARAMTYPE) + eeGetMethodSig(resolvedToken.hMethod, &sig); + if (sig.numArgs != info.compMethodInfo->args.numArgs || + sig.retType != info.compMethodInfo->args.retType || + sig.callConv != info.compMethodInfo->args.callConv) { - NO_WAY("Currently do not support LDFTN of Parameterized functions"); + BADCODE("Incompatible target for CEE_JMPs"); } - // Do this before DO_LDFTN since CEE_LDVIRTFN does it on its own. - impHandleAccessAllowed(callInfo.accessAllowed, &callInfo.callsiteCalloutHelper); + op1 = new (this, GT_JMP) GenTreeVal(GT_JMP, TYP_VOID, (size_t)resolvedToken.hMethod); - DO_LDFTN: - op1 = impMethodPointer(&resolvedToken, &callInfo); + /* Mark the basic block as being a JUMP instead of RETURN */ - if (compDonotInline()) - { - return; - } + block->bbFlags |= BBF_HAS_JMP; - // Call info may have more precise information about the function than - // the resolved token. - mdToken constrainedToken = prefixFlags & PREFIX_CONSTRAINED ? constrainedResolvedToken.token : 0; - methodPointerInfo* heapToken = impAllocateMethodPointerInfo(resolvedToken, constrainedToken); - assert(callInfo.hMethod != nullptr); - heapToken->m_token.hMethod = callInfo.hMethod; - impPushOnStack(op1, typeInfo(heapToken)); + /* Set this flag to make sure register arguments have a location assigned + * even if we don't use them inside the method */ - break; - } + compJmpOpUsed = true; - case CEE_LDVIRTFTN: + fgNoStructPromotion = true; + + goto APPEND; + + case CEE_LDELEMA: { - /* Get the method token */ + assertImp(sz == sizeof(unsigned)); - _impResolveToken(CORINFO_TOKENKIND_Method); + _impResolveToken(CORINFO_TOKENKIND_Class); JITDUMP(" %08X", resolvedToken.token); - eeGetCallInfo(&resolvedToken, nullptr /* constraint typeRef */, - combine(combine(CORINFO_CALLINFO_SECURITYCHECKS, CORINFO_CALLINFO_LDFTN), - CORINFO_CALLINFO_CALLVIRT), - &callInfo); + ldelemClsHnd = resolvedToken.hClass; + lclTyp = JITtype2varType(info.compCompHnd->asCorInfoType(ldelemClsHnd)); - // This check really only applies to intrinsic Array.Address methods - if (callInfo.sig.callConv & CORINFO_CALLCONV_PARAMTYPE) + // If it's a value class / pointer array, or a readonly access, we don't need a type check. + // TODO-CQ: adapt "impCanSkipCovariantStoreCheck" to handle "ldelema"s and call it here to + // skip using the helper in more cases. + if ((lclTyp != TYP_REF) || ((prefixFlags & PREFIX_READONLY) != 0)) { - NO_WAY("Currently do not support LDFTN of Parameterized functions"); + goto ARR_LD; } - mflags = callInfo.methodFlags; - - impHandleAccessAllowed(callInfo.accessAllowed, &callInfo.callsiteCalloutHelper); - - if (compIsForInlining()) + // Otherwise we need the full helper function with run-time type check + GenTree* type = impTokenToHandle(&resolvedToken); + if (type == nullptr) { - if (mflags & (CORINFO_FLG_FINAL | CORINFO_FLG_STATIC) || !(mflags & CORINFO_FLG_VIRTUAL)) - { - compInlineResult->NoteFatal(InlineObservation::CALLSITE_LDVIRTFN_ON_NON_VIRTUAL); - return; - } + assert(compDonotInline()); + return; } - CORINFO_SIG_INFO& ftnSig = callInfo.sig; - - /* Get the object-ref */ - op1 = impPopStack().val; - assertImp(op1->gtType == TYP_REF); + GenTree* index = impPopStack().val; + GenTree* arr = impPopStack().val; - if (opts.IsReadyToRun()) +#ifdef TARGET_64BIT + // The CLI Spec allows an array to be indexed by either an int32 or a native int. + // The array helper takes a native int for array length. + // So if we have an int, explicitly extend it to be a native int. + if (genActualType(index->TypeGet()) != TYP_I_IMPL) { - if (callInfo.kind != CORINFO_VIRTUALCALL_LDVIRTFTN) + if (index->IsIntegralConst()) { - if (op1->gtFlags & GTF_SIDE_EFFECT) - { - op1 = gtUnusedValNode(op1); - impAppendTree(op1, (unsigned)CHECK_SPILL_ALL, impCurStmtDI); - } - goto DO_LDFTN; + index->gtType = TYP_I_IMPL; } - } - else if (mflags & (CORINFO_FLG_FINAL | CORINFO_FLG_STATIC) || !(mflags & CORINFO_FLG_VIRTUAL)) - { - if (op1->gtFlags & GTF_SIDE_EFFECT) + else { - op1 = gtUnusedValNode(op1); - impAppendTree(op1, (unsigned)CHECK_SPILL_ALL, impCurStmtDI); + bool isUnsigned = false; + index = gtNewCastNode(TYP_I_IMPL, index, isUnsigned, TYP_I_IMPL); } - goto DO_LDFTN; } +#endif // TARGET_64BIT - GenTree* fptr = impImportLdvirtftn(op1, &resolvedToken, &callInfo); - if (compDonotInline()) + op1 = gtNewHelperCallNode(CORINFO_HELP_LDELEMA_REF, TYP_BYREF, arr, index, type); + impPushOnStack(op1, tiRetVal); + } + break; + + // ldelem for reference and value types + case CEE_LDELEM: + assertImp(sz == sizeof(unsigned)); + + _impResolveToken(CORINFO_TOKENKIND_Class); + + JITDUMP(" %08X", resolvedToken.token); + + ldelemClsHnd = resolvedToken.hClass; + lclTyp = JITtype2varType(info.compCompHnd->asCorInfoType(ldelemClsHnd)); + tiRetVal = verMakeTypeInfo(ldelemClsHnd); + tiRetVal.NormaliseForStack(); + goto ARR_LD; + + case CEE_LDELEM_I1: + lclTyp = TYP_BYTE; + goto ARR_LD; + case CEE_LDELEM_I2: + lclTyp = TYP_SHORT; + goto ARR_LD; + case CEE_LDELEM_I: + lclTyp = TYP_I_IMPL; + goto ARR_LD; + case CEE_LDELEM_U4: + lclTyp = TYP_INT; + goto ARR_LD; + case CEE_LDELEM_I4: + lclTyp = TYP_INT; + goto ARR_LD; + case CEE_LDELEM_I8: + lclTyp = TYP_LONG; + goto ARR_LD; + case CEE_LDELEM_REF: + lclTyp = TYP_REF; + goto ARR_LD; + case CEE_LDELEM_R4: + lclTyp = TYP_FLOAT; + goto ARR_LD; + case CEE_LDELEM_R8: + lclTyp = TYP_DOUBLE; + goto ARR_LD; + case CEE_LDELEM_U1: + lclTyp = TYP_UBYTE; + goto ARR_LD; + case CEE_LDELEM_U2: + lclTyp = TYP_USHORT; + goto ARR_LD; + + ARR_LD: + + op2 = impPopStack().val; // index + op1 = impPopStack().val; // array + assertImp(op1->TypeIs(TYP_REF)); + + // Check for null pointer - in the inliner case we simply abort. + if (compIsForInlining() && op1->IsCnsIntOrI()) { + compInlineResult->NoteFatal(InlineObservation::CALLEE_HAS_NULL_FOR_LDELEM); return; } - methodPointerInfo* heapToken = impAllocateMethodPointerInfo(resolvedToken, 0); + // Mark the block as containing an index expression. - assert(heapToken->m_token.tokenType == CORINFO_TOKENKIND_Method); - assert(callInfo.hMethod != nullptr); + if (op1->OperIs(GT_LCL_VAR) && op2->OperIs(GT_LCL_VAR, GT_CNS_INT, GT_ADD)) + { + block->bbFlags |= BBF_HAS_IDX_LEN; + optMethodFlags |= OMF_HAS_ARRAYREF; + } - heapToken->m_token.tokenType = CORINFO_TOKENKIND_Ldvirtftn; - heapToken->m_token.hMethod = callInfo.hMethod; - impPushOnStack(fptr, typeInfo(heapToken)); + op1 = gtNewArrayIndexAddr(op1, op2, lclTyp, ldelemClsHnd); + + if (opcode != CEE_LDELEMA) + { + op1 = gtNewIndexIndir(op1->AsIndexAddr()); + } + impPushOnStack(op1, tiRetVal); break; - } - case CEE_CONSTRAINED: + // stelem for reference and value types + case CEE_STELEM: assertImp(sz == sizeof(unsigned)); - impResolveToken(codeAddr, &constrainedResolvedToken, CORINFO_TOKENKIND_Constrained); - codeAddr += sizeof(unsigned); // prefix instructions must increment codeAddr manually - JITDUMP(" (%08X) ", constrainedResolvedToken.token); - Verify(!(prefixFlags & PREFIX_CONSTRAINED), "Multiple constrained. prefixes"); - prefixFlags |= PREFIX_CONSTRAINED; + _impResolveToken(CORINFO_TOKENKIND_Class); - { - OPCODE actualOpcode = impGetNonPrefixOpcode(codeAddr, codeEndp); - if (actualOpcode != CEE_CALLVIRT && actualOpcode != CEE_CALL && actualOpcode != CEE_LDFTN) - { - BADCODE("constrained. has to be followed by callvirt, call or ldftn"); - } - } + JITDUMP(" %08X", resolvedToken.token); - goto PREFIX; + stelemClsHnd = resolvedToken.hClass; + lclTyp = JITtype2varType(info.compCompHnd->asCorInfoType(stelemClsHnd)); - case CEE_READONLY: - JITDUMP(" readonly."); + if (lclTyp != TYP_REF) + { + goto ARR_ST; + } + FALLTHROUGH; - Verify(!(prefixFlags & PREFIX_READONLY), "Multiple readonly. prefixes"); - prefixFlags |= PREFIX_READONLY; + case CEE_STELEM_REF: + { + GenTree* value = impStackTop(0).val; + GenTree* index = impStackTop(1).val; + GenTree* array = impStackTop(2).val; + if (opts.OptimizationEnabled()) { - OPCODE actualOpcode = impGetNonPrefixOpcode(codeAddr, codeEndp); - if (actualOpcode != CEE_LDELEMA && !impOpcodeIsCallOpcode(actualOpcode)) + // Is this a case where we can skip the covariant store check? + if (impCanSkipCovariantStoreCheck(value, array)) { - BADCODE("readonly. has to be followed by ldelema or call"); + lclTyp = TYP_REF; + goto ARR_ST; } } - assert(sz == 0); - goto PREFIX; - - case CEE_TAILCALL: - JITDUMP(" tail."); - - Verify(!(prefixFlags & PREFIX_TAILCALL_EXPLICIT), "Multiple tailcall. prefixes"); - prefixFlags |= PREFIX_TAILCALL_EXPLICIT; + impPopStack(3); +// Else call a helper function to do the assignment +#ifdef TARGET_64BIT + // The CLI Spec allows an array to be indexed by either an int32 or a native int. + // The array helper takes a native int for array length. + // So if we have an int, explicitly extend it to be a native int. + if (genActualType(index->TypeGet()) != TYP_I_IMPL) { - OPCODE actualOpcode = impGetNonPrefixOpcode(codeAddr, codeEndp); - if (!impOpcodeIsCallOpcode(actualOpcode)) + if (index->IsIntegralConst()) { - BADCODE("tailcall. has to be followed by call, callvirt or calli"); + index->gtType = TYP_I_IMPL; + } + else + { + bool isUnsigned = false; + index = gtNewCastNode(TYP_I_IMPL, index, isUnsigned, TYP_I_IMPL); } } - assert(sz == 0); - goto PREFIX; - - case CEE_NEWOBJ: - - /* Since we will implicitly insert newObjThisPtr at the start of the - argument list, spill any GTF_ORDER_SIDEEFF */ - impSpillSpecialSideEff(); - - /* NEWOBJ does not respond to TAIL */ - prefixFlags &= ~PREFIX_TAILCALL_EXPLICIT; - - /* NEWOBJ does not respond to CONSTRAINED */ - prefixFlags &= ~PREFIX_CONSTRAINED; - - _impResolveToken(CORINFO_TOKENKIND_NewObj); +#endif // TARGET_64BIT + op1 = gtNewHelperCallNode(CORINFO_HELP_ARRADDR_ST, TYP_VOID, array, index, value); + goto SPILL_APPEND; + } - eeGetCallInfo(&resolvedToken, nullptr /* constraint typeRef*/, - combine(CORINFO_CALLINFO_SECURITYCHECKS, CORINFO_CALLINFO_ALLOWINSTPARAM), &callInfo); + case CEE_STELEM_I1: + lclTyp = TYP_BYTE; + goto ARR_ST; + case CEE_STELEM_I2: + lclTyp = TYP_SHORT; + goto ARR_ST; + case CEE_STELEM_I: + lclTyp = TYP_I_IMPL; + goto ARR_ST; + case CEE_STELEM_I4: + lclTyp = TYP_INT; + goto ARR_ST; + case CEE_STELEM_I8: + lclTyp = TYP_LONG; + goto ARR_ST; + case CEE_STELEM_R4: + lclTyp = TYP_FLOAT; + goto ARR_ST; + case CEE_STELEM_R8: + lclTyp = TYP_DOUBLE; + goto ARR_ST; - mflags = callInfo.methodFlags; + ARR_ST: + // TODO-Review: this comment is no longer correct. + /* The strict order of evaluation is LHS-operands, RHS-operands, + range-check, and then assignment. However, codegen currently + does the range-check before evaluation the RHS-operands. So to + maintain strict ordering, we spill the stack. */ - if ((mflags & (CORINFO_FLG_STATIC | CORINFO_FLG_ABSTRACT)) != 0) + if (impStackTop().val->gtFlags & GTF_SIDE_EFFECT) { - BADCODE("newobj on static or abstract method"); + impSpillSideEffects(false, + CHECK_SPILL_ALL DEBUGARG("Strict ordering of exceptions for Array store")); } - // Insert the security callout before any actual code is generated - impHandleAccessAllowed(callInfo.accessAllowed, &callInfo.callsiteCalloutHelper); + // Pull the new value from the stack. + op2 = impPopStack().val; + impBashVarAddrsToI(op2); - // There are three different cases for new. - // Object size is variable (depends on arguments). - // 1) Object is an array (arrays treated specially by the EE) - // 2) Object is some other variable sized object (e.g. String) - // 3) Class Size can be determined beforehand (normal case) - // In the first case, we need to call a NEWOBJ helper (multinewarray). - // In the second case we call the constructor with a '0' this pointer. - // In the third case we alloc the memory, then call the constructor. + // Pull the index value. + op1 = impPopStack().val; - clsFlags = callInfo.classFlags; - if (clsFlags & CORINFO_FLG_ARRAY) + // Pull the array address. + op3 = impPopStack().val; + assertImp(op3->TypeIs(TYP_REF)); + + // Mark the block as containing an index expression + if (op3->OperIs(GT_LCL_VAR) && op1->OperIs(GT_LCL_VAR, GT_CNS_INT, GT_ADD)) { - // Arrays need to call the NEWOBJ helper. - assertImp(clsFlags & CORINFO_FLG_VAROBJSIZE); + block->bbFlags |= BBF_HAS_IDX_LEN; + optMethodFlags |= OMF_HAS_ARRAYREF; + } - impImportNewObjArray(&resolvedToken, &callInfo); - if (compDonotInline()) - { - return; - } + // Create the index node. + op1 = gtNewArrayIndexAddr(op3, op1, lclTyp, stelemClsHnd); + op1 = gtNewIndexIndir(op1->AsIndexAddr()); - callTyp = TYP_REF; - break; - } - // At present this can only be String - else if (clsFlags & CORINFO_FLG_VAROBJSIZE) + // Create the assignment node and append it. + if (varTypeIsStruct(op1)) { - // Skip this thisPtr argument - newObjThisPtr = nullptr; - - /* Remember that this basic block contains 'new' of an object */ - block->bbFlags |= BBF_HAS_NEWOBJ; - optMethodFlags |= OMF_HAS_NEWOBJ; + op1 = impAssignStruct(op1, op2, CHECK_SPILL_ALL); } else { - // This is the normal case where the size of the object is - // fixed. Allocate the memory and call the constructor. - - // Note: We cannot add a peep to avoid use of temp here - // becase we don't have enough interference info to detect when - // sources and destination interfere, example: s = new S(ref); - - // TODO: We find the correct place to introduce a general - // reverse copy prop for struct return values from newobj or - // any function returning structs. - - /* get a temporary for the new object */ - lclNum = lvaGrabTemp(true DEBUGARG("NewObj constructor temp")); - if (compDonotInline()) - { - // Fail fast if lvaGrabTemp fails with CALLSITE_TOO_MANY_LOCALS. - assert(compInlineResult->GetObservation() == InlineObservation::CALLSITE_TOO_MANY_LOCALS); - return; - } + op2 = impImplicitR4orR8Cast(op2, op1->TypeGet()); + op1 = gtNewAssignNode(op1, op2); + } + goto SPILL_APPEND; - // In the value class case we only need clsHnd for size calcs. - // - // The lookup of the code pointer will be handled by CALL in this case - if (clsFlags & CORINFO_FLG_VALUECLASS) - { - if (compIsForInlining()) - { - // If value class has GC fields, inform the inliner. It may choose to - // bail out on the inline. - DWORD typeFlags = info.compCompHnd->getClassAttribs(resolvedToken.hClass); - if ((typeFlags & CORINFO_FLG_CONTAINS_GC_PTR) != 0) - { - compInlineResult->Note(InlineObservation::CALLEE_HAS_GC_STRUCT); - if (compInlineResult->IsFailure()) - { - return; - } + case CEE_ADD: + oper = GT_ADD; + goto MATH_OP2; - // Do further notification in the case where the call site is rare; - // some policies do not track the relative hotness of call sites for - // "always" inline cases. - if (impInlineInfo->iciBlock->isRunRarely()) - { - compInlineResult->Note(InlineObservation::CALLSITE_RARE_GC_STRUCT); - if (compInlineResult->IsFailure()) - { - return; - } - } - } - } + case CEE_ADD_OVF: + uns = false; + goto ADD_OVF; + case CEE_ADD_OVF_UN: + uns = true; + goto ADD_OVF; - CorInfoType jitTyp = info.compCompHnd->asCorInfoType(resolvedToken.hClass); + ADD_OVF: + ovfl = true; + callNode = false; + oper = GT_ADD; + goto MATH_OP2_FLAGS; - if (impIsPrimitive(jitTyp)) - { - lvaTable[lclNum].lvType = JITtype2varType(jitTyp); - } - else - { - // The local variable itself is the allocated space. - // Here we need unsafe value cls check, since the address of struct is taken for further use - // and potentially exploitable. - lvaSetStruct(lclNum, resolvedToken.hClass, true /* unsafe value cls check */); - } + case CEE_SUB: + oper = GT_SUB; + goto MATH_OP2; - bool bbInALoop = impBlockIsInALoop(block); - bool bbIsReturn = (block->bbJumpKind == BBJ_RETURN) && - (!compIsForInlining() || (impInlineInfo->iciBlock->bbJumpKind == BBJ_RETURN)); - LclVarDsc* const lclDsc = lvaGetDesc(lclNum); - if (fgVarNeedsExplicitZeroInit(lclNum, bbInALoop, bbIsReturn)) - { - // Append a tree to zero-out the temp - newObjThisPtr = gtNewLclvNode(lclNum, lclDsc->TypeGet()); + case CEE_SUB_OVF: + uns = false; + goto SUB_OVF; + case CEE_SUB_OVF_UN: + uns = true; + goto SUB_OVF; - newObjThisPtr = gtNewBlkOpNode(newObjThisPtr, // Dest - gtNewIconNode(0), // Value - false, // isVolatile - false); // not copyBlock - impAppendTree(newObjThisPtr, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); - } - else - { - JITDUMP("\nSuppressing zero-init for V%02u -- expect to zero in prolog\n", lclNum); - lclDsc->lvSuppressedZeroInit = 1; - compSuppressedZeroInit = true; - } + SUB_OVF: + ovfl = true; + callNode = false; + oper = GT_SUB; + goto MATH_OP2_FLAGS; - // Obtain the address of the temp - newObjThisPtr = - gtNewOperNode(GT_ADDR, TYP_BYREF, gtNewLclvNode(lclNum, lvaTable[lclNum].TypeGet())); - } - else - { - // If we're newing up a finalizable object, spill anything that can cause exceptions. - // - bool hasSideEffects = false; - CorInfoHelpFunc newHelper = - info.compCompHnd->getNewHelper(&resolvedToken, info.compMethodHnd, &hasSideEffects); + case CEE_MUL: + oper = GT_MUL; + goto MATH_MAYBE_CALL_NO_OVF; - if (hasSideEffects) - { - JITDUMP("\nSpilling stack for finalizable newobj\n"); - impSpillSideEffects(true, (unsigned)CHECK_SPILL_ALL DEBUGARG("finalizable newobj spill")); - } + case CEE_MUL_OVF: + uns = false; + goto MUL_OVF; + case CEE_MUL_OVF_UN: + uns = true; + goto MUL_OVF; - const bool useParent = true; - op1 = gtNewAllocObjNode(&resolvedToken, useParent); - if (op1 == nullptr) - { - return; - } + MUL_OVF: + ovfl = true; + oper = GT_MUL; + goto MATH_MAYBE_CALL_OVF; - // Remember that this basic block contains 'new' of an object - block->bbFlags |= BBF_HAS_NEWOBJ; - optMethodFlags |= OMF_HAS_NEWOBJ; + // Other binary math operations - // Append the assignment to the temp/local. Dont need to spill - // at all as we are just calling an EE-Jit helper which can only - // cause an (async) OutOfMemoryException. + case CEE_DIV: + oper = GT_DIV; + goto MATH_MAYBE_CALL_NO_OVF; - // We assign the newly allocated object (by a GT_ALLOCOBJ node) - // to a temp. Note that the pattern "temp = allocObj" is required - // by ObjectAllocator phase to be able to determine GT_ALLOCOBJ nodes - // without exhaustive walk over all expressions. + case CEE_DIV_UN: + oper = GT_UDIV; + goto MATH_MAYBE_CALL_NO_OVF; - impAssignTempGen(lclNum, op1, (unsigned)CHECK_SPILL_NONE); + case CEE_REM: + oper = GT_MOD; + goto MATH_MAYBE_CALL_NO_OVF; - assert(lvaTable[lclNum].lvSingleDef == 0); - lvaTable[lclNum].lvSingleDef = 1; - JITDUMP("Marked V%02u as a single def local\n", lclNum); - lvaSetClass(lclNum, resolvedToken.hClass, true /* is Exact */); + case CEE_REM_UN: + oper = GT_UMOD; + goto MATH_MAYBE_CALL_NO_OVF; - newObjThisPtr = gtNewLclvNode(lclNum, TYP_REF); - } - } - goto CALL; + MATH_MAYBE_CALL_NO_OVF: + ovfl = false; + MATH_MAYBE_CALL_OVF: + // Morpher has some complex logic about when to turn different + // typed nodes on different platforms into helper calls. We + // need to either duplicate that logic here, or just + // pessimistically make all the nodes large enough to become + // call nodes. Since call nodes aren't that much larger and + // these opcodes are infrequent enough I chose the latter. + callNode = true; + goto MATH_OP2_FLAGS; - case CEE_CALLI: + case CEE_AND: + oper = GT_AND; + goto MATH_OP2; + case CEE_OR: + oper = GT_OR; + goto MATH_OP2; + case CEE_XOR: + oper = GT_XOR; + goto MATH_OP2; - /* CALLI does not respond to CONSTRAINED */ - prefixFlags &= ~PREFIX_CONSTRAINED; + MATH_OP2: // For default values of 'ovfl' and 'callNode' - FALLTHROUGH; + ovfl = false; + callNode = false; - case CEE_CALLVIRT: - case CEE_CALL: + MATH_OP2_FLAGS: // If 'ovfl' and 'callNode' have already been set - // We can't call getCallInfo on the token from a CALLI, but we need it in - // many other places. We unfortunately embed that knowledge here. - if (opcode != CEE_CALLI) - { - _impResolveToken(CORINFO_TOKENKIND_Method); + /* Pull two values and push back the result */ - eeGetCallInfo(&resolvedToken, - (prefixFlags & PREFIX_CONSTRAINED) ? &constrainedResolvedToken : nullptr, - // this is how impImportCall invokes getCallInfo - combine(combine(CORINFO_CALLINFO_ALLOWINSTPARAM, CORINFO_CALLINFO_SECURITYCHECKS), - (opcode == CEE_CALLVIRT) ? CORINFO_CALLINFO_CALLVIRT : CORINFO_CALLINFO_NONE), - &callInfo); - } - else - { - // Suppress uninitialized use warning. - memset(&resolvedToken, 0, sizeof(resolvedToken)); - memset(&callInfo, 0, sizeof(callInfo)); + op2 = impPopStack().val; + op1 = impPopStack().val; - resolvedToken.token = getU4LittleEndian(codeAddr); - resolvedToken.tokenContext = impTokenLookupContextHandle; - resolvedToken.tokenScope = info.compScopeHnd; - } + /* Can't do arithmetic with references */ + assertImp(genActualType(op1->TypeGet()) != TYP_REF && genActualType(op2->TypeGet()) != TYP_REF); - CALL: // memberRef should be set. - // newObjThisPtr should be set for CEE_NEWOBJ + // Change both to TYP_I_IMPL (impBashVarAddrsToI won't change if its a true byref, only + // if it is in the stack) + impBashVarAddrsToI(op1, op2); - JITDUMP(" %08X", resolvedToken.token); - constraintCall = (prefixFlags & PREFIX_CONSTRAINED) != 0; + type = impGetByRefResultType(oper, uns, &op1, &op2); - bool newBBcreatedForTailcallStress; - bool passedStressModeValidation; + assert(!ovfl || !varTypeIsFloating(op1->gtType)); - newBBcreatedForTailcallStress = false; - passedStressModeValidation = true; + /* Special case: "int+0", "int-0", "int*1", "int/1" */ - if (compIsForInlining()) - { - if (compDonotInline()) - { - return; - } - // We rule out inlinees with explicit tail calls in fgMakeBasicBlocks. - assert((prefixFlags & PREFIX_TAILCALL_EXPLICIT) == 0); - } - else + if (op2->gtOper == GT_CNS_INT) { - if (compTailCallStress()) - { - // Have we created a new BB after the "call" instruction in fgMakeBasicBlocks()? - // Tail call stress only recognizes call+ret patterns and forces them to be - // explicit tail prefixed calls. Also fgMakeBasicBlocks() under tail call stress - // doesn't import 'ret' opcode following the call into the basic block containing - // the call instead imports it to a new basic block. Note that fgMakeBasicBlocks() - // is already checking that there is an opcode following call and hence it is - // safe here to read next opcode without bounds check. - newBBcreatedForTailcallStress = - impOpcodeIsCallOpcode(opcode) && // Current opcode is a CALL, (not a CEE_NEWOBJ). So, don't - // make it jump to RET. - (OPCODE)getU1LittleEndian(codeAddr + sz) == CEE_RET; // Next opcode is a CEE_RET - - bool hasTailPrefix = (prefixFlags & PREFIX_TAILCALL_EXPLICIT); - if (newBBcreatedForTailcallStress && !hasTailPrefix) - { - // Do a more detailed evaluation of legality - const bool returnFalseIfInvalid = true; - const bool passedConstraintCheck = - verCheckTailCallConstraint(opcode, &resolvedToken, - constraintCall ? &constrainedResolvedToken : nullptr, - returnFalseIfInvalid); + if ((op2->IsIntegralConst(0) && (oper == GT_ADD || oper == GT_SUB)) || + (op2->IsIntegralConst(1) && (oper == GT_MUL || oper == GT_DIV))) - if (passedConstraintCheck) - { - // Now check with the runtime - CORINFO_METHOD_HANDLE declaredCalleeHnd = callInfo.hMethod; - bool isVirtual = (callInfo.kind == CORINFO_VIRTUALCALL_STUB) || - (callInfo.kind == CORINFO_VIRTUALCALL_VTABLE); - CORINFO_METHOD_HANDLE exactCalleeHnd = isVirtual ? nullptr : declaredCalleeHnd; - if (info.compCompHnd->canTailCall(info.compMethodHnd, declaredCalleeHnd, exactCalleeHnd, - hasTailPrefix)) // Is it legal to do tailcall? - { - // Stress the tailcall. - JITDUMP(" (Tailcall stress: prefixFlags |= PREFIX_TAILCALL_EXPLICIT)"); - prefixFlags |= PREFIX_TAILCALL_EXPLICIT; - prefixFlags |= PREFIX_TAILCALL_STRESS; - } - else - { - // Runtime disallows this tail call - JITDUMP(" (Tailcall stress: runtime preventing tailcall)"); - passedStressModeValidation = false; - } - } - else - { - // Constraints disallow this tail call - JITDUMP(" (Tailcall stress: constraint check failed)"); - passedStressModeValidation = false; - } - } + { + impPushOnStack(op1, tiRetVal); + break; } } - // This is split up to avoid goto flow warnings. - bool isRecursive; - isRecursive = !compIsForInlining() && (callInfo.hMethod == info.compMethodHnd); - - // If we've already disqualified this call as a tail call under tail call stress, - // don't consider it for implicit tail calling either. - // - // When not running under tail call stress, we may mark this call as an implicit - // tail call candidate. We'll do an "equivalent" validation during impImportCall. + // We can generate a TYP_FLOAT operation that has a TYP_DOUBLE operand // - // Note that when running under tail call stress, a call marked as explicit - // tail prefixed will not be considered for implicit tail calling. - if (passedStressModeValidation && - impIsImplicitTailCallCandidate(opcode, codeAddr + sz, codeEndp, prefixFlags, isRecursive)) + if (varTypeIsFloating(type) && varTypeIsFloating(op1->gtType) && varTypeIsFloating(op2->gtType)) { - if (compIsForInlining()) + if (op1->TypeGet() != type) { -#if FEATURE_TAILCALL_OPT_SHARED_RETURN - // Are we inlining at an implicit tail call site? If so the we can flag - // implicit tail call sites in the inline body. These call sites - // often end up in non BBJ_RETURN blocks, so only flag them when - // we're able to handle shared returns. - if (impInlineInfo->iciCall->IsImplicitTailCall()) - { - JITDUMP("\n (Inline Implicit Tail call: prefixFlags |= PREFIX_TAILCALL_IMPLICIT)"); - prefixFlags |= PREFIX_TAILCALL_IMPLICIT; - } -#endif // FEATURE_TAILCALL_OPT_SHARED_RETURN + // We insert a cast of op1 to 'type' + op1 = gtNewCastNode(type, op1, false, type); } - else + if (op2->TypeGet() != type) { - JITDUMP("\n (Implicit Tail call: prefixFlags |= PREFIX_TAILCALL_IMPLICIT)"); - prefixFlags |= PREFIX_TAILCALL_IMPLICIT; + // We insert a cast of op2 to 'type' + op2 = gtNewCastNode(type, op2, false, type); } } - // Treat this call as tail call for verification only if "tail" prefixed (i.e. explicit tail call). - explicitTailCall = (prefixFlags & PREFIX_TAILCALL_EXPLICIT) != 0; - readonlyCall = (prefixFlags & PREFIX_READONLY) != 0; - - if (opcode != CEE_CALLI && opcode != CEE_NEWOBJ) - { - // All calls and delegates need a security callout. - // For delegates, this is the call to the delegate constructor, not the access check on the - // LD(virt)FTN. - impHandleAccessAllowed(callInfo.accessAllowed, &callInfo.callsiteCalloutHelper); - } - - callTyp = impImportCall(opcode, &resolvedToken, constraintCall ? &constrainedResolvedToken : nullptr, - newObjThisPtr, prefixFlags, &callInfo, opcodeOffs); - if (compDonotInline()) - { - // We do not check fails after lvaGrabTemp. It is covered with CoreCLR_13272 issue. - assert((callTyp == TYP_UNDEF) || - (compInlineResult->GetObservation() == InlineObservation::CALLSITE_TOO_MANY_LOCALS)); - return; - } - - if (explicitTailCall || newBBcreatedForTailcallStress) // If newBBcreatedForTailcallStress is true, we - // have created a new BB after the "call" - // instruction in fgMakeBasicBlocks(). So we need to jump to RET regardless. + if (callNode) { - assert(!compIsForInlining()); - goto RET; - } - - break; - - case CEE_LDFLD: - case CEE_LDSFLD: - case CEE_LDFLDA: - case CEE_LDSFLDA: - { - - bool isLoadAddress = (opcode == CEE_LDFLDA || opcode == CEE_LDSFLDA); - bool isLoadStatic = (opcode == CEE_LDSFLD || opcode == CEE_LDSFLDA); - - /* Get the CP_Fieldref index */ - assertImp(sz == sizeof(unsigned)); - - _impResolveToken(CORINFO_TOKENKIND_Field); - - JITDUMP(" %08X", resolvedToken.token); + /* These operators can later be transformed into 'GT_CALL' */ - int aflags = isLoadAddress ? CORINFO_ACCESS_ADDRESS : CORINFO_ACCESS_GET; + assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_MUL]); +#ifndef TARGET_ARM + assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_DIV]); + assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_UDIV]); + assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_MOD]); + assert(GenTree::s_gtNodeSizes[GT_CALL] > GenTree::s_gtNodeSizes[GT_UMOD]); +#endif + // It's tempting to use LargeOpOpcode() here, but this logic is *not* saying + // that we'll need to transform into a general large node, but rather specifically + // to a call: by doing it this way, things keep working if there are multiple sizes, + // and a CALL is no longer the largest. + // That said, as of now it *is* a large node, so we'll do this with an assert rather + // than an "if". + assert(GenTree::s_gtNodeSizes[GT_CALL] == TREE_NODE_SZ_LARGE); + op1 = new (this, GT_CALL) GenTreeOp(oper, type, op1, op2 DEBUGARG(/*largeNode*/ true)); + } + else + { + op1 = gtNewOperNode(oper, type, op1, op2); + } - GenTree* obj = nullptr; - typeInfo* tiObj = nullptr; - CORINFO_CLASS_HANDLE objType = nullptr; // used for fields + /* Special case: integer/long division may throw an exception */ - if (opcode == CEE_LDFLD || opcode == CEE_LDFLDA) + if (varTypeIsIntegral(op1->TypeGet()) && op1->OperMayThrow(this)) { - tiObj = &impStackTop().seTypeInfo; - StackEntry se = impPopStack(); - objType = se.seTypeInfo.GetClassHandle(); - obj = se.val; + op1->gtFlags |= GTF_EXCEPT; + } - if (impIsThis(obj)) + if (ovfl) + { + assert(oper == GT_ADD || oper == GT_SUB || oper == GT_MUL); + if (ovflType != TYP_UNKNOWN) { - aflags |= CORINFO_ACCESS_THIS; + op1->gtType = ovflType; + } + op1->gtFlags |= (GTF_EXCEPT | GTF_OVERFLOW); + if (uns) + { + op1->gtFlags |= GTF_UNSIGNED; } } - eeGetFieldInfo(&resolvedToken, (CORINFO_ACCESS_FLAGS)aflags, &fieldInfo); + impPushOnStack(op1, tiRetVal); + break; - // Figure out the type of the member. We always call canAccessField, so you always need this - // handle - CorInfoType ciType = fieldInfo.fieldType; - clsHnd = fieldInfo.structType; + case CEE_SHL: + oper = GT_LSH; + goto CEE_SH_OP2; - lclTyp = JITtype2varType(ciType); + case CEE_SHR: + oper = GT_RSH; + goto CEE_SH_OP2; + case CEE_SHR_UN: + oper = GT_RSZ; + goto CEE_SH_OP2; - if (compIsForInlining()) - { - switch (fieldInfo.fieldAccessor) - { - case CORINFO_FIELD_INSTANCE_HELPER: - case CORINFO_FIELD_INSTANCE_ADDR_HELPER: - case CORINFO_FIELD_STATIC_ADDR_HELPER: - case CORINFO_FIELD_STATIC_TLS: + CEE_SH_OP2: + op2 = impPopStack().val; + op1 = impPopStack().val; // operand to be shifted + impBashVarAddrsToI(op1, op2); - compInlineResult->NoteFatal(InlineObservation::CALLEE_LDFLD_NEEDS_HELPER); - return; + type = genActualType(op1->TypeGet()); + op1 = gtNewOperNode(oper, type, op1, op2); - case CORINFO_FIELD_STATIC_GENERICS_STATIC_HELPER: - case CORINFO_FIELD_STATIC_READYTORUN_HELPER: - /* We may be able to inline the field accessors in specific instantiations of generic - * methods */ - compInlineResult->NoteFatal(InlineObservation::CALLSITE_LDFLD_NEEDS_HELPER); - return; + impPushOnStack(op1, tiRetVal); + break; - default: - break; - } + case CEE_NOT: + op1 = impPopStack().val; + impBashVarAddrsToI(op1, nullptr); + type = genActualType(op1->TypeGet()); + impPushOnStack(gtNewOperNode(GT_NOT, type, op1), tiRetVal); + break; - if (!isLoadAddress && (fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC) && lclTyp == TYP_STRUCT && - clsHnd) - { - if ((info.compCompHnd->getTypeForPrimitiveValueClass(clsHnd) == CORINFO_TYPE_UNDEF) && - !(info.compFlags & CORINFO_FLG_FORCEINLINE)) - { - // Loading a static valuetype field usually will cause a JitHelper to be called - // for the static base. This will bloat the code. - compInlineResult->Note(InlineObservation::CALLEE_LDFLD_STATIC_VALUECLASS); + case CEE_CKFINITE: + op1 = impPopStack().val; + type = op1->TypeGet(); + op1 = gtNewOperNode(GT_CKFINITE, type, op1); + op1->gtFlags |= GTF_EXCEPT; - if (compInlineResult->IsFailure()) - { - return; - } - } - } - } + impPushOnStack(op1, tiRetVal); + break; - tiRetVal = verMakeTypeInfo(ciType, clsHnd); - if (isLoadAddress) - { - tiRetVal.MakeByRef(); - } - else - { - tiRetVal.NormaliseForStack(); - } + case CEE_LEAVE: - // Perform this check always to ensure that we get field access exceptions even with - // SkipVerification. - impHandleAccessAllowed(fieldInfo.accessAllowed, &fieldInfo.accessCalloutHelper); + val = getI4LittleEndian(codeAddr); // jump distance + jmpAddr = (IL_OFFSET)((codeAddr - info.compCode + sizeof(__int32)) + val); + goto LEAVE; - // Raise InvalidProgramException if static load accesses non-static field - if (isLoadStatic && ((fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC) == 0)) - { - BADCODE("static access on an instance field"); - } + case CEE_LEAVE_S: + val = getI1LittleEndian(codeAddr); // jump distance + jmpAddr = (IL_OFFSET)((codeAddr - info.compCode + sizeof(__int8)) + val); - // We are using ldfld/a on a static field. We allow it, but need to get side-effect from obj. - if ((fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC) && obj != nullptr) + LEAVE: + + if (compIsForInlining()) { - if (obj->gtFlags & GTF_SIDE_EFFECT) - { - obj = gtUnusedValNode(obj); - impAppendTree(obj, (unsigned)CHECK_SPILL_ALL, impCurStmtDI); - } - obj = nullptr; + compInlineResult->NoteFatal(InlineObservation::CALLEE_HAS_LEAVE); + return; } - /* Preserve 'small' int types */ - if (!varTypeIsSmall(lclTyp)) + JITDUMP(" %04X", jmpAddr); + if (block->bbJumpKind != BBJ_LEAVE) { - lclTyp = genActualType(lclTyp); + impResetLeaveBlock(block, jmpAddr); } - bool usesHelper = false; + assert(jmpAddr == block->bbJumpDest->bbCodeOffs); + impImportLeave(block); + impNoteBranchOffs(); - switch (fieldInfo.fieldAccessor) - { - case CORINFO_FIELD_INSTANCE: -#ifdef FEATURE_READYTORUN - case CORINFO_FIELD_INSTANCE_WITH_BASE: -#endif - { - // If the object is a struct, what we really want is - // for the field to operate on the address of the struct. - if (!varTypeGCtype(obj->TypeGet()) && impIsValueType(tiObj)) - { - assert(opcode == CEE_LDFLD && objType != nullptr); + break; - obj = impGetStructAddr(obj, objType, (unsigned)CHECK_SPILL_ALL, true); - } + case CEE_BR: + case CEE_BR_S: + jmpDist = (sz == 1) ? getI1LittleEndian(codeAddr) : getI4LittleEndian(codeAddr); - /* Create the data member node */ - op1 = gtNewFieldRef(lclTyp, resolvedToken.hField, obj, fieldInfo.offset); + if (compIsForInlining() && jmpDist == 0) + { + break; /* NOP */ + } -#ifdef FEATURE_READYTORUN - if (fieldInfo.fieldAccessor == CORINFO_FIELD_INSTANCE_WITH_BASE) - { - op1->AsField()->gtFieldLookup = fieldInfo.fieldLookup; - } -#endif + impNoteBranchOffs(); + break; - if (fgAddrCouldBeNull(obj)) - { - op1->gtFlags |= GTF_EXCEPT; - } + case CEE_BRTRUE: + case CEE_BRTRUE_S: + case CEE_BRFALSE: + case CEE_BRFALSE_S: - DWORD typeFlags = info.compCompHnd->getClassAttribs(resolvedToken.hClass); - if (StructHasOverlappingFields(typeFlags)) - { - op1->AsField()->gtFldMayOverlap = true; - } + /* Pop the comparand (now there's a neat term) from the stack */ - // wrap it in a address of operator if necessary - if (isLoadAddress) - { - op1 = gtNewOperNode(GT_ADDR, - (var_types)(varTypeIsGC(obj->TypeGet()) ? TYP_BYREF : TYP_I_IMPL), op1); - } - else - { - if (compIsForInlining() && - impInlineIsGuaranteedThisDerefBeforeAnySideEffects(nullptr, nullptr, obj, - impInlineInfo->inlArgInfo)) - { - impInlineInfo->thisDereferencedFirst = true; - } - } - } - break; + op1 = impPopStack().val; + type = op1->TypeGet(); - case CORINFO_FIELD_STATIC_TLS: -#ifdef TARGET_X86 - // Legacy TLS access is implemented as intrinsic on x86 only + // Per Ecma-355, brfalse and brtrue are only specified for nint, ref, and byref. + // + // We've historically been a bit more permissive, so here we allow + // any type that gtNewZeroConNode can handle. + if (!varTypeIsArithmetic(type) && !varTypeIsGC(type)) + { + BADCODE("invalid type for brtrue/brfalse"); + } - /* Create the data member node */ - op1 = gtNewFieldRef(lclTyp, resolvedToken.hField, NULL, fieldInfo.offset); - op1->gtFlags |= GTF_IND_TLS_REF; // fgMorphField will handle the transformation + if (opts.OptimizationEnabled() && (block->bbJumpDest == block->bbNext)) + { + block->bbJumpKind = BBJ_NONE; - if (isLoadAddress) - { - op1 = gtNewOperNode(GT_ADDR, (var_types)TYP_I_IMPL, op1); - } + if (op1->gtFlags & GTF_GLOB_EFFECT) + { + op1 = gtUnusedValNode(op1); + goto SPILL_APPEND; + } + else + { break; -#else - fieldInfo.fieldAccessor = CORINFO_FIELD_STATIC_ADDR_HELPER; + } + } - FALLTHROUGH; -#endif + if (op1->OperIsCompare()) + { + if (opcode == CEE_BRFALSE || opcode == CEE_BRFALSE_S) + { + // Flip the sense of the compare - case CORINFO_FIELD_STATIC_ADDR_HELPER: - case CORINFO_FIELD_INSTANCE_HELPER: - case CORINFO_FIELD_INSTANCE_ADDR_HELPER: - op1 = gtNewRefCOMfield(obj, &resolvedToken, (CORINFO_ACCESS_FLAGS)aflags, &fieldInfo, lclTyp, - clsHnd, nullptr); - usesHelper = true; - break; + op1 = gtReverseCond(op1); + } + } + else + { + // We'll compare against an equally-sized integer 0 + // For small types, we always compare against int + op2 = gtNewZeroConNode(genActualType(op1->gtType)); - case CORINFO_FIELD_STATIC_ADDRESS: - // Replace static read-only fields with constant if possible - if ((aflags & CORINFO_ACCESS_GET) && (fieldInfo.fieldFlags & CORINFO_FLG_FIELD_FINAL) && - !(fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC_IN_HEAP) && - (varTypeIsIntegral(lclTyp) || varTypeIsFloating(lclTyp))) - { - CorInfoInitClassResult initClassResult = - info.compCompHnd->initClass(resolvedToken.hField, info.compMethodHnd, - impTokenLookupContextHandle); + // Create the comparison operator and try to fold it + oper = (opcode == CEE_BRTRUE || opcode == CEE_BRTRUE_S) ? GT_NE : GT_EQ; + op1 = gtNewOperNode(oper, TYP_INT, op1, op2); + } - if (initClassResult & CORINFO_INITCLASS_INITIALIZED) - { - void** pFldAddr = nullptr; - void* fldAddr = - info.compCompHnd->getFieldAddress(resolvedToken.hField, (void**)&pFldAddr); + // fall through - // We should always be able to access this static's address directly - // - assert(pFldAddr == nullptr); + COND_JUMP: - op1 = impImportStaticReadOnlyField(fldAddr, lclTyp); + /* Fold comparison if we can */ - // Widen small types since we're propagating the value - // instead of producing an indir. - // - op1->gtType = genActualType(lclTyp); + op1 = gtFoldExpr(op1); - goto FIELD_DONE; - } - } + /* Try to fold the really simple cases like 'iconst *, ifne/ifeq'*/ + /* Don't make any blocks unreachable in import only mode */ - FALLTHROUGH; + if ((op1->gtOper == GT_CNS_INT) && !compIsForImportOnly()) + { + /* gtFoldExpr() should prevent this as we don't want to make any blocks + unreachable under compDbgCode */ + assert(!opts.compDbgCode); - case CORINFO_FIELD_STATIC_RVA_ADDRESS: - case CORINFO_FIELD_STATIC_SHARED_STATIC_HELPER: - case CORINFO_FIELD_STATIC_GENERICS_STATIC_HELPER: - case CORINFO_FIELD_STATIC_READYTORUN_HELPER: - op1 = impImportStaticFieldAccess(&resolvedToken, (CORINFO_ACCESS_FLAGS)aflags, &fieldInfo, - lclTyp); - break; + BBjumpKinds foldedJumpKind = (BBjumpKinds)(op1->AsIntCon()->gtIconVal ? BBJ_ALWAYS : BBJ_NONE); + assertImp((block->bbJumpKind == BBJ_COND) // normal case + || (block->bbJumpKind == foldedJumpKind)); // this can happen if we are reimporting the + // block for the second time - case CORINFO_FIELD_INTRINSIC_ZERO: + block->bbJumpKind = foldedJumpKind; +#ifdef DEBUG + if (verbose) { - assert(aflags & CORINFO_ACCESS_GET); - // Widen to stack type - lclTyp = genActualType(lclTyp); - op1 = gtNewIconNode(0, lclTyp); - goto FIELD_DONE; + if (op1->AsIntCon()->gtIconVal) + { + printf("\nThe conditional jump becomes an unconditional jump to " FMT_BB "\n", + block->bbJumpDest->bbNum); + } + else + { + printf("\nThe block falls through into the next " FMT_BB "\n", block->bbNext->bbNum); + } } +#endif break; + } - case CORINFO_FIELD_INTRINSIC_EMPTY_STRING: - { - assert(aflags & CORINFO_ACCESS_GET); - - // Import String.Empty as "" (GT_CNS_STR with a fake SconCPX = 0) - op1 = gtNewSconNode(EMPTY_STRING_SCON, nullptr); - goto FIELD_DONE; - } - break; + op1 = gtNewOperNode(GT_JTRUE, TYP_VOID, op1); - case CORINFO_FIELD_INTRINSIC_ISLITTLEENDIAN: - { - assert(aflags & CORINFO_ACCESS_GET); - // Widen to stack type - lclTyp = genActualType(lclTyp); -#if BIGENDIAN - op1 = gtNewIconNode(0, lclTyp); -#else - op1 = gtNewIconNode(1, lclTyp); -#endif - goto FIELD_DONE; - } - break; + /* GT_JTRUE is handled specially for non-empty stacks. See 'addStmt' + in impImportBlock(block). For correct line numbers, spill stack. */ - default: - assert(!"Unexpected fieldAccessor"); + if (opts.compDbgCode && impCurStmtDI.IsValid()) + { + impSpillStackEnsure(true); } - if (!isLoadAddress) + goto SPILL_APPEND; + + case CEE_CEQ: + oper = GT_EQ; + uns = false; + goto CMP_2_OPs; + case CEE_CGT_UN: + oper = GT_GT; + uns = true; + goto CMP_2_OPs; + case CEE_CGT: + oper = GT_GT; + uns = false; + goto CMP_2_OPs; + case CEE_CLT_UN: + oper = GT_LT; + uns = true; + goto CMP_2_OPs; + case CEE_CLT: + oper = GT_LT; + uns = false; + goto CMP_2_OPs; + + CMP_2_OPs: + op2 = impPopStack().val; + op1 = impPopStack().val; + + // Recognize the IL idiom of CGT_UN(op1, 0) and normalize + // it so that downstream optimizations don't have to. + if ((opcode == CEE_CGT_UN) && op2->IsIntegralConst(0)) { + oper = GT_NE; + uns = false; + } - if (prefixFlags & PREFIX_VOLATILE) - { - op1->gtFlags |= GTF_ORDER_SIDEEFF; // Prevent this from being reordered +#ifdef TARGET_64BIT + // TODO-Casts: create a helper that upcasts int32 -> native int when necessary. + // See also identical code in impGetByRefResultType and STSFLD import. + if (varTypeIsI(op1) && (genActualType(op2) == TYP_INT)) + { + op2 = gtNewCastNode(TYP_I_IMPL, op2, uns, TYP_I_IMPL); + } + else if (varTypeIsI(op2) && (genActualType(op1) == TYP_INT)) + { + op1 = gtNewCastNode(TYP_I_IMPL, op1, uns, TYP_I_IMPL); + } +#endif // TARGET_64BIT - if (!usesHelper) - { - assert((op1->OperGet() == GT_FIELD) || (op1->OperGet() == GT_IND) || - (op1->OperGet() == GT_OBJ)); - op1->gtFlags |= GTF_IND_VOLATILE; - } - } + assertImp(genActualType(op1) == genActualType(op2) || (varTypeIsI(op1) && varTypeIsI(op2)) || + (varTypeIsFloating(op1) && varTypeIsFloating(op2))); - if ((prefixFlags & PREFIX_UNALIGNED) && !varTypeIsByte(lclTyp)) - { - if (!usesHelper) - { - assert((op1->OperGet() == GT_FIELD) || (op1->OperGet() == GT_IND) || - (op1->OperGet() == GT_OBJ)); - op1->gtFlags |= GTF_IND_UNALIGNED; - } - } + if ((op1->TypeGet() != op2->TypeGet()) && varTypeIsFloating(op1)) + { + op1 = impImplicitR4orR8Cast(op1, TYP_DOUBLE); + op2 = impImplicitR4orR8Cast(op2, TYP_DOUBLE); } - /* Check if the class needs explicit initialization */ + // Create the comparison node. + op1 = gtNewOperNode(oper, TYP_INT, op1, op2); - if (fieldInfo.fieldFlags & CORINFO_FLG_FIELD_INITCLASS) + // TODO: setting both flags when only one is appropriate. + if (uns) { - GenTree* helperNode = impInitClass(&resolvedToken); - if (compDonotInline()) - { - return; - } - if (helperNode != nullptr) - { - op1 = gtNewOperNode(GT_COMMA, op1->TypeGet(), helperNode, op1); - } + op1->gtFlags |= GTF_RELOP_NAN_UN | GTF_UNSIGNED; } - FIELD_DONE: - impPushOnStack(op1, tiRetVal); - } - break; + // Fold result, if possible. + op1 = gtFoldExpr(op1); - case CEE_STFLD: - case CEE_STSFLD: - { + impPushOnStack(op1, tiRetVal); + break; - bool isStoreStatic = (opcode == CEE_STSFLD); + case CEE_BEQ_S: + case CEE_BEQ: + oper = GT_EQ; + goto CMP_2_OPs_AND_BR; - CORINFO_CLASS_HANDLE fieldClsHnd; // class of the field (if it's a ref type) + case CEE_BGE_S: + case CEE_BGE: + oper = GT_GE; + goto CMP_2_OPs_AND_BR; - /* Get the CP_Fieldref index */ + case CEE_BGE_UN_S: + case CEE_BGE_UN: + oper = GT_GE; + goto CMP_2_OPs_AND_BR_UN; - assertImp(sz == sizeof(unsigned)); + case CEE_BGT_S: + case CEE_BGT: + oper = GT_GT; + goto CMP_2_OPs_AND_BR; - _impResolveToken(CORINFO_TOKENKIND_Field); + case CEE_BGT_UN_S: + case CEE_BGT_UN: + oper = GT_GT; + goto CMP_2_OPs_AND_BR_UN; - JITDUMP(" %08X", resolvedToken.token); + case CEE_BLE_S: + case CEE_BLE: + oper = GT_LE; + goto CMP_2_OPs_AND_BR; - int aflags = CORINFO_ACCESS_SET; - GenTree* obj = nullptr; - typeInfo* tiObj = nullptr; - typeInfo tiVal; + case CEE_BLE_UN_S: + case CEE_BLE_UN: + oper = GT_LE; + goto CMP_2_OPs_AND_BR_UN; - /* Pull the value from the stack */ - StackEntry se = impPopStack(); - op2 = se.val; - tiVal = se.seTypeInfo; - clsHnd = tiVal.GetClassHandle(); + case CEE_BLT_S: + case CEE_BLT: + oper = GT_LT; + goto CMP_2_OPs_AND_BR; - if (opcode == CEE_STFLD) - { - tiObj = &impStackTop().seTypeInfo; - obj = impPopStack().val; + case CEE_BLT_UN_S: + case CEE_BLT_UN: + oper = GT_LT; + goto CMP_2_OPs_AND_BR_UN; - if (impIsThis(obj)) - { - aflags |= CORINFO_ACCESS_THIS; - } - } + case CEE_BNE_UN_S: + case CEE_BNE_UN: + oper = GT_NE; + goto CMP_2_OPs_AND_BR_UN; - eeGetFieldInfo(&resolvedToken, (CORINFO_ACCESS_FLAGS)aflags, &fieldInfo); + CMP_2_OPs_AND_BR_UN: + uns = true; + unordered = true; + goto CMP_2_OPs_AND_BR_ALL; + CMP_2_OPs_AND_BR: + uns = false; + unordered = false; + goto CMP_2_OPs_AND_BR_ALL; + CMP_2_OPs_AND_BR_ALL: + /* Pull two values */ + op2 = impPopStack().val; + op1 = impPopStack().val; - // Figure out the type of the member. We always call canAccessField, so you always need this - // handle - CorInfoType ciType = fieldInfo.fieldType; - fieldClsHnd = fieldInfo.structType; +#ifdef TARGET_64BIT + if ((op1->TypeGet() == TYP_I_IMPL) && (genActualType(op2->TypeGet()) == TYP_INT)) + { + op2 = gtNewCastNode(TYP_I_IMPL, op2, uns, uns ? TYP_U_IMPL : TYP_I_IMPL); + } + else if ((op2->TypeGet() == TYP_I_IMPL) && (genActualType(op1->TypeGet()) == TYP_INT)) + { + op1 = gtNewCastNode(TYP_I_IMPL, op1, uns, uns ? TYP_U_IMPL : TYP_I_IMPL); + } +#endif // TARGET_64BIT - lclTyp = JITtype2varType(ciType); + assertImp(genActualType(op1->TypeGet()) == genActualType(op2->TypeGet()) || + (varTypeIsI(op1->TypeGet()) && varTypeIsI(op2->TypeGet())) || + (varTypeIsFloating(op1->gtType) && varTypeIsFloating(op2->gtType))); - if (compIsForInlining()) + if (opts.OptimizationEnabled() && (block->bbJumpDest == block->bbNext)) { - /* Is this a 'special' (COM) field? or a TLS ref static field?, field stored int GC heap? or - * per-inst static? */ + block->bbJumpKind = BBJ_NONE; - switch (fieldInfo.fieldAccessor) + if (op1->gtFlags & GTF_GLOB_EFFECT) { - case CORINFO_FIELD_INSTANCE_HELPER: - case CORINFO_FIELD_INSTANCE_ADDR_HELPER: - case CORINFO_FIELD_STATIC_ADDR_HELPER: - case CORINFO_FIELD_STATIC_TLS: + impSpillSideEffects(false, + CHECK_SPILL_ALL DEBUGARG("Branch to next Optimization, op1 side effect")); + impAppendTree(gtUnusedValNode(op1), CHECK_SPILL_NONE, impCurStmtDI); + } + if (op2->gtFlags & GTF_GLOB_EFFECT) + { + impSpillSideEffects(false, + CHECK_SPILL_ALL DEBUGARG("Branch to next Optimization, op2 side effect")); + impAppendTree(gtUnusedValNode(op2), CHECK_SPILL_NONE, impCurStmtDI); + } - compInlineResult->NoteFatal(InlineObservation::CALLEE_STFLD_NEEDS_HELPER); - return; +#ifdef DEBUG + if ((op1->gtFlags | op2->gtFlags) & GTF_GLOB_EFFECT) + { + impNoteLastILoffs(); + } +#endif + break; + } - case CORINFO_FIELD_STATIC_GENERICS_STATIC_HELPER: - case CORINFO_FIELD_STATIC_READYTORUN_HELPER: - /* We may be able to inline the field accessors in specific instantiations of generic - * methods */ - compInlineResult->NoteFatal(InlineObservation::CALLSITE_STFLD_NEEDS_HELPER); - return; + // We can generate an compare of different sized floating point op1 and op2 + // We insert a cast + // + if (varTypeIsFloating(op1->TypeGet())) + { + if (op1->TypeGet() != op2->TypeGet()) + { + assert(varTypeIsFloating(op2->TypeGet())); - default: - break; + // say op1=double, op2=float. To avoid loss of precision + // while comparing, op2 is converted to double and double + // comparison is done. + if (op1->TypeGet() == TYP_DOUBLE) + { + // We insert a cast of op2 to TYP_DOUBLE + op2 = gtNewCastNode(TYP_DOUBLE, op2, false, TYP_DOUBLE); + } + else if (op2->TypeGet() == TYP_DOUBLE) + { + // We insert a cast of op1 to TYP_DOUBLE + op1 = gtNewCastNode(TYP_DOUBLE, op1, false, TYP_DOUBLE); + } } } - impHandleAccessAllowed(fieldInfo.accessAllowed, &fieldInfo.accessCalloutHelper); + /* Create and append the operator */ - // Raise InvalidProgramException if static store accesses non-static field - if (isStoreStatic && ((fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC) == 0)) - { - BADCODE("static access on an instance field"); - } + op1 = gtNewOperNode(oper, TYP_INT, op1, op2); - // We are using stfld on a static field. - // We allow it, but need to eval any side-effects for obj - if ((fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC) && obj != nullptr) + if (uns) { - if (obj->gtFlags & GTF_SIDE_EFFECT) - { - obj = gtUnusedValNode(obj); - impAppendTree(obj, (unsigned)CHECK_SPILL_ALL, impCurStmtDI); - } - obj = nullptr; + op1->gtFlags |= GTF_UNSIGNED; } - /* Preserve 'small' int types */ - if (!varTypeIsSmall(lclTyp)) + if (unordered) { - lclTyp = genActualType(lclTyp); + op1->gtFlags |= GTF_RELOP_NAN_UN; } - switch (fieldInfo.fieldAccessor) - { - case CORINFO_FIELD_INSTANCE: -#ifdef FEATURE_READYTORUN - case CORINFO_FIELD_INSTANCE_WITH_BASE: -#endif - { - /* Create the data member node */ - op1 = gtNewFieldRef(lclTyp, resolvedToken.hField, obj, fieldInfo.offset); - DWORD typeFlags = info.compCompHnd->getClassAttribs(resolvedToken.hClass); - if (StructHasOverlappingFields(typeFlags)) - { - op1->AsField()->gtFldMayOverlap = true; - } + goto COND_JUMP; -#ifdef FEATURE_READYTORUN - if (fieldInfo.fieldAccessor == CORINFO_FIELD_INSTANCE_WITH_BASE) - { - op1->AsField()->gtFieldLookup = fieldInfo.fieldLookup; - } -#endif + case CEE_SWITCH: + /* Pop the switch value off the stack */ + op1 = impPopStack().val; + assertImp(genActualTypeIsIntOrI(op1->TypeGet())); - if (fgAddrCouldBeNull(obj)) - { - op1->gtFlags |= GTF_EXCEPT; - } + /* We can create a switch node */ - if (compIsForInlining() && - impInlineIsGuaranteedThisDerefBeforeAnySideEffects(op2, nullptr, obj, - impInlineInfo->inlArgInfo)) - { - impInlineInfo->thisDereferencedFirst = true; - } - } - break; + op1 = gtNewOperNode(GT_SWITCH, TYP_VOID, op1); - case CORINFO_FIELD_STATIC_TLS: -#ifdef TARGET_X86 - // Legacy TLS access is implemented as intrinsic on x86 only + val = (int)getU4LittleEndian(codeAddr); + codeAddr += 4 + val * 4; // skip over the switch-table - /* Create the data member node */ - op1 = gtNewFieldRef(lclTyp, resolvedToken.hField, NULL, fieldInfo.offset); - op1->gtFlags |= GTF_IND_TLS_REF; // fgMorphField will handle the transformation + goto SPILL_APPEND; - break; -#else - fieldInfo.fieldAccessor = CORINFO_FIELD_STATIC_ADDR_HELPER; + /************************** Casting OPCODES ***************************/ - FALLTHROUGH; -#endif + case CEE_CONV_OVF_I1: + lclTyp = TYP_BYTE; + goto CONV_OVF; + case CEE_CONV_OVF_I2: + lclTyp = TYP_SHORT; + goto CONV_OVF; + case CEE_CONV_OVF_I: + lclTyp = TYP_I_IMPL; + goto CONV_OVF; + case CEE_CONV_OVF_I4: + lclTyp = TYP_INT; + goto CONV_OVF; + case CEE_CONV_OVF_I8: + lclTyp = TYP_LONG; + goto CONV_OVF; - case CORINFO_FIELD_STATIC_ADDR_HELPER: - case CORINFO_FIELD_INSTANCE_HELPER: - case CORINFO_FIELD_INSTANCE_ADDR_HELPER: - op1 = gtNewRefCOMfield(obj, &resolvedToken, (CORINFO_ACCESS_FLAGS)aflags, &fieldInfo, lclTyp, - clsHnd, op2); - goto SPILL_APPEND; + case CEE_CONV_OVF_U1: + lclTyp = TYP_UBYTE; + goto CONV_OVF; + case CEE_CONV_OVF_U2: + lclTyp = TYP_USHORT; + goto CONV_OVF; + case CEE_CONV_OVF_U: + lclTyp = TYP_U_IMPL; + goto CONV_OVF; + case CEE_CONV_OVF_U4: + lclTyp = TYP_UINT; + goto CONV_OVF; + case CEE_CONV_OVF_U8: + lclTyp = TYP_ULONG; + goto CONV_OVF; - case CORINFO_FIELD_STATIC_ADDRESS: - case CORINFO_FIELD_STATIC_RVA_ADDRESS: - case CORINFO_FIELD_STATIC_SHARED_STATIC_HELPER: - case CORINFO_FIELD_STATIC_GENERICS_STATIC_HELPER: - case CORINFO_FIELD_STATIC_READYTORUN_HELPER: - op1 = impImportStaticFieldAccess(&resolvedToken, (CORINFO_ACCESS_FLAGS)aflags, &fieldInfo, - lclTyp); - break; + case CEE_CONV_OVF_I1_UN: + lclTyp = TYP_BYTE; + goto CONV_OVF_UN; + case CEE_CONV_OVF_I2_UN: + lclTyp = TYP_SHORT; + goto CONV_OVF_UN; + case CEE_CONV_OVF_I_UN: + lclTyp = TYP_I_IMPL; + goto CONV_OVF_UN; + case CEE_CONV_OVF_I4_UN: + lclTyp = TYP_INT; + goto CONV_OVF_UN; + case CEE_CONV_OVF_I8_UN: + lclTyp = TYP_LONG; + goto CONV_OVF_UN; - default: - assert(!"Unexpected fieldAccessor"); - } + case CEE_CONV_OVF_U1_UN: + lclTyp = TYP_UBYTE; + goto CONV_OVF_UN; + case CEE_CONV_OVF_U2_UN: + lclTyp = TYP_USHORT; + goto CONV_OVF_UN; + case CEE_CONV_OVF_U_UN: + lclTyp = TYP_U_IMPL; + goto CONV_OVF_UN; + case CEE_CONV_OVF_U4_UN: + lclTyp = TYP_UINT; + goto CONV_OVF_UN; + case CEE_CONV_OVF_U8_UN: + lclTyp = TYP_ULONG; + goto CONV_OVF_UN; - // "impAssignStruct" will back-substitute the field address tree into calls that return things via - // return buffers, so we have to delay calling it until after we have spilled everything needed. - bool deferStructAssign = (lclTyp == TYP_STRUCT); + CONV_OVF_UN: + uns = true; + goto CONV_OVF_COMMON; + CONV_OVF: + uns = false; + goto CONV_OVF_COMMON; - if (!deferStructAssign) - { - assert(op1->OperIs(GT_FIELD, GT_IND)); + CONV_OVF_COMMON: + ovfl = true; + goto _CONV; - if (prefixFlags & PREFIX_VOLATILE) - { - op1->gtFlags |= GTF_ORDER_SIDEEFF; // Prevent this from being reordered - op1->gtFlags |= GTF_IND_VOLATILE; - } - if ((prefixFlags & PREFIX_UNALIGNED) && !varTypeIsByte(lclTyp)) - { - op1->gtFlags |= GTF_IND_UNALIGNED; - } + case CEE_CONV_I1: + lclTyp = TYP_BYTE; + goto CONV; + case CEE_CONV_I2: + lclTyp = TYP_SHORT; + goto CONV; + case CEE_CONV_I: + lclTyp = TYP_I_IMPL; + goto CONV; + case CEE_CONV_I4: + lclTyp = TYP_INT; + goto CONV; + case CEE_CONV_I8: + lclTyp = TYP_LONG; + goto CONV; - // Currently, *all* TYP_REF statics are stored inside an "object[]" array that itself - // resides on the managed heap, and so we can use an unchecked write barrier for this - // store. Likewise if we're storing to a field of an on-heap object. - if ((lclTyp == TYP_REF) && - (((fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC) != 0) || obj->TypeIs(TYP_REF))) - { - static_assert_no_msg(GTF_FLD_TGT_HEAP == GTF_IND_TGT_HEAP); - op1->gtFlags |= GTF_FLD_TGT_HEAP; - } + case CEE_CONV_U1: + lclTyp = TYP_UBYTE; + goto CONV; + case CEE_CONV_U2: + lclTyp = TYP_USHORT; + goto CONV; +#if (REGSIZE_BYTES == 8) + case CEE_CONV_U: + lclTyp = TYP_U_IMPL; + goto CONV_UN; +#else + case CEE_CONV_U: + lclTyp = TYP_U_IMPL; + goto CONV; +#endif + case CEE_CONV_U4: + lclTyp = TYP_UINT; + goto CONV; + case CEE_CONV_U8: + lclTyp = TYP_ULONG; + goto CONV_UN; - /* V4.0 allows assignment of i4 constant values to i8 type vars when IL verifier is bypassed (full - trust apps). The reason this works is that JIT stores an i4 constant in Gentree union during - importation and reads from the union as if it were a long during code generation. Though this - can potentially read garbage, one can get lucky to have this working correctly. + case CEE_CONV_R4: + lclTyp = TYP_FLOAT; + goto CONV; + case CEE_CONV_R8: + lclTyp = TYP_DOUBLE; + goto CONV; - This code pattern is generated by Dev10 MC++ compiler while storing to fields when compiled with - /O2 switch (default when compiling retail configs in Dev10) and a customer app has taken a - dependency on it. To be backward compatible, we will explicitly add an upward cast here so that - it works correctly always. + case CEE_CONV_R_UN: + lclTyp = TYP_DOUBLE; + goto CONV_UN; - Note that this is limited to x86 alone as there is no back compat to be addressed for Arm JIT - for V4.0. - */ - CLANG_FORMAT_COMMENT_ANCHOR; + CONV_UN: + uns = true; + ovfl = false; + goto _CONV; -#ifndef TARGET_64BIT - // In UWP6.0 and beyond (post-.NET Core 2.0), we decided to let this cast from int to long be - // generated for ARM as well as x86, so the following IR will be accepted: - // STMTx (IL 0x... ???) - // * ASG long - // +--* LCL_VAR long - // \--* CNS_INT int 2 + CONV: + uns = false; + ovfl = false; + goto _CONV; - if ((op1->TypeGet() != op2->TypeGet()) && op2->OperIsConst() && varTypeIsIntOrI(op2->TypeGet()) && - varTypeIsLong(op1->TypeGet())) - { - op2 = gtNewCastNode(op1->TypeGet(), op2, false, op1->TypeGet()); - } -#endif + _CONV: + // only converts from FLOAT or DOUBLE to an integer type + // and converts from ULONG (or LONG on ARM) to DOUBLE are morphed to calls + if (varTypeIsFloating(lclTyp)) + { + callNode = varTypeIsLong(impStackTop().val) || uns // uint->dbl gets turned into uint->long->dbl #ifdef TARGET_64BIT - // Automatic upcast for a GT_CNS_INT into TYP_I_IMPL - if ((op2->OperGet() == GT_CNS_INT) && varTypeIsI(lclTyp) && !varTypeIsI(op2->gtType)) - { - op2->gtType = TYP_I_IMPL; - } - else - { - // Allow a downcast of op2 from TYP_I_IMPL into a 32-bit Int for x86 JIT compatiblity - // - if (varTypeIsI(op2->gtType) && (genActualType(lclTyp) == TYP_INT)) - { - op2 = gtNewCastNode(TYP_INT, op2, false, TYP_INT); - } - // Allow an upcast of op2 from a 32-bit Int into TYP_I_IMPL for x86 JIT compatiblity - // - if (varTypeIsI(lclTyp) && (genActualType(op2->gtType) == TYP_INT)) - { - op2 = gtNewCastNode(TYP_I_IMPL, op2, false, TYP_I_IMPL); - } - } + // TODO-ARM64-Bug?: This was AMD64; I enabled it for ARM64 also. OK? + // TYP_BYREF could be used as TYP_I_IMPL which is long. + // TODO-CQ: remove this when we lower casts long/ulong --> float/double + // and generate SSE2 code instead of going through helper calls. + || (impStackTop().val->TypeGet() == TYP_BYREF) #endif - - // We can generate an assignment to a TYP_FLOAT from a TYP_DOUBLE - // We insert a cast to the dest 'op1' type - // - if ((op1->TypeGet() != op2->TypeGet()) && varTypeIsFloating(op1->gtType) && - varTypeIsFloating(op2->gtType)) - { - op2 = gtNewCastNode(op1->TypeGet(), op2, false, op1->TypeGet()); - } - - op1 = gtNewAssignNode(op1, op2); + ; + } + else + { + callNode = varTypeIsFloating(impStackTop().val->TypeGet()); } - /* Check if the class needs explicit initialization */ + op1 = impPopStack().val; + + impBashVarAddrsToI(op1); - if (fieldInfo.fieldFlags & CORINFO_FLG_FIELD_INITCLASS) + // Casts from floating point types must not have GTF_UNSIGNED set. + if (varTypeIsFloating(op1)) { - GenTree* helperNode = impInitClass(&resolvedToken); - if (compDonotInline()) - { - return; - } - if (helperNode != nullptr) - { - op1 = gtNewOperNode(GT_COMMA, op1->TypeGet(), helperNode, op1); - } + uns = false; } - // An indirect store such as "st[s]fld" interferes with indirect accesses, so we must spill - // global refs and potentially aliased locals. - impSpillSideEffects(true, (unsigned)CHECK_SPILL_ALL DEBUGARG("spill side effects before STFLD")); + // At this point uns, ovf, callNode are all set. - if (deferStructAssign) + if (varTypeIsSmall(lclTyp) && !ovfl && op1->gtType == TYP_INT && op1->gtOper == GT_AND) { - op1 = impAssignStruct(op1, op2, clsHnd, (unsigned)CHECK_SPILL_ALL); - } - } - goto APPEND; + op2 = op1->AsOp()->gtOp2; - case CEE_NEWARR: - { + if (op2->gtOper == GT_CNS_INT) + { + ssize_t ival = op2->AsIntCon()->gtIconVal; + ssize_t mask, umask; - /* Get the class type index operand */ + switch (lclTyp) + { + case TYP_BYTE: + case TYP_UBYTE: + mask = 0x00FF; + umask = 0x007F; + break; + case TYP_USHORT: + case TYP_SHORT: + mask = 0xFFFF; + umask = 0x7FFF; + break; - _impResolveToken(CORINFO_TOKENKIND_Newarr); + default: + assert(!"unexpected type"); + return; + } - JITDUMP(" %08X", resolvedToken.token); + if (((ival & umask) == ival) || ((ival & mask) == ival && uns)) + { + /* Toss the cast, it's a waste of time */ - if (!opts.IsReadyToRun()) - { - // Need to restore array classes before creating array objects on the heap - op1 = impTokenToHandle(&resolvedToken, nullptr, true /*mustRestoreHandle*/); - if (op1 == nullptr) - { // compDonotInline() - return; + impPushOnStack(op1, tiRetVal); + break; + } + else if (ival == mask) + { + /* Toss the masking, it's a waste of time, since + we sign-extend from the small value anyways */ + + op1 = op1->AsOp()->gtOp1; + } } } - tiRetVal = verMakeTypeInfo(resolvedToken.hClass); - - accessAllowedResult = - info.compCompHnd->canAccessClass(&resolvedToken, info.compMethodHnd, &calloutHelper); - impHandleAccessAllowed(accessAllowedResult, &calloutHelper); + /* The 'op2' sub-operand of a cast is the 'real' type number, + since the result of a cast to one of the 'small' integer + types is an integer. + */ - /* Form the arglist: array class handle, size */ - op2 = impPopStack().val; - assertImp(genActualTypeIsIntOrI(op2->gtType)); + type = genActualType(lclTyp); -#ifdef TARGET_64BIT - // The array helper takes a native int for array length. - // So if we have an int, explicitly extend it to be a native int. - if (genActualType(op2->TypeGet()) != TYP_I_IMPL) + // If this is a no-op cast, just use op1. + if (!ovfl && (type == op1->TypeGet()) && (genTypeSize(type) == genTypeSize(lclTyp))) { - if (op2->IsIntegralConst()) + // Nothing needs to change + } + // Work is evidently required, add cast node + else + { + if (callNode) { - op2->gtType = TYP_I_IMPL; + op1 = gtNewCastNodeL(type, op1, uns, lclTyp); } else { - bool isUnsigned = false; - op2 = gtNewCastNode(TYP_I_IMPL, op2, isUnsigned, TYP_I_IMPL); + op1 = gtNewCastNode(type, op1, uns, lclTyp); } - } -#endif // TARGET_64BIT - -#ifdef FEATURE_READYTORUN - if (opts.IsReadyToRun()) - { - op1 = impReadyToRunHelperToTree(&resolvedToken, CORINFO_HELP_READYTORUN_NEWARR_1, TYP_REF, nullptr, - op2); - usingReadyToRunHelper = (op1 != nullptr); - if (!usingReadyToRunHelper) + if (ovfl) { - // TODO: ReadyToRun: When generic dictionary lookups are necessary, replace the lookup call - // and the newarr call with a single call to a dynamic R2R cell that will: - // 1) Load the context - // 2) Perform the generic dictionary lookup and caching, and generate the appropriate stub - // 3) Allocate the new array - // Reason: performance (today, we'll always use the slow helper for the R2R generics case) - - // Need to restore array classes before creating array objects on the heap - op1 = impTokenToHandle(&resolvedToken, nullptr, true /*mustRestoreHandle*/); - if (op1 == nullptr) - { // compDonotInline() - return; - } + op1->gtFlags |= (GTF_OVERFLOW | GTF_EXCEPT); } - } - - if (!usingReadyToRunHelper) -#endif - { - /* Create a call to 'new' */ - // Note that this only works for shared generic code because the same helper is used for all - // reference array types - op1 = - gtNewHelperCallNode(info.compCompHnd->getNewArrHelper(resolvedToken.hClass), TYP_REF, op1, op2); + if (op1->gtGetOp1()->OperIsConst() && opts.OptimizationEnabled()) + { + // Try and fold the introduced cast + op1 = gtFoldExprConst(op1); + } } - op1->AsCall()->compileTimeHelperArgumentHandle = (CORINFO_GENERIC_HANDLE)resolvedToken.hClass; - - // Remember that this function contains 'new' of an SD array. - optMethodFlags |= OMF_HAS_NEWARRAY; - - /* Push the result of the call on the stack */ - impPushOnStack(op1, tiRetVal); + break; - callTyp = TYP_REF; - } - break; + case CEE_NEG: + op1 = impPopStack().val; + impBashVarAddrsToI(op1, nullptr); + impPushOnStack(gtNewOperNode(GT_NEG, genActualType(op1->gtType), op1), tiRetVal); + break; + + case CEE_POP: + { + /* Pull the top value from the stack */ - case CEE_LOCALLOC: - // We don't allow locallocs inside handlers - if (block->hasHndIndex()) - { - BADCODE("Localloc can't be inside handler"); - } + StackEntry se = impPopStack(); + clsHnd = se.seTypeInfo.GetClassHandle(); + op1 = se.val; - // Get the size to allocate + /* Get hold of the type of the value being duplicated */ - op2 = impPopStack().val; - assertImp(genActualTypeIsIntOrI(op2->gtType)); + lclTyp = genActualType(op1->gtType); - if (verCurrentState.esStackDepth != 0) - { - BADCODE("Localloc can only be used when the stack is empty"); - } + /* Does the value have any side effects? */ - // If the localloc is not in a loop and its size is a small constant, - // create a new local var of TYP_BLK and return its address. + if ((op1->gtFlags & GTF_SIDE_EFFECT) || opts.compDbgCode) { - bool convertedToLocal = false; - - // Need to aggressively fold here, as even fixed-size locallocs - // will have casts in the way. - op2 = gtFoldExpr(op2); + // Since we are throwing away the value, just normalize + // it to its address. This is more efficient. - if (op2->IsIntegralConst()) + if (varTypeIsStruct(op1)) { - const ssize_t allocSize = op2->AsIntCon()->IconValue(); - - bool bbInALoop = impBlockIsInALoop(block); - - if (allocSize == 0) - { - // Result is nullptr - JITDUMP("Converting stackalloc of 0 bytes to push null unmanaged pointer\n"); - op1 = gtNewIconNode(0, TYP_I_IMPL); - convertedToLocal = true; - } - else if ((allocSize > 0) && !bbInALoop) + JITDUMP("\n ... CEE_POP struct ...\n"); + DISPTREE(op1); +#ifdef UNIX_AMD64_ABI + // Non-calls, such as obj or ret_expr, have to go through this. + // Calls with large struct return value have to go through this. + // Helper calls with small struct return value also have to go + // through this since they do not follow Unix calling convention. + if (op1->gtOper != GT_CALL || + !IsMultiRegReturnedType(clsHnd, op1->AsCall()->GetUnmanagedCallConv()) || + op1->AsCall()->gtCallType == CT_HELPER) +#endif // UNIX_AMD64_ABI { - // Get the size threshold for local conversion - ssize_t maxSize = DEFAULT_MAX_LOCALLOC_TO_LOCAL_SIZE; - -#ifdef DEBUG - // Optionally allow this to be modified - maxSize = JitConfig.JitStackAllocToLocalSize(); -#endif // DEBUG - - if (allocSize <= maxSize) + // If the value being produced comes from loading + // via an underlying address, just null check the address. + if (op1->OperIs(GT_FIELD, GT_IND, GT_OBJ)) { - const unsigned stackallocAsLocal = lvaGrabTemp(false DEBUGARG("stackallocLocal")); - JITDUMP("Converting stackalloc of %zd bytes to new local V%02u\n", allocSize, - stackallocAsLocal); - lvaTable[stackallocAsLocal].lvType = TYP_BLK; - lvaTable[stackallocAsLocal].lvExactSize = (unsigned)allocSize; - lvaTable[stackallocAsLocal].lvIsUnsafeBuffer = true; - op1 = gtNewLclvNode(stackallocAsLocal, TYP_BLK); - op1 = gtNewOperNode(GT_ADDR, TYP_I_IMPL, op1); - convertedToLocal = true; - - if (!this->opts.compDbgEnC) - { - // Ensure we have stack security for this method. - // Reorder layout since the converted localloc is treated as an unsafe buffer. - setNeedsGSSecurityCookie(); - compGSReorderStackLayout = true; - } + gtChangeOperToNullCheck(op1, block); + } + else + { + op1 = impGetStructAddr(op1, clsHnd, CHECK_SPILL_ALL, false); } + + JITDUMP("\n ... optimized to ...\n"); + DISPTREE(op1); } } - if (!convertedToLocal) + // If op1 is non-overflow cast, throw it away since it is useless. + // Another reason for throwing away the useless cast is in the context of + // implicit tail calls when the operand of pop is GT_CAST(GT_CALL(..)). + // The cast gets added as part of importing GT_CALL, which gets in the way + // of fgMorphCall() on the forms of tail call nodes that we assert. + if ((op1->gtOper == GT_CAST) && !op1->gtOverflow()) { - // Bail out if inlining and the localloc was not converted. - // - // Note we might consider allowing the inline, if the call - // site is not in a loop. - if (compIsForInlining()) + op1 = op1->AsOp()->gtOp1; + } + + if (op1->gtOper != GT_CALL) + { + if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0) { - InlineObservation obs = op2->IsIntegralConst() - ? InlineObservation::CALLEE_LOCALLOC_TOO_LARGE - : InlineObservation::CALLSITE_LOCALLOC_SIZE_UNKNOWN; - compInlineResult->NoteFatal(obs); - return; + op1 = gtUnusedValNode(op1); + } + else + { + // Can't bash to NOP here because op1 can be referenced from `currentBlock->bbEntryState`, + // if we ever need to reimport we need a valid LCL_VAR on it. + op1 = gtNewNothingNode(); } + } - op1 = gtNewOperNode(GT_LCLHEAP, TYP_I_IMPL, op2); - // May throw a stack overflow exception. Obviously, we don't want locallocs to be CSE'd. - op1->gtFlags |= (GTF_EXCEPT | GTF_DONT_CSE); + /* Append the value to the tree list */ + goto SPILL_APPEND; + } - // Ensure we have stack security for this method. - setNeedsGSSecurityCookie(); + /* No side effects - just throw the thing away */ + } + break; - /* The FP register may not be back to the original value at the end - of the method, even if the frame size is 0, as localloc may - have modified it. So we will HAVE to reset it */ - compLocallocUsed = true; + case CEE_DUP: + { + StackEntry se = impPopStack(); + GenTree* tree = se.val; + tiRetVal = se.seTypeInfo; + op1 = tree; + + // In unoptimized code we leave the decision of + // cloning/creating temps up to impCloneExpr, while in + // optimized code we prefer temps except for some cases we know + // are profitable. + + if (opts.OptimizationEnabled()) + { + bool clone = false; + // Duplicate 0 and +0.0 + if (op1->IsIntegralConst(0) || op1->IsFloatPositiveZero()) + { + clone = true; + } + // Duplicate locals and addresses of them + else if (op1->IsLocal()) + { + clone = true; + } + else if (op1->TypeIs(TYP_BYREF, TYP_I_IMPL) && impIsAddressInLocal(op1)) + { + // We mark implicit byrefs with GTF_GLOB_REF (see gtNewFieldRef for why). + // Avoid cloning for these. + clone = (op1->gtFlags & GTF_GLOB_REF) == 0; + } + + if (clone) + { + op2 = gtCloneExpr(op1); } else { - compLocallocOptimized = true; + const unsigned tmpNum = lvaGrabTemp(true DEBUGARG("dup spill")); + impAssignTempGen(tmpNum, op1, tiRetVal.GetClassHandle(), CHECK_SPILL_ALL); + var_types type = genActualType(lvaTable[tmpNum].TypeGet()); + + // Propagate type info to the temp from the stack and the original tree + if (type == TYP_REF) + { + assert(lvaTable[tmpNum].lvSingleDef == 0); + lvaTable[tmpNum].lvSingleDef = 1; + JITDUMP("Marked V%02u as a single def local\n", tmpNum); + lvaSetClass(tmpNum, tree, tiRetVal.GetClassHandle()); + } + + op1 = gtNewLclvNode(tmpNum, type); + op2 = gtNewLclvNode(tmpNum, type); } } + else + { + op1 = impCloneExpr(op1, &op2, tiRetVal.GetClassHandle(), CHECK_SPILL_ALL, + nullptr DEBUGARG("DUP instruction")); + } + assert(!(op1->gtFlags & GTF_GLOB_EFFECT) && !(op2->gtFlags & GTF_GLOB_EFFECT)); impPushOnStack(op1, tiRetVal); - break; - - case CEE_ISINST: - { - /* Get the type token */ - assertImp(sz == sizeof(unsigned)); + impPushOnStack(op2, tiRetVal); + } + break; - _impResolveToken(CORINFO_TOKENKIND_Casting); + case CEE_STIND_I1: + lclTyp = TYP_BYTE; + goto STIND; + case CEE_STIND_I2: + lclTyp = TYP_SHORT; + goto STIND; + case CEE_STIND_I4: + lclTyp = TYP_INT; + goto STIND; + case CEE_STIND_I8: + lclTyp = TYP_LONG; + goto STIND; + case CEE_STIND_I: + lclTyp = TYP_I_IMPL; + goto STIND; + case CEE_STIND_REF: + lclTyp = TYP_REF; + goto STIND; + case CEE_STIND_R4: + lclTyp = TYP_FLOAT; + goto STIND; + case CEE_STIND_R8: + lclTyp = TYP_DOUBLE; + goto STIND; - JITDUMP(" %08X", resolvedToken.token); + STIND: + op2 = impPopStack().val; // value to store - if (!opts.IsReadyToRun()) - { - op2 = impTokenToHandle(&resolvedToken, nullptr, false); - if (op2 == nullptr) - { // compDonotInline() - return; - } - } + STIND_VALUE: + op1 = impPopStack().val; // address to store to - accessAllowedResult = - info.compCompHnd->canAccessClass(&resolvedToken, info.compMethodHnd, &calloutHelper); - impHandleAccessAllowed(accessAllowedResult, &calloutHelper); + // you can indirect off of a TYP_I_IMPL (if we are in C) or a BYREF + assertImp(genActualType(op1->gtType) == TYP_I_IMPL || op1->gtType == TYP_BYREF); - op1 = impPopStack().val; + impBashVarAddrsToI(op1, op2); - GenTree* optTree = impOptimizeCastClassOrIsInst(op1, &resolvedToken, false); + op2 = impImplicitR4orR8Cast(op2, lclTyp); - if (optTree != nullptr) +#ifdef TARGET_64BIT + // Automatic upcast for a GT_CNS_INT into TYP_I_IMPL + if ((op2->OperGet() == GT_CNS_INT) && varTypeIsI(lclTyp) && !varTypeIsI(op2->gtType)) { - impPushOnStack(optTree, tiRetVal); + op2->gtType = TYP_I_IMPL; } else { - -#ifdef FEATURE_READYTORUN - if (opts.IsReadyToRun()) + // Allow a downcast of op2 from TYP_I_IMPL into a 32-bit Int for x86 JIT compatibility + // + if (varTypeIsI(op2->gtType) && (genActualType(lclTyp) == TYP_INT)) { - GenTreeCall* opLookup = - impReadyToRunHelperToTree(&resolvedToken, CORINFO_HELP_READYTORUN_ISINSTANCEOF, TYP_REF, - nullptr, op1); - usingReadyToRunHelper = (opLookup != nullptr); - op1 = (usingReadyToRunHelper ? opLookup : op1); - - if (!usingReadyToRunHelper) - { - // TODO: ReadyToRun: When generic dictionary lookups are necessary, replace the lookup call - // and the isinstanceof_any call with a single call to a dynamic R2R cell that will: - // 1) Load the context - // 2) Perform the generic dictionary lookup and caching, and generate the appropriate - // stub - // 3) Perform the 'is instance' check on the input object - // Reason: performance (today, we'll always use the slow helper for the R2R generics case) - - op2 = impTokenToHandle(&resolvedToken, nullptr, false); - if (op2 == nullptr) - { // compDonotInline() - return; - } - } + op2 = gtNewCastNode(TYP_INT, op2, false, TYP_INT); + } + // Allow an upcast of op2 from a 32-bit Int into TYP_I_IMPL for x86 JIT compatibility + // + if (varTypeIsI(lclTyp) && (genActualType(op2->gtType) == TYP_INT)) + { + op2 = gtNewCastNode(TYP_I_IMPL, op2, false, TYP_I_IMPL); } + } +#endif // TARGET_64BIT - if (!usingReadyToRunHelper) -#endif + if (opcode == CEE_STIND_REF) + { + // STIND_REF can be used to store TYP_INT, TYP_I_IMPL, TYP_REF, or TYP_BYREF + assertImp(varTypeIsIntOrI(op2->gtType) || varTypeIsGC(op2->gtType)); + lclTyp = genActualType(op2->TypeGet()); + } + +// Check target type. +#ifdef DEBUG + if (op2->gtType == TYP_BYREF || lclTyp == TYP_BYREF) + { + if (op2->gtType == TYP_BYREF) { - op1 = impCastClassOrIsInstToTree(op1, op2, &resolvedToken, false, opcodeOffs); + assertImp(lclTyp == TYP_BYREF || lclTyp == TYP_I_IMPL); } - if (compDonotInline()) + else if (lclTyp == TYP_BYREF) { - return; + assertImp(op2->gtType == TYP_BYREF || varTypeIsIntOrI(op2->gtType)); } + } + else + { + assertImp(genActualType(op2->gtType) == genActualType(lclTyp) || + ((lclTyp == TYP_I_IMPL) && (genActualType(op2->gtType) == TYP_INT)) || + (varTypeIsFloating(op2->gtType) && varTypeIsFloating(lclTyp))); + } +#endif - impPushOnStack(op1, tiRetVal); + op1 = gtNewOperNode(GT_IND, lclTyp, op1); + op1->gtFlags |= GTF_EXCEPT | GTF_GLOB_REF; + + if (prefixFlags & PREFIX_VOLATILE) + { + assert(op1->OperGet() == GT_IND); + op1->gtFlags |= GTF_ORDER_SIDEEFF; // Prevent this from being reordered + op1->gtFlags |= GTF_IND_VOLATILE; } - break; - } - case CEE_REFANYVAL: - { + if ((prefixFlags & PREFIX_UNALIGNED) && !varTypeIsByte(lclTyp)) + { + assert(op1->OperGet() == GT_IND); + op1->gtFlags |= GTF_IND_UNALIGNED; + } - // get the class handle and make a ICON node out of it + op1 = gtNewAssignNode(op1, op2); + goto SPILL_APPEND; - _impResolveToken(CORINFO_TOKENKIND_Class); + case CEE_LDIND_I1: + lclTyp = TYP_BYTE; + goto LDIND; + case CEE_LDIND_I2: + lclTyp = TYP_SHORT; + goto LDIND; + case CEE_LDIND_U4: + case CEE_LDIND_I4: + lclTyp = TYP_INT; + goto LDIND; + case CEE_LDIND_I8: + lclTyp = TYP_LONG; + goto LDIND; + case CEE_LDIND_REF: + lclTyp = TYP_REF; + goto LDIND; + case CEE_LDIND_I: + lclTyp = TYP_I_IMPL; + goto LDIND; + case CEE_LDIND_R4: + lclTyp = TYP_FLOAT; + goto LDIND; + case CEE_LDIND_R8: + lclTyp = TYP_DOUBLE; + goto LDIND; + case CEE_LDIND_U1: + lclTyp = TYP_UBYTE; + goto LDIND; + case CEE_LDIND_U2: + lclTyp = TYP_USHORT; + goto LDIND; + LDIND: - JITDUMP(" %08X", resolvedToken.token); + op1 = impPopStack().val; // address to load from + impBashVarAddrsToI(op1); - op2 = impTokenToHandle(&resolvedToken); - if (op2 == nullptr) - { // compDonotInline() - return; +#ifdef TARGET_64BIT + // Allow an upcast of op1 from a 32-bit Int into TYP_I_IMPL for x86 JIT compatibility + // + if (genActualType(op1->gtType) == TYP_INT) + { + op1 = gtNewCastNode(TYP_I_IMPL, op1, false, TYP_I_IMPL); } +#endif - op1 = impPopStack().val; - // make certain it is normalized; - op1 = impNormStructVal(op1, impGetRefAnyClass(), (unsigned)CHECK_SPILL_ALL); + assertImp(genActualType(op1->gtType) == TYP_I_IMPL || op1->gtType == TYP_BYREF); - // Call helper GETREFANY(classHandle, op1); - GenTreeCall* helperCall = gtNewHelperCallNode(CORINFO_HELP_GETREFANY, TYP_BYREF); - NewCallArg clsHandleArg = NewCallArg::Primitive(op2); - NewCallArg typedRefArg = NewCallArg::Struct(op1, TYP_STRUCT, impGetRefAnyClass()); - helperCall->gtArgs.PushFront(this, clsHandleArg, typedRefArg); - helperCall->gtFlags |= (op1->gtFlags | op2->gtFlags) & GTF_ALL_EFFECT; - op1 = helperCall; + op1 = gtNewOperNode(GT_IND, lclTyp, op1); + + // ldind could point anywhere, example a boxed class static int + op1->gtFlags |= (GTF_EXCEPT | GTF_GLOB_REF); + + if (prefixFlags & PREFIX_VOLATILE) + { + assert(op1->OperGet() == GT_IND); + op1->gtFlags |= GTF_ORDER_SIDEEFF; // Prevent this from being reordered + op1->gtFlags |= GTF_IND_VOLATILE; + } + + if ((prefixFlags & PREFIX_UNALIGNED) && !varTypeIsByte(lclTyp)) + { + assert(op1->OperGet() == GT_IND); + op1->gtFlags |= GTF_IND_UNALIGNED; + } impPushOnStack(op1, tiRetVal); - break; - } - case CEE_REFANYTYPE: - op1 = impPopStack().val; + break; - // make certain it is normalized; - op1 = impNormStructVal(op1, impGetRefAnyClass(), (unsigned)CHECK_SPILL_ALL); + case CEE_UNALIGNED: - if (op1->gtOper == GT_OBJ) + assert(sz == 1); + val = getU1LittleEndian(codeAddr); + ++codeAddr; + JITDUMP(" %u", val); + if ((val != 1) && (val != 2) && (val != 4)) { - // Get the address of the refany - op1 = op1->AsOp()->gtOp1; - - // Fetch the type from the correct slot - op1 = gtNewOperNode(GT_ADD, TYP_BYREF, op1, - gtNewIconNode(OFFSETOF__CORINFO_TypedReference__type, TYP_I_IMPL)); - op1 = gtNewOperNode(GT_IND, TYP_BYREF, op1); + BADCODE("Alignment unaligned. must be 1, 2, or 4"); } - else - { - assertImp(op1->gtOper == GT_MKREFANY); - // The pointer may have side-effects - if (op1->AsOp()->gtOp1->gtFlags & GTF_SIDE_EFFECT) - { - impAppendTree(op1->AsOp()->gtOp1, (unsigned)CHECK_SPILL_ALL, impCurStmtDI); -#ifdef DEBUG - impNoteLastILoffs(); -#endif - } + Verify(!(prefixFlags & PREFIX_UNALIGNED), "Multiple unaligned. prefixes"); + prefixFlags |= PREFIX_UNALIGNED; - // We already have the class handle - op1 = op1->AsOp()->gtOp2; - } + impValidateMemoryAccessOpcode(codeAddr, codeEndp, false); - // convert native TypeHandle to RuntimeTypeHandle - { - op1 = gtNewHelperCallNode(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPEHANDLE_MAYBENULL, TYP_STRUCT, op1); + PREFIX: + opcode = (OPCODE)getU1LittleEndian(codeAddr); + opcodeOffs = (IL_OFFSET)(codeAddr - info.compCode); + codeAddr += sizeof(__int8); + goto DECODE_OPCODE; - CORINFO_CLASS_HANDLE classHandle = impGetTypeHandleClass(); + case CEE_VOLATILE: - // The handle struct is returned in register - op1->AsCall()->gtReturnType = GetRuntimeHandleUnderlyingType(); - op1->AsCall()->gtRetClsHnd = classHandle; -#if FEATURE_MULTIREG_RET - op1->AsCall()->InitializeStructReturnType(this, classHandle, op1->AsCall()->GetUnmanagedCallConv()); -#endif + Verify(!(prefixFlags & PREFIX_VOLATILE), "Multiple volatile. prefixes"); + prefixFlags |= PREFIX_VOLATILE; - tiRetVal = typeInfo(TI_STRUCT, classHandle); - } + impValidateMemoryAccessOpcode(codeAddr, codeEndp, true); - impPushOnStack(op1, tiRetVal); - break; + assert(sz == 0); + goto PREFIX; - case CEE_LDTOKEN: + case CEE_LDFTN: { - /* Get the Class index */ - assertImp(sz == sizeof(unsigned)); - lastLoadToken = codeAddr; - _impResolveToken(CORINFO_TOKENKIND_Ldtoken); - - tokenType = info.compCompHnd->getTokenTypeAsHandle(&resolvedToken); + // Need to do a lookup here so that we perform an access check + // and do a NOWAY if protections are violated + _impResolveToken(CORINFO_TOKENKIND_Method); - op1 = impTokenToHandle(&resolvedToken, nullptr, true); - if (op1 == nullptr) - { // compDonotInline() - return; - } + JITDUMP(" %08X", resolvedToken.token); - helper = CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPEHANDLE; - assert(resolvedToken.hClass != nullptr); + eeGetCallInfo(&resolvedToken, (prefixFlags & PREFIX_CONSTRAINED) ? &constrainedResolvedToken : nullptr, + combine(CORINFO_CALLINFO_SECURITYCHECKS, CORINFO_CALLINFO_LDFTN), &callInfo); - if (resolvedToken.hMethod != nullptr) + // This check really only applies to intrinsic Array.Address methods + if (callInfo.sig.callConv & CORINFO_CALLCONV_PARAMTYPE) { - helper = CORINFO_HELP_METHODDESC_TO_STUBRUNTIMEMETHOD; + NO_WAY("Currently do not support LDFTN of Parameterized functions"); } - else if (resolvedToken.hField != nullptr) + + // Do this before DO_LDFTN since CEE_LDVIRTFN does it on its own. + impHandleAccessAllowed(callInfo.accessAllowed, &callInfo.callsiteCalloutHelper); + + DO_LDFTN: + op1 = impMethodPointer(&resolvedToken, &callInfo); + + if (compDonotInline()) { - helper = CORINFO_HELP_FIELDDESC_TO_STUBRUNTIMEFIELD; + return; } - op1 = gtNewHelperCallNode(helper, TYP_STRUCT, op1); - - // The handle struct is returned in register and - // it could be consumed both as `TYP_STRUCT` and `TYP_REF`. - op1->AsCall()->gtReturnType = GetRuntimeHandleUnderlyingType(); -#if FEATURE_MULTIREG_RET - op1->AsCall()->InitializeStructReturnType(this, tokenType, op1->AsCall()->GetUnmanagedCallConv()); -#endif - op1->AsCall()->gtRetClsHnd = tokenType; + // Call info may have more precise information about the function than + // the resolved token. + mdToken constrainedToken = prefixFlags & PREFIX_CONSTRAINED ? constrainedResolvedToken.token : 0; + methodPointerInfo* heapToken = impAllocateMethodPointerInfo(resolvedToken, constrainedToken); + assert(callInfo.hMethod != nullptr); + heapToken->m_token.hMethod = callInfo.hMethod; + impPushOnStack(op1, typeInfo(heapToken)); - tiRetVal = verMakeTypeInfo(tokenType); - impPushOnStack(op1, tiRetVal); + break; } - break; - case CEE_UNBOX: - case CEE_UNBOX_ANY: + case CEE_LDVIRTFTN: { - /* Get the Class index */ - assertImp(sz == sizeof(unsigned)); + /* Get the method token */ - _impResolveToken(CORINFO_TOKENKIND_Class); + _impResolveToken(CORINFO_TOKENKIND_Method); JITDUMP(" %08X", resolvedToken.token); - bool runtimeLookup; - op2 = impTokenToHandle(&resolvedToken, &runtimeLookup); - if (op2 == nullptr) + eeGetCallInfo(&resolvedToken, nullptr /* constraint typeRef */, + combine(combine(CORINFO_CALLINFO_SECURITYCHECKS, CORINFO_CALLINFO_LDFTN), + CORINFO_CALLINFO_CALLVIRT), + &callInfo); + + // This check really only applies to intrinsic Array.Address methods + if (callInfo.sig.callConv & CORINFO_CALLCONV_PARAMTYPE) { - assert(compDonotInline()); - return; + NO_WAY("Currently do not support LDFTN of Parameterized functions"); } - // Run this always so we can get access exceptions even with SkipVerification. - accessAllowedResult = - info.compCompHnd->canAccessClass(&resolvedToken, info.compMethodHnd, &calloutHelper); - impHandleAccessAllowed(accessAllowedResult, &calloutHelper); + mflags = callInfo.methodFlags; - if (opcode == CEE_UNBOX_ANY && !eeIsValueClass(resolvedToken.hClass)) + impHandleAccessAllowed(callInfo.accessAllowed, &callInfo.callsiteCalloutHelper); + + if (compIsForInlining()) { - JITDUMP("\n Importing UNBOX.ANY(refClass) as CASTCLASS\n"); - op1 = impPopStack().val; - goto CASTCLASS; + if (mflags & (CORINFO_FLG_FINAL | CORINFO_FLG_STATIC) || !(mflags & CORINFO_FLG_VIRTUAL)) + { + compInlineResult->NoteFatal(InlineObservation::CALLSITE_LDVIRTFN_ON_NON_VIRTUAL); + return; + } } - /* Pop the object and create the unbox helper call */ - /* You might think that for UNBOX_ANY we need to push a different */ - /* (non-byref) type, but here we're making the tiRetVal that is used */ - /* for the intermediate pointer which we then transfer onto the OBJ */ - /* instruction. OBJ then creates the appropriate tiRetVal. */ + CORINFO_SIG_INFO& ftnSig = callInfo.sig; + /* Get the object-ref */ op1 = impPopStack().val; assertImp(op1->gtType == TYP_REF); - helper = info.compCompHnd->getUnBoxHelper(resolvedToken.hClass); - assert(helper == CORINFO_HELP_UNBOX || helper == CORINFO_HELP_UNBOX_NULLABLE); - - // Check legality and profitability of inline expansion for unboxing. - const bool canExpandInline = (helper == CORINFO_HELP_UNBOX); - const bool shouldExpandInline = !compCurBB->isRunRarely() && opts.OptimizationEnabled(); - - if (canExpandInline && shouldExpandInline) + if (opts.IsReadyToRun()) { - // See if we know anything about the type of op1, the object being unboxed. - bool isExact = false; - bool isNonNull = false; - CORINFO_CLASS_HANDLE clsHnd = gtGetClassHandle(op1, &isExact, &isNonNull); - - // We can skip the "exact" bit here as we are comparing to a value class. - // compareTypesForEquality should bail on comparisons for shared value classes. - if (clsHnd != NO_CLASS_HANDLE) + if (callInfo.kind != CORINFO_VIRTUALCALL_LDVIRTFTN) { - const TypeCompareState compare = - info.compCompHnd->compareTypesForEquality(resolvedToken.hClass, clsHnd); - - if (compare == TypeCompareState::Must) - { - JITDUMP("\nOptimizing %s (%s) -- type test will succeed\n", - opcode == CEE_UNBOX ? "UNBOX" : "UNBOX.ANY", eeGetClassName(clsHnd)); - - // For UNBOX, null check (if necessary), and then leave the box payload byref on the stack. - if (opcode == CEE_UNBOX) - { - GenTree* cloneOperand; - op1 = impCloneExpr(op1, &cloneOperand, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, - nullptr DEBUGARG("optimized unbox clone")); - - GenTree* boxPayloadOffset = gtNewIconNode(TARGET_POINTER_SIZE, TYP_I_IMPL); - GenTree* boxPayloadAddress = - gtNewOperNode(GT_ADD, TYP_BYREF, cloneOperand, boxPayloadOffset); - GenTree* nullcheck = gtNewNullCheck(op1, block); - GenTree* result = gtNewOperNode(GT_COMMA, TYP_BYREF, nullcheck, boxPayloadAddress); - impPushOnStack(result, tiRetVal); - break; - } - - // For UNBOX.ANY load the struct from the box payload byref (the load will nullcheck) - assert(opcode == CEE_UNBOX_ANY); - GenTree* boxPayloadOffset = gtNewIconNode(TARGET_POINTER_SIZE, TYP_I_IMPL); - GenTree* boxPayloadAddress = gtNewOperNode(GT_ADD, TYP_BYREF, op1, boxPayloadOffset); - impPushOnStack(boxPayloadAddress, tiRetVal); - oper = GT_OBJ; - goto OBJ; - } - else + if (op1->gtFlags & GTF_SIDE_EFFECT) { - JITDUMP("\nUnable to optimize %s -- can't resolve type comparison\n", - opcode == CEE_UNBOX ? "UNBOX" : "UNBOX.ANY"); + op1 = gtUnusedValNode(op1); + impAppendTree(op1, CHECK_SPILL_ALL, impCurStmtDI); } + goto DO_LDFTN; } - else + } + else if (mflags & (CORINFO_FLG_FINAL | CORINFO_FLG_STATIC) || !(mflags & CORINFO_FLG_VIRTUAL)) + { + if (op1->gtFlags & GTF_SIDE_EFFECT) { - JITDUMP("\nUnable to optimize %s -- class for [%06u] not known\n", - opcode == CEE_UNBOX ? "UNBOX" : "UNBOX.ANY", dspTreeID(op1)); + op1 = gtUnusedValNode(op1); + impAppendTree(op1, CHECK_SPILL_ALL, impCurStmtDI); } + goto DO_LDFTN; + } - JITDUMP("\n Importing %s as inline sequence\n", opcode == CEE_UNBOX ? "UNBOX" : "UNBOX.ANY"); - // we are doing normal unboxing - // inline the common case of the unbox helper - // UNBOX(exp) morphs into - // clone = pop(exp); - // ((*clone == typeToken) ? nop : helper(clone, typeToken)); - // push(clone + TARGET_POINTER_SIZE) - // - GenTree* cloneOperand; - op1 = impCloneExpr(op1, &cloneOperand, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, - nullptr DEBUGARG("inline UNBOX clone1")); - op1 = gtNewMethodTableLookup(op1); + GenTree* fptr = impImportLdvirtftn(op1, &resolvedToken, &callInfo); + if (compDonotInline()) + { + return; + } - GenTree* condBox = gtNewOperNode(GT_EQ, TYP_INT, op1, op2); + methodPointerInfo* heapToken = impAllocateMethodPointerInfo(resolvedToken, 0); - op1 = impCloneExpr(cloneOperand, &cloneOperand, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, - nullptr DEBUGARG("inline UNBOX clone2")); - op2 = impTokenToHandle(&resolvedToken); - if (op2 == nullptr) - { // compDonotInline() - return; - } - op1 = gtNewHelperCallNode(helper, TYP_VOID, op2, op1); + assert(heapToken->m_token.tokenType == CORINFO_TOKENKIND_Method); + assert(callInfo.hMethod != nullptr); - op1 = new (this, GT_COLON) GenTreeColon(TYP_VOID, gtNewNothingNode(), op1); - op1 = gtNewQmarkNode(TYP_VOID, condBox, op1->AsColon()); + heapToken->m_token.tokenType = CORINFO_TOKENKIND_Ldvirtftn; + heapToken->m_token.hMethod = callInfo.hMethod; + impPushOnStack(fptr, typeInfo(heapToken)); - // QMARK nodes cannot reside on the evaluation stack. Because there - // may be other trees on the evaluation stack that side-effect the - // sources of the UNBOX operation we must spill the stack. + break; + } - impAppendTree(op1, (unsigned)CHECK_SPILL_ALL, impCurStmtDI); + case CEE_CONSTRAINED: - // Create the address-expression to reference past the object header - // to the beginning of the value-type. Today this means adjusting - // past the base of the objects vtable field which is pointer sized. + assertImp(sz == sizeof(unsigned)); + impResolveToken(codeAddr, &constrainedResolvedToken, CORINFO_TOKENKIND_Constrained); + codeAddr += sizeof(unsigned); // prefix instructions must increment codeAddr manually + JITDUMP(" (%08X) ", constrainedResolvedToken.token); - op2 = gtNewIconNode(TARGET_POINTER_SIZE, TYP_I_IMPL); - op1 = gtNewOperNode(GT_ADD, TYP_BYREF, cloneOperand, op2); - } - else - { - JITDUMP("\n Importing %s as helper call because %s\n", opcode == CEE_UNBOX ? "UNBOX" : "UNBOX.ANY", - canExpandInline ? "want smaller code or faster jitting" : "inline expansion not legal"); + Verify(!(prefixFlags & PREFIX_CONSTRAINED), "Multiple constrained. prefixes"); + prefixFlags |= PREFIX_CONSTRAINED; - // Don't optimize, just call the helper and be done with it - op1 = gtNewHelperCallNode(helper, - (var_types)((helper == CORINFO_HELP_UNBOX) ? TYP_BYREF : TYP_STRUCT), op2, - op1); - if (op1->gtType == TYP_STRUCT) + { + OPCODE actualOpcode = impGetNonPrefixOpcode(codeAddr, codeEndp); + if (actualOpcode != CEE_CALLVIRT && actualOpcode != CEE_CALL && actualOpcode != CEE_LDFTN) { - op1->AsCall()->gtRetClsHnd = resolvedToken.hClass; + BADCODE("constrained. has to be followed by callvirt, call or ldftn"); } } - assert((helper == CORINFO_HELP_UNBOX && op1->gtType == TYP_BYREF) || // Unbox helper returns a byref. - (helper == CORINFO_HELP_UNBOX_NULLABLE && - varTypeIsStruct(op1)) // UnboxNullable helper returns a struct. - ); - - /* - ---------------------------------------------------------------------- - | \ helper | | | - | \ | | | - | \ | CORINFO_HELP_UNBOX | CORINFO_HELP_UNBOX_NULLABLE | - | \ | (which returns a BYREF) | (which returns a STRUCT) | | - | opcode \ | | | - |--------------------------------------------------------------------- - | UNBOX | push the BYREF | spill the STRUCT to a local, | - | | | push the BYREF to this local | - |--------------------------------------------------------------------- - | UNBOX_ANY | push a GT_OBJ of | push the STRUCT | - | | the BYREF | For Linux when the | - | | | struct is returned in two | - | | | registers create a temp | - | | | which address is passed to | - | | | the unbox_nullable helper. | - |--------------------------------------------------------------------- - */ - - if (opcode == CEE_UNBOX) - { - if (helper == CORINFO_HELP_UNBOX_NULLABLE) - { - // Unbox nullable helper returns a struct type. - // We need to spill it to a temp so than can take the address of it. - // Here we need unsafe value cls check, since the address of struct is taken to be used - // further along and potetially be exploitable. - - unsigned tmp = lvaGrabTemp(true DEBUGARG("UNBOXing a nullable")); - lvaSetStruct(tmp, resolvedToken.hClass, true /* unsafe value cls check */); + goto PREFIX; - op2 = gtNewLclvNode(tmp, TYP_STRUCT); - op1 = impAssignStruct(op2, op1, resolvedToken.hClass, (unsigned)CHECK_SPILL_ALL); - assert(op1->gtType == TYP_VOID); // We must be assigning the return struct to the temp. + case CEE_READONLY: + JITDUMP(" readonly."); - op2 = gtNewLclvNode(tmp, TYP_STRUCT); - op2 = gtNewOperNode(GT_ADDR, TYP_BYREF, op2); - op1 = gtNewOperNode(GT_COMMA, TYP_BYREF, op1, op2); - } + Verify(!(prefixFlags & PREFIX_READONLY), "Multiple readonly. prefixes"); + prefixFlags |= PREFIX_READONLY; - assert(op1->gtType == TYP_BYREF); - } - else { - assert(opcode == CEE_UNBOX_ANY); - - if (helper == CORINFO_HELP_UNBOX) + OPCODE actualOpcode = impGetNonPrefixOpcode(codeAddr, codeEndp); + if (actualOpcode != CEE_LDELEMA && !impOpcodeIsCallOpcode(actualOpcode)) { - // Normal unbox helper returns a TYP_BYREF. - impPushOnStack(op1, tiRetVal); - oper = GT_OBJ; - goto OBJ; + BADCODE("readonly. has to be followed by ldelema or call"); } + } - assert(helper == CORINFO_HELP_UNBOX_NULLABLE && "Make sure the helper is nullable!"); - -#if FEATURE_MULTIREG_RET - - if (varTypeIsStruct(op1) && - IsMultiRegReturnedType(resolvedToken.hClass, CorInfoCallConvExtension::Managed)) - { - // Unbox nullable helper returns a TYP_STRUCT. - // For the multi-reg case we need to spill it to a temp so that - // we can pass the address to the unbox_nullable jit helper. - - unsigned tmp = lvaGrabTemp(true DEBUGARG("UNBOXing a register returnable nullable")); - lvaTable[tmp].lvIsMultiRegArg = true; - lvaSetStruct(tmp, resolvedToken.hClass, true /* unsafe value cls check */); - - op2 = gtNewLclvNode(tmp, TYP_STRUCT); - op1 = impAssignStruct(op2, op1, resolvedToken.hClass, (unsigned)CHECK_SPILL_ALL); - assert(op1->gtType == TYP_VOID); // We must be assigning the return struct to the temp. - - op2 = gtNewLclvNode(tmp, TYP_STRUCT); - op2 = gtNewOperNode(GT_ADDR, TYP_BYREF, op2); - op1 = gtNewOperNode(GT_COMMA, TYP_BYREF, op1, op2); - - // In this case the return value of the unbox helper is TYP_BYREF. - // Make sure the right type is placed on the operand type stack. - impPushOnStack(op1, tiRetVal); - - // Load the struct. - oper = GT_OBJ; - - assert(op1->gtType == TYP_BYREF); + assert(sz == 0); + goto PREFIX; - goto OBJ; - } - else + case CEE_TAILCALL: + JITDUMP(" tail."); -#endif // !FEATURE_MULTIREG_RET + Verify(!(prefixFlags & PREFIX_TAILCALL_EXPLICIT), "Multiple tailcall. prefixes"); + prefixFlags |= PREFIX_TAILCALL_EXPLICIT; + { + OPCODE actualOpcode = impGetNonPrefixOpcode(codeAddr, codeEndp); + if (!impOpcodeIsCallOpcode(actualOpcode)) { - // If non register passable struct we have it materialized in the RetBuf. - assert(op1->gtType == TYP_STRUCT); - tiRetVal = verMakeTypeInfo(resolvedToken.hClass); - assert(tiRetVal.IsValueClass()); + BADCODE("tailcall. has to be followed by call, callvirt or calli"); } } + assert(sz == 0); + goto PREFIX; - impPushOnStack(op1, tiRetVal); - } - break; + case CEE_NEWOBJ: - case CEE_BOX: - { - /* Get the Class index */ - assertImp(sz == sizeof(unsigned)); + /* Since we will implicitly insert newObjThisPtr at the start of the + argument list, spill any GTF_ORDER_SIDEEFF */ + impSpillSpecialSideEff(); - _impResolveToken(CORINFO_TOKENKIND_Box); + /* NEWOBJ does not respond to TAIL */ + prefixFlags &= ~PREFIX_TAILCALL_EXPLICIT; - JITDUMP(" %08X", resolvedToken.token); + /* NEWOBJ does not respond to CONSTRAINED */ + prefixFlags &= ~PREFIX_CONSTRAINED; - accessAllowedResult = - info.compCompHnd->canAccessClass(&resolvedToken, info.compMethodHnd, &calloutHelper); - impHandleAccessAllowed(accessAllowedResult, &calloutHelper); + _impResolveToken(CORINFO_TOKENKIND_NewObj); - // Note BOX can be used on things that are not value classes, in which - // case we get a NOP. However the verifier's view of the type on the - // stack changes (in generic code a 'T' becomes a 'boxed T') - if (!eeIsValueClass(resolvedToken.hClass)) - { - JITDUMP("\n Importing BOX(refClass) as NOP\n"); - verCurrentState.esStack[verCurrentState.esStackDepth - 1].seTypeInfo = tiRetVal; - break; - } + eeGetCallInfo(&resolvedToken, nullptr /* constraint typeRef*/, + combine(CORINFO_CALLINFO_SECURITYCHECKS, CORINFO_CALLINFO_ALLOWINSTPARAM), &callInfo); - bool isByRefLike = - (info.compCompHnd->getClassAttribs(resolvedToken.hClass) & CORINFO_FLG_BYREF_LIKE) != 0; - if (isByRefLike) + mflags = callInfo.methodFlags; + + if ((mflags & (CORINFO_FLG_STATIC | CORINFO_FLG_ABSTRACT)) != 0) { - // For ByRefLike types we are required to either fold the - // recognized patterns in impBoxPatternMatch or otherwise - // throw InvalidProgramException at runtime. In either case - // we will need to spill side effects of the expression. - impSpillSideEffects(false, CHECK_SPILL_ALL DEBUGARG("Required for box of ByRefLike type")); + BADCODE("newobj on static or abstract method"); } - // Look ahead for box idioms - int matched = impBoxPatternMatch(&resolvedToken, codeAddr + sz, codeEndp, - isByRefLike ? BoxPatterns::IsByRefLike : BoxPatterns::None); - if (matched >= 0) + // Insert the security callout before any actual code is generated + impHandleAccessAllowed(callInfo.accessAllowed, &callInfo.callsiteCalloutHelper); + + // There are three different cases for new. + // Object size is variable (depends on arguments). + // 1) Object is an array (arrays treated specially by the EE) + // 2) Object is some other variable sized object (e.g. String) + // 3) Class Size can be determined beforehand (normal case) + // In the first case, we need to call a NEWOBJ helper (multinewarray). + // In the second case we call the constructor with a '0' this pointer. + // In the third case we alloc the memory, then call the constructor. + + clsFlags = callInfo.classFlags; + if (clsFlags & CORINFO_FLG_ARRAY) { - // Skip the matched IL instructions - sz += matched; + // Arrays need to call the NEWOBJ helper. + assertImp(clsFlags & CORINFO_FLG_VAROBJSIZE); + + impImportNewObjArray(&resolvedToken, &callInfo); + if (compDonotInline()) + { + return; + } + + callTyp = TYP_REF; break; } - - if (isByRefLike) + // At present this can only be String + else if (clsFlags & CORINFO_FLG_VAROBJSIZE) { - // ByRefLike types are supported in boxing scenarios when the instruction can be elided - // due to a recognized pattern above. If the pattern is not recognized, the code is invalid. - BADCODE("ByRefLike types cannot be boxed"); + // Skip this thisPtr argument + newObjThisPtr = nullptr; + + /* Remember that this basic block contains 'new' of an object */ + block->bbFlags |= BBF_HAS_NEWOBJ; + optMethodFlags |= OMF_HAS_NEWOBJ; } else { - impImportAndPushBox(&resolvedToken); + // This is the normal case where the size of the object is + // fixed. Allocate the memory and call the constructor. + + // Note: We cannot add a peep to avoid use of temp here + // because we don't have enough interference info to detect when + // sources and destination interfere, example: s = new S(ref); + + // TODO: We find the correct place to introduce a general + // reverse copy prop for struct return values from newobj or + // any function returning structs. + + /* get a temporary for the new object */ + lclNum = lvaGrabTemp(true DEBUGARG("NewObj constructor temp")); if (compDonotInline()) { + // Fail fast if lvaGrabTemp fails with CALLSITE_TOO_MANY_LOCALS. + assert(compInlineResult->GetObservation() == InlineObservation::CALLSITE_TOO_MANY_LOCALS); return; } - } - } - break; - case CEE_SIZEOF: + // In the value class case we only need clsHnd for size calcs. + // + // The lookup of the code pointer will be handled by CALL in this case + if (clsFlags & CORINFO_FLG_VALUECLASS) + { + if (compIsForInlining()) + { + // If value class has GC fields, inform the inliner. It may choose to + // bail out on the inline. + DWORD typeFlags = info.compCompHnd->getClassAttribs(resolvedToken.hClass); + if ((typeFlags & CORINFO_FLG_CONTAINS_GC_PTR) != 0) + { + compInlineResult->Note(InlineObservation::CALLEE_HAS_GC_STRUCT); + if (compInlineResult->IsFailure()) + { + return; + } - /* Get the Class index */ - assertImp(sz == sizeof(unsigned)); + // Do further notification in the case where the call site is rare; + // some policies do not track the relative hotness of call sites for + // "always" inline cases. + if (impInlineInfo->iciBlock->isRunRarely()) + { + compInlineResult->Note(InlineObservation::CALLSITE_RARE_GC_STRUCT); + if (compInlineResult->IsFailure()) + { + return; + } + } + } + } - _impResolveToken(CORINFO_TOKENKIND_Class); + CorInfoType jitTyp = info.compCompHnd->asCorInfoType(resolvedToken.hClass); - JITDUMP(" %08X", resolvedToken.token); + if (impIsPrimitive(jitTyp)) + { + lvaTable[lclNum].lvType = JITtype2varType(jitTyp); + } + else + { + // The local variable itself is the allocated space. + // Here we need unsafe value cls check, since the address of struct is taken for further use + // and potentially exploitable. + lvaSetStruct(lclNum, resolvedToken.hClass, true /* unsafe value cls check */); + } - op1 = gtNewIconNode(info.compCompHnd->getClassSize(resolvedToken.hClass)); - impPushOnStack(op1, tiRetVal); - break; + bool bbInALoop = impBlockIsInALoop(block); + bool bbIsReturn = (block->bbJumpKind == BBJ_RETURN) && + (!compIsForInlining() || (impInlineInfo->iciBlock->bbJumpKind == BBJ_RETURN)); + LclVarDsc* const lclDsc = lvaGetDesc(lclNum); + if (fgVarNeedsExplicitZeroInit(lclNum, bbInALoop, bbIsReturn)) + { + // Append a tree to zero-out the temp + GenTree* newObjDst = gtNewLclvNode(lclNum, lclDsc->TypeGet()); + GenTree* newObjInit; + if (lclDsc->TypeGet() == TYP_STRUCT) + { + newObjInit = gtNewBlkOpNode(newObjDst, gtNewIconNode(0)); + } + else + { + newObjInit = gtNewAssignNode(newObjDst, gtNewZeroConNode(lclDsc->TypeGet())); + } + impAppendTree(newObjInit, CHECK_SPILL_NONE, impCurStmtDI); + } + else + { + JITDUMP("\nSuppressing zero-init for V%02u -- expect to zero in prolog\n", lclNum); + lclDsc->lvSuppressedZeroInit = 1; + compSuppressedZeroInit = true; + } - case CEE_CASTCLASS: + // The constructor may store "this", with subsequent code mutating the underlying local + // through the captured reference. To correctly spill the node we'll push onto the stack + // in such a case, we must mark the temp as potentially aliased. + lclDsc->lvHasLdAddrOp = true; - /* Get the Class index */ + // Obtain the address of the temp + newObjThisPtr = gtNewLclVarAddrNode(lclNum, TYP_BYREF); + } + else + { + // If we're newing up a finalizable object, spill anything that can cause exceptions. + // + bool hasSideEffects = false; + CorInfoHelpFunc newHelper = + info.compCompHnd->getNewHelper(&resolvedToken, info.compMethodHnd, &hasSideEffects); - assertImp(sz == sizeof(unsigned)); + if (hasSideEffects) + { + JITDUMP("\nSpilling stack for finalizable newobj\n"); + impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("finalizable newobj spill")); + } - _impResolveToken(CORINFO_TOKENKIND_Casting); + const bool useParent = true; + op1 = gtNewAllocObjNode(&resolvedToken, useParent); + if (op1 == nullptr) + { + return; + } - JITDUMP(" %08X", resolvedToken.token); + // Remember that this basic block contains 'new' of an object + block->bbFlags |= BBF_HAS_NEWOBJ; + optMethodFlags |= OMF_HAS_NEWOBJ; - if (!opts.IsReadyToRun()) + // Append the assignment to the temp/local. Dont need to spill + // at all as we are just calling an EE-Jit helper which can only + // cause an (async) OutOfMemoryException. + + // We assign the newly allocated object (by a GT_ALLOCOBJ node) + // to a temp. Note that the pattern "temp = allocObj" is required + // by ObjectAllocator phase to be able to determine GT_ALLOCOBJ nodes + // without exhaustive walk over all expressions. + + impAssignTempGen(lclNum, op1, CHECK_SPILL_NONE); + + assert(lvaTable[lclNum].lvSingleDef == 0); + lvaTable[lclNum].lvSingleDef = 1; + JITDUMP("Marked V%02u as a single def local\n", lclNum); + lvaSetClass(lclNum, resolvedToken.hClass, true /* is Exact */); + + newObjThisPtr = gtNewLclvNode(lclNum, TYP_REF); + } + } + goto CALL; + + case CEE_CALLI: + + /* CALLI does not respond to CONSTRAINED */ + prefixFlags &= ~PREFIX_CONSTRAINED; + + FALLTHROUGH; + + case CEE_CALLVIRT: + case CEE_CALL: + + // We can't call getCallInfo on the token from a CALLI, but we need it in + // many other places. We unfortunately embed that knowledge here. + if (opcode != CEE_CALLI) + { + _impResolveToken(CORINFO_TOKENKIND_Method); + + eeGetCallInfo(&resolvedToken, + (prefixFlags & PREFIX_CONSTRAINED) ? &constrainedResolvedToken : nullptr, + // this is how impImportCall invokes getCallInfo + combine(combine(CORINFO_CALLINFO_ALLOWINSTPARAM, CORINFO_CALLINFO_SECURITYCHECKS), + (opcode == CEE_CALLVIRT) ? CORINFO_CALLINFO_CALLVIRT : CORINFO_CALLINFO_NONE), + &callInfo); + } + else { - op2 = impTokenToHandle(&resolvedToken, nullptr, false); - if (op2 == nullptr) - { // compDonotInline() - return; - } + // Suppress uninitialized use warning. + memset(&resolvedToken, 0, sizeof(resolvedToken)); + memset(&callInfo, 0, sizeof(callInfo)); + + resolvedToken.token = getU4LittleEndian(codeAddr); + resolvedToken.tokenContext = impTokenLookupContextHandle; + resolvedToken.tokenScope = info.compScopeHnd; } - accessAllowedResult = - info.compCompHnd->canAccessClass(&resolvedToken, info.compMethodHnd, &calloutHelper); - impHandleAccessAllowed(accessAllowedResult, &calloutHelper); + CALL: // memberRef should be set. + // newObjThisPtr should be set for CEE_NEWOBJ - op1 = impPopStack().val; + JITDUMP(" %08X", resolvedToken.token); + constraintCall = (prefixFlags & PREFIX_CONSTRAINED) != 0; - /* Pop the address and create the 'checked cast' helper call */ + bool newBBcreatedForTailcallStress; + bool passedStressModeValidation; - // At this point we expect typeRef to contain the token, op1 to contain the value being cast, - // and op2 to contain code that creates the type handle corresponding to typeRef - CASTCLASS: - { - GenTree* optTree = impOptimizeCastClassOrIsInst(op1, &resolvedToken, true); + newBBcreatedForTailcallStress = false; + passedStressModeValidation = true; - if (optTree != nullptr) + if (compIsForInlining()) { - impPushOnStack(optTree, tiRetVal); + if (compDonotInline()) + { + return; + } + // We rule out inlinees with explicit tail calls in fgMakeBasicBlocks. + assert((prefixFlags & PREFIX_TAILCALL_EXPLICIT) == 0); } else { - -#ifdef FEATURE_READYTORUN - if (opts.IsReadyToRun()) + if (compTailCallStress()) { - GenTreeCall* opLookup = - impReadyToRunHelperToTree(&resolvedToken, CORINFO_HELP_READYTORUN_CHKCAST, TYP_REF, nullptr, - op1); - usingReadyToRunHelper = (opLookup != nullptr); - op1 = (usingReadyToRunHelper ? opLookup : op1); + // Have we created a new BB after the "call" instruction in fgMakeBasicBlocks()? + // Tail call stress only recognizes call+ret patterns and forces them to be + // explicit tail prefixed calls. Also fgMakeBasicBlocks() under tail call stress + // doesn't import 'ret' opcode following the call into the basic block containing + // the call instead imports it to a new basic block. Note that fgMakeBasicBlocks() + // is already checking that there is an opcode following call and hence it is + // safe here to read next opcode without bounds check. + newBBcreatedForTailcallStress = + impOpcodeIsCallOpcode(opcode) && // Current opcode is a CALL, (not a CEE_NEWOBJ). So, don't + // make it jump to RET. + (OPCODE)getU1LittleEndian(codeAddr + sz) == CEE_RET; // Next opcode is a CEE_RET - if (!usingReadyToRunHelper) + bool hasTailPrefix = (prefixFlags & PREFIX_TAILCALL_EXPLICIT); + if (newBBcreatedForTailcallStress && !hasTailPrefix) { - // TODO: ReadyToRun: When generic dictionary lookups are necessary, replace the lookup call - // and the chkcastany call with a single call to a dynamic R2R cell that will: - // 1) Load the context - // 2) Perform the generic dictionary lookup and caching, and generate the appropriate - // stub - // 3) Check the object on the stack for the type-cast - // Reason: performance (today, we'll always use the slow helper for the R2R generics case) + // Do a more detailed evaluation of legality + const bool passedConstraintCheck = + verCheckTailCallConstraint(opcode, &resolvedToken, + constraintCall ? &constrainedResolvedToken : nullptr); - op2 = impTokenToHandle(&resolvedToken, nullptr, false); - if (op2 == nullptr) - { // compDonotInline() - return; + // Avoid setting compHasBackwardsJump = true via tail call stress if the method cannot have + // patchpoints. + // + const bool mayHavePatchpoints = opts.jitFlags->IsSet(JitFlags::JIT_FLAG_TIER0) && + (JitConfig.TC_OnStackReplacement() > 0) && + compCanHavePatchpoints(); + if (passedConstraintCheck && (mayHavePatchpoints || compHasBackwardJump)) + { + // Now check with the runtime + CORINFO_METHOD_HANDLE declaredCalleeHnd = callInfo.hMethod; + bool isVirtual = (callInfo.kind == CORINFO_VIRTUALCALL_STUB) || + (callInfo.kind == CORINFO_VIRTUALCALL_VTABLE); + CORINFO_METHOD_HANDLE exactCalleeHnd = isVirtual ? nullptr : declaredCalleeHnd; + if (info.compCompHnd->canTailCall(info.compMethodHnd, declaredCalleeHnd, exactCalleeHnd, + hasTailPrefix)) // Is it legal to do tailcall? + { + // Stress the tailcall. + JITDUMP(" (Tailcall stress: prefixFlags |= PREFIX_TAILCALL_EXPLICIT)"); + prefixFlags |= PREFIX_TAILCALL_EXPLICIT; + prefixFlags |= PREFIX_TAILCALL_STRESS; + } + else + { + // Runtime disallows this tail call + JITDUMP(" (Tailcall stress: runtime preventing tailcall)"); + passedStressModeValidation = false; + } + } + else + { + // Constraints disallow this tail call + JITDUMP(" (Tailcall stress: constraint check failed)"); + passedStressModeValidation = false; } } } + } - if (!usingReadyToRunHelper) -#endif + // This is split up to avoid goto flow warnings. + bool isRecursive; + isRecursive = !compIsForInlining() && (callInfo.hMethod == info.compMethodHnd); + + // If we've already disqualified this call as a tail call under tail call stress, + // don't consider it for implicit tail calling either. + // + // When not running under tail call stress, we may mark this call as an implicit + // tail call candidate. We'll do an "equivalent" validation during impImportCall. + // + // Note that when running under tail call stress, a call marked as explicit + // tail prefixed will not be considered for implicit tail calling. + if (passedStressModeValidation && + impIsImplicitTailCallCandidate(opcode, codeAddr + sz, codeEndp, prefixFlags, isRecursive)) + { + if (compIsForInlining()) { - op1 = impCastClassOrIsInstToTree(op1, op2, &resolvedToken, true, opcodeOffs); +#if FEATURE_TAILCALL_OPT_SHARED_RETURN + // Are we inlining at an implicit tail call site? If so the we can flag + // implicit tail call sites in the inline body. These call sites + // often end up in non BBJ_RETURN blocks, so only flag them when + // we're able to handle shared returns. + if (impInlineInfo->iciCall->IsImplicitTailCall()) + { + JITDUMP("\n (Inline Implicit Tail call: prefixFlags |= PREFIX_TAILCALL_IMPLICIT)"); + prefixFlags |= PREFIX_TAILCALL_IMPLICIT; + } +#endif // FEATURE_TAILCALL_OPT_SHARED_RETURN } - if (compDonotInline()) + else { - return; + JITDUMP("\n (Implicit Tail call: prefixFlags |= PREFIX_TAILCALL_IMPLICIT)"); + prefixFlags |= PREFIX_TAILCALL_IMPLICIT; } - - /* Push the result back on the stack */ - impPushOnStack(op1, tiRetVal); } - } - break; - case CEE_THROW: - - // Any block with a throw is rarely executed. - block->bbSetRunRarely(); + // Treat this call as tail call for verification only if "tail" prefixed (i.e. explicit tail call). + explicitTailCall = (prefixFlags & PREFIX_TAILCALL_EXPLICIT) != 0; + readonlyCall = (prefixFlags & PREFIX_READONLY) != 0; - // Pop the exception object and create the 'throw' helper call - op1 = gtNewHelperCallNode(CORINFO_HELP_THROW, TYP_VOID, impPopStack().val); + if (opcode != CEE_CALLI && opcode != CEE_NEWOBJ) + { + // All calls and delegates need a security callout. + // For delegates, this is the call to the delegate constructor, not the access check on the + // LD(virt)FTN. + impHandleAccessAllowed(callInfo.accessAllowed, &callInfo.callsiteCalloutHelper); + } - // Fall through to clear out the eval stack. + callTyp = impImportCall(opcode, &resolvedToken, constraintCall ? &constrainedResolvedToken : nullptr, + newObjThisPtr, prefixFlags, &callInfo, opcodeOffs); + if (compDonotInline()) + { + // We do not check fails after lvaGrabTemp. It is covered with CoreCLR_13272 issue. + assert((callTyp == TYP_UNDEF) || + (compInlineResult->GetObservation() == InlineObservation::CALLSITE_TOO_MANY_LOCALS)); + return; + } - EVAL_APPEND: - if (verCurrentState.esStackDepth > 0) + if (explicitTailCall || newBBcreatedForTailcallStress) // If newBBcreatedForTailcallStress is true, we + // have created a new BB after the "call" + // instruction in fgMakeBasicBlocks(). So we need to jump to RET regardless. { - impEvalSideEffects(); + assert(!compIsForInlining()); + goto RET; } - assert(verCurrentState.esStackDepth == 0); + break; - goto APPEND; + case CEE_LDFLD: + case CEE_LDSFLD: + case CEE_LDFLDA: + case CEE_LDSFLDA: + { - case CEE_RETHROW: + bool isLoadAddress = (opcode == CEE_LDFLDA || opcode == CEE_LDSFLDA); + bool isLoadStatic = (opcode == CEE_LDSFLD || opcode == CEE_LDSFLDA); - assert(!compIsForInlining()); + /* Get the CP_Fieldref index */ + assertImp(sz == sizeof(unsigned)); - if (info.compXcptnsCount == 0) + _impResolveToken(CORINFO_TOKENKIND_Field); + + JITDUMP(" %08X", resolvedToken.token); + + int aflags = isLoadAddress ? CORINFO_ACCESS_ADDRESS : CORINFO_ACCESS_GET; + + GenTree* obj = nullptr; + CORINFO_CLASS_HANDLE objType = nullptr; // used for fields + + if ((opcode == CEE_LDFLD) || (opcode == CEE_LDFLDA)) { - BADCODE("rethrow outside catch"); + StackEntry se = impPopStack(); + objType = se.seTypeInfo.GetClassHandle(); + obj = se.val; + + if (impIsThis(obj)) + { + aflags |= CORINFO_ACCESS_THIS; + } } - /* Create the 'rethrow' helper call */ + eeGetFieldInfo(&resolvedToken, (CORINFO_ACCESS_FLAGS)aflags, &fieldInfo); - op1 = gtNewHelperCallNode(CORINFO_HELP_RETHROW, TYP_VOID); + // Figure out the type of the member. We always call canAccessField, so you always need this + // handle + CorInfoType ciType = fieldInfo.fieldType; + clsHnd = fieldInfo.structType; - goto EVAL_APPEND; + lclTyp = JITtype2varType(ciType); - case CEE_INITOBJ: + if (compIsForInlining()) + { + switch (fieldInfo.fieldAccessor) + { + case CORINFO_FIELD_INSTANCE_HELPER: + case CORINFO_FIELD_INSTANCE_ADDR_HELPER: + case CORINFO_FIELD_STATIC_ADDR_HELPER: + case CORINFO_FIELD_STATIC_TLS: - assertImp(sz == sizeof(unsigned)); + compInlineResult->NoteFatal(InlineObservation::CALLEE_LDFLD_NEEDS_HELPER); + return; - _impResolveToken(CORINFO_TOKENKIND_Class); + case CORINFO_FIELD_STATIC_GENERICS_STATIC_HELPER: + case CORINFO_FIELD_STATIC_READYTORUN_HELPER: + /* We may be able to inline the field accessors in specific instantiations of generic + * methods */ + compInlineResult->NoteFatal(InlineObservation::CALLSITE_LDFLD_NEEDS_HELPER); + return; - JITDUMP(" %08X", resolvedToken.token); + default: + break; + } + + if (!isLoadAddress && (fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC) && lclTyp == TYP_STRUCT && + clsHnd) + { + if ((info.compCompHnd->getTypeForPrimitiveValueClass(clsHnd) == CORINFO_TYPE_UNDEF) && + !(info.compFlags & CORINFO_FLG_FORCEINLINE)) + { + // Loading a static valuetype field usually will cause a JitHelper to be called + // for the static base. This will bloat the code. + compInlineResult->Note(InlineObservation::CALLEE_LDFLD_STATIC_VALUECLASS); - op2 = gtNewIconNode(0); // Value - op1 = impPopStack().val; // Dest + if (compInlineResult->IsFailure()) + { + return; + } + } + } + } - if (eeIsValueClass(resolvedToken.hClass)) + tiRetVal = verMakeTypeInfo(ciType, clsHnd); + if (isLoadAddress) { - op1 = gtNewStructVal(typGetObjLayout(resolvedToken.hClass), op1); - if (op1->OperIs(GT_OBJ)) - { - gtSetObjGcInfo(op1->AsObj()); - } + tiRetVal.MakeByRef(); } else { - size = info.compCompHnd->getClassSize(resolvedToken.hClass); - assert(size == TARGET_POINTER_SIZE); - op1 = gtNewBlockVal(op1, size); + tiRetVal.NormaliseForStack(); } - op1 = gtNewBlkOpNode(op1, op2, (prefixFlags & PREFIX_VOLATILE) != 0, false); - goto SPILL_APPEND; - - case CEE_INITBLK: - - op3 = impPopStack().val; // Size - op2 = impPopStack().val; // Value - op1 = impPopStack().val; // Dst addr + // Perform this check always to ensure that we get field access exceptions even with + // SkipVerification. + impHandleAccessAllowed(fieldInfo.accessAllowed, &fieldInfo.accessCalloutHelper); - if (op3->IsCnsIntOrI()) + // Raise InvalidProgramException if static load accesses non-static field + if (isLoadStatic && ((fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC) == 0)) { - size = (unsigned)op3->AsIntConCommon()->IconValue(); - op1 = new (this, GT_BLK) GenTreeBlk(GT_BLK, TYP_STRUCT, op1, typGetBlkLayout(size)); - op1 = gtNewBlkOpNode(op1, op2, (prefixFlags & PREFIX_VOLATILE) != 0, false); + BADCODE("static access on an instance field"); } - else - { - if (!op2->IsIntegralConst(0)) - { - op2 = gtNewOperNode(GT_INIT_VAL, TYP_INT, op2); - } - - op1 = new (this, GT_STORE_DYN_BLK) GenTreeStoreDynBlk(op1, op2, op3); - size = 0; - if ((prefixFlags & PREFIX_VOLATILE) != 0) + // We are using ldfld/a on a static field. We allow it, but need to get side-effect from obj. + if ((fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC) && obj != nullptr) + { + if (obj->gtFlags & GTF_SIDE_EFFECT) { - op1->gtFlags |= GTF_BLK_VOLATILE; + obj = gtUnusedValNode(obj); + impAppendTree(obj, CHECK_SPILL_ALL, impCurStmtDI); } + obj = nullptr; } - goto SPILL_APPEND; - - case CEE_CPBLK: - - op3 = impPopStack().val; // Size - op2 = impPopStack().val; // Src addr - op1 = impPopStack().val; // Dst addr - - if (op2->OperGet() == GT_ADDR) - { - op2 = op2->AsOp()->gtOp1; - } - else + /* Preserve 'small' int types */ + if (!varTypeIsSmall(lclTyp)) { - op2 = gtNewOperNode(GT_IND, TYP_STRUCT, op2); + lclTyp = genActualType(lclTyp); } - if (op3->IsCnsIntOrI()) - { - size = (unsigned)op3->AsIntConCommon()->IconValue(); - op1 = new (this, GT_BLK) GenTreeBlk(GT_BLK, TYP_STRUCT, op1, typGetBlkLayout(size)); - op1 = gtNewBlkOpNode(op1, op2, (prefixFlags & PREFIX_VOLATILE) != 0, true); - } - else - { - op1 = new (this, GT_STORE_DYN_BLK) GenTreeStoreDynBlk(op1, op2, op3); - size = 0; + bool usesHelper = false; - if ((prefixFlags & PREFIX_VOLATILE) != 0) + switch (fieldInfo.fieldAccessor) + { + case CORINFO_FIELD_INSTANCE: +#ifdef FEATURE_READYTORUN + case CORINFO_FIELD_INSTANCE_WITH_BASE: +#endif { - op1->gtFlags |= GTF_BLK_VOLATILE; - } - } - - goto SPILL_APPEND; + // If the object is a struct, what we really want is + // for the field to operate on the address of the struct. + if (varTypeIsStruct(obj)) + { + if (opcode != CEE_LDFLD) + { + BADCODE3("Unexpected opcode (has to be LDFLD)", ": %02X", (int)opcode); + } + if (objType == nullptr) + { + BADCODE("top of stack must be a value type"); + } + obj = impGetStructAddr(obj, objType, CHECK_SPILL_ALL, true); + } - case CEE_CPOBJ: + if (isLoadAddress) + { + op1 = gtNewFieldAddrNode(varTypeIsGC(obj) ? TYP_BYREF : TYP_I_IMPL, resolvedToken.hField, + obj, fieldInfo.offset); + } + else + { + op1 = gtNewFieldRef(lclTyp, resolvedToken.hField, obj, fieldInfo.offset); + } - assertImp(sz == sizeof(unsigned)); +#ifdef FEATURE_READYTORUN + if (fieldInfo.fieldAccessor == CORINFO_FIELD_INSTANCE_WITH_BASE) + { + op1->AsField()->gtFieldLookup = fieldInfo.fieldLookup; + } +#endif - _impResolveToken(CORINFO_TOKENKIND_Class); + if (fgAddrCouldBeNull(obj)) + { + op1->gtFlags |= GTF_EXCEPT; + } - JITDUMP(" %08X", resolvedToken.token); + if (StructHasOverlappingFields(info.compCompHnd->getClassAttribs(resolvedToken.hClass))) + { + op1->AsField()->gtFldMayOverlap = true; + } - if (!eeIsValueClass(resolvedToken.hClass)) - { - op1 = impPopStack().val; // address to load from + if (!isLoadAddress && compIsForInlining() && + impInlineIsGuaranteedThisDerefBeforeAnySideEffects(nullptr, nullptr, obj, + impInlineInfo->inlArgInfo)) + { + impInlineInfo->thisDereferencedFirst = true; + } + } + break; - impBashVarAddrsToI(op1); + case CORINFO_FIELD_STATIC_TLS: +#ifdef TARGET_X86 + // Legacy TLS access is implemented as intrinsic on x86 only + op1 = gtNewFieldAddrNode(TYP_I_IMPL, resolvedToken.hField, nullptr, fieldInfo.offset); + op1->gtFlags |= GTF_FLD_TLS; // fgMorphExpandTlsField will handle the transformation. - assertImp(genActualType(op1->gtType) == TYP_I_IMPL || op1->gtType == TYP_BYREF); + if (!isLoadAddress) + { + if (varTypeIsStruct(lclTyp)) + { + op1 = gtNewObjNode(fieldInfo.structType, op1); + op1->gtFlags |= GTF_IND_NONFAULTING; + } + else + { + op1 = gtNewIndir(lclTyp, op1, GTF_IND_NONFAULTING); + op1->gtFlags |= GTF_GLOB_REF; + } + } + break; +#else + fieldInfo.fieldAccessor = CORINFO_FIELD_STATIC_ADDR_HELPER; + FALLTHROUGH; +#endif + case CORINFO_FIELD_STATIC_ADDR_HELPER: + case CORINFO_FIELD_INSTANCE_HELPER: + case CORINFO_FIELD_INSTANCE_ADDR_HELPER: + op1 = gtNewRefCOMfield(obj, &resolvedToken, (CORINFO_ACCESS_FLAGS)aflags, &fieldInfo, lclTyp, + clsHnd, nullptr); + usesHelper = true; + break; - op1 = gtNewOperNode(GT_IND, TYP_REF, op1); - op1->gtFlags |= GTF_EXCEPT | GTF_GLOB_REF; + case CORINFO_FIELD_STATIC_SHARED_STATIC_HELPER: + case CORINFO_FIELD_STATIC_ADDRESS: + // Replace static read-only fields with constant if possible + if ((aflags & CORINFO_ACCESS_GET) && (fieldInfo.fieldFlags & CORINFO_FLG_FIELD_FINAL)) + { + GenTree* newTree = impImportStaticReadOnlyField(resolvedToken.hField, resolvedToken.hClass); - impPushOnStack(op1, typeInfo()); - opcode = CEE_STIND_REF; - lclTyp = TYP_REF; - goto STIND; - } + if (newTree != nullptr) + { + op1 = newTree; + goto FIELD_DONE; + } + } + FALLTHROUGH; - op2 = impPopStack().val; // Src - op1 = impPopStack().val; // Dest - op1 = gtNewCpObjNode(op1, op2, resolvedToken.hClass, ((prefixFlags & PREFIX_VOLATILE) != 0)); - goto SPILL_APPEND; + case CORINFO_FIELD_STATIC_RVA_ADDRESS: + case CORINFO_FIELD_STATIC_GENERICS_STATIC_HELPER: + case CORINFO_FIELD_STATIC_READYTORUN_HELPER: + op1 = impImportStaticFieldAccess(&resolvedToken, (CORINFO_ACCESS_FLAGS)aflags, &fieldInfo, + lclTyp); + break; - case CEE_STOBJ: - { - assertImp(sz == sizeof(unsigned)); + case CORINFO_FIELD_INTRINSIC_ZERO: + { + assert(aflags & CORINFO_ACCESS_GET); + // Widen to stack type + lclTyp = genActualType(lclTyp); + op1 = gtNewIconNode(0, lclTyp); + goto FIELD_DONE; + } + break; - _impResolveToken(CORINFO_TOKENKIND_Class); + case CORINFO_FIELD_INTRINSIC_EMPTY_STRING: + { + assert(aflags & CORINFO_ACCESS_GET); - JITDUMP(" %08X", resolvedToken.token); + // Import String.Empty as "" (GT_CNS_STR with a fake SconCPX = 0) + op1 = gtNewSconNode(EMPTY_STRING_SCON, nullptr); + goto FIELD_DONE; + } + break; - if (eeIsValueClass(resolvedToken.hClass)) - { - lclTyp = TYP_STRUCT; - } - else - { - lclTyp = TYP_REF; - } + case CORINFO_FIELD_INTRINSIC_ISLITTLEENDIAN: + { + assert(aflags & CORINFO_ACCESS_GET); + // Widen to stack type + lclTyp = genActualType(lclTyp); +#if BIGENDIAN + op1 = gtNewIconNode(0, lclTyp); +#else + op1 = gtNewIconNode(1, lclTyp); +#endif + goto FIELD_DONE; + } + break; - if (lclTyp == TYP_REF) - { - opcode = CEE_STIND_REF; - goto STIND; + default: + assert(!"Unexpected fieldAccessor"); } - CorInfoType jitTyp = info.compCompHnd->asCorInfoType(resolvedToken.hClass); - if (impIsPrimitive(jitTyp)) + if (!isLoadAddress) { - lclTyp = JITtype2varType(jitTyp); - goto STIND; - } - op2 = impPopStack().val; // Value - op1 = impPopStack().val; // Ptr + if (prefixFlags & PREFIX_VOLATILE) + { + op1->gtFlags |= GTF_ORDER_SIDEEFF; // Prevent this from being reordered - assertImp(varTypeIsStruct(op2)); + if (!usesHelper) + { + assert(op1->OperIs(GT_FIELD, GT_IND, GT_OBJ)); + op1->gtFlags |= GTF_IND_VOLATILE; + } + } - op1 = impAssignStructPtr(op1, op2, resolvedToken.hClass, (unsigned)CHECK_SPILL_ALL); + if ((prefixFlags & PREFIX_UNALIGNED) && !varTypeIsByte(lclTyp)) + { + if (!usesHelper) + { + assert(op1->OperIs(GT_FIELD, GT_IND, GT_OBJ)); + op1->gtFlags |= GTF_IND_UNALIGNED; + } + } + } - if (op1->OperIsBlkOp() && (prefixFlags & PREFIX_UNALIGNED)) + // Check if the class needs explicit initialization. + if (fieldInfo.fieldFlags & CORINFO_FLG_FIELD_INITCLASS) { - op1->gtFlags |= GTF_BLK_UNALIGNED; + GenTree* helperNode = impInitClass(&resolvedToken); + if (compDonotInline()) + { + return; + } + if (helperNode != nullptr) + { + op1 = gtNewOperNode(GT_COMMA, op1->TypeGet(), helperNode, op1); + } } - goto SPILL_APPEND; + + FIELD_DONE: + impPushOnStack(op1, tiRetVal); } + break; - case CEE_MKREFANY: + case CEE_STFLD: + case CEE_STSFLD: + { - assert(!compIsForInlining()); + bool isStoreStatic = (opcode == CEE_STSFLD); - // Being lazy here. Refanys are tricky in terms of gc tracking. - // Since it is uncommon, just don't perform struct promotion in any method that contains mkrefany. + CORINFO_CLASS_HANDLE fieldClsHnd; // class of the field (if it's a ref type) - JITDUMP("disabling struct promotion because of mkrefany\n"); - fgNoStructPromotion = true; + /* Get the CP_Fieldref index */ - oper = GT_MKREFANY; assertImp(sz == sizeof(unsigned)); - _impResolveToken(CORINFO_TOKENKIND_Class); + _impResolveToken(CORINFO_TOKENKIND_Field); JITDUMP(" %08X", resolvedToken.token); - op2 = impTokenToHandle(&resolvedToken, nullptr, true); - if (op2 == nullptr) - { // compDonotInline() - return; - } + int aflags = CORINFO_ACCESS_SET; + GenTree* obj = nullptr; - accessAllowedResult = - info.compCompHnd->canAccessClass(&resolvedToken, info.compMethodHnd, &calloutHelper); - impHandleAccessAllowed(accessAllowedResult, &calloutHelper); + // Pull the value from the stack. + StackEntry se = impPopStack(); + op2 = se.val; + clsHnd = se.seTypeInfo.GetClassHandle(); - op1 = impPopStack().val; + if (opcode == CEE_STFLD) + { + obj = impPopStack().val; - // @SPECVIOLATION: TYP_INT should not be allowed here by a strict reading of the spec. - // But JIT32 allowed it, so we continue to allow it. - assertImp(op1->TypeGet() == TYP_BYREF || op1->TypeGet() == TYP_I_IMPL || op1->TypeGet() == TYP_INT); + if (impIsThis(obj)) + { + aflags |= CORINFO_ACCESS_THIS; + } + } - // MKREFANY returns a struct. op2 is the class token. - op1 = gtNewOperNode(oper, TYP_STRUCT, op1, op2); + eeGetFieldInfo(&resolvedToken, (CORINFO_ACCESS_FLAGS)aflags, &fieldInfo); - impPushOnStack(op1, verMakeTypeInfo(impGetRefAnyClass())); - break; + // Figure out the type of the member. We always call canAccessField, so you always need this + // handle + CorInfoType ciType = fieldInfo.fieldType; + fieldClsHnd = fieldInfo.structType; - case CEE_LDOBJ: - { - oper = GT_OBJ; - assertImp(sz == sizeof(unsigned)); + lclTyp = JITtype2varType(ciType); - _impResolveToken(CORINFO_TOKENKIND_Class); + if (compIsForInlining()) + { + /* Is this a 'special' (COM) field? or a TLS ref static field?, field stored int GC heap? or + * per-inst static? */ - JITDUMP(" %08X", resolvedToken.token); + switch (fieldInfo.fieldAccessor) + { + case CORINFO_FIELD_INSTANCE_HELPER: + case CORINFO_FIELD_INSTANCE_ADDR_HELPER: + case CORINFO_FIELD_STATIC_ADDR_HELPER: + case CORINFO_FIELD_STATIC_TLS: - OBJ: + compInlineResult->NoteFatal(InlineObservation::CALLEE_STFLD_NEEDS_HELPER); + return; - tiRetVal = verMakeTypeInfo(resolvedToken.hClass); + case CORINFO_FIELD_STATIC_GENERICS_STATIC_HELPER: + case CORINFO_FIELD_STATIC_READYTORUN_HELPER: + /* We may be able to inline the field accessors in specific instantiations of generic + * methods */ + compInlineResult->NoteFatal(InlineObservation::CALLSITE_STFLD_NEEDS_HELPER); + return; - if (eeIsValueClass(resolvedToken.hClass)) - { - lclTyp = TYP_STRUCT; - } - else - { - lclTyp = TYP_REF; - opcode = CEE_LDIND_REF; - goto LDIND; + default: + break; + } } - op1 = impPopStack().val; - - assertImp(op1->TypeGet() == TYP_BYREF || op1->TypeGet() == TYP_I_IMPL); + impHandleAccessAllowed(fieldInfo.accessAllowed, &fieldInfo.accessCalloutHelper); - CorInfoType jitTyp = info.compCompHnd->asCorInfoType(resolvedToken.hClass); - if (impIsPrimitive(jitTyp)) + // Raise InvalidProgramException if static store accesses non-static field + if (isStoreStatic && ((fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC) == 0)) { - op1 = gtNewOperNode(GT_IND, JITtype2varType(jitTyp), op1); - - // Could point anywhere, example a boxed class static int - op1->gtFlags |= GTF_GLOB_REF; - assertImp(varTypeIsArithmetic(op1->gtType)); + BADCODE("static access on an instance field"); } - else + + // We are using stfld on a static field. + // We allow it, but need to eval any side-effects for obj + if ((fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC) && obj != nullptr) { - // OBJ returns a struct - // and an inline argument which is the class token of the loaded obj - op1 = gtNewObjNode(resolvedToken.hClass, op1); + if (obj->gtFlags & GTF_SIDE_EFFECT) + { + obj = gtUnusedValNode(obj); + impAppendTree(obj, CHECK_SPILL_ALL, impCurStmtDI); + } + obj = nullptr; } - op1->gtFlags |= GTF_EXCEPT; - if (prefixFlags & PREFIX_UNALIGNED) + /* Preserve 'small' int types */ + if (!varTypeIsSmall(lclTyp)) { - op1->gtFlags |= GTF_IND_UNALIGNED; + lclTyp = genActualType(lclTyp); } - impPushOnStack(op1, tiRetVal); - break; - } - - case CEE_LDLEN: - op1 = impPopStack().val; - if (opts.OptimizationEnabled()) + switch (fieldInfo.fieldAccessor) { - /* Use GT_ARR_LENGTH operator so rng check opts see this */ - GenTreeArrLen* arrLen = gtNewArrLen(TYP_INT, op1, OFFSETOF__CORINFO_Array__length, block); + case CORINFO_FIELD_INSTANCE: +#ifdef FEATURE_READYTORUN + case CORINFO_FIELD_INSTANCE_WITH_BASE: +#endif + { + /* Create the data member node */ + op1 = gtNewFieldRef(lclTyp, resolvedToken.hField, obj, fieldInfo.offset); + DWORD typeFlags = info.compCompHnd->getClassAttribs(resolvedToken.hClass); + if (StructHasOverlappingFields(typeFlags)) + { + op1->AsField()->gtFldMayOverlap = true; + } - op1 = arrLen; +#ifdef FEATURE_READYTORUN + if (fieldInfo.fieldAccessor == CORINFO_FIELD_INSTANCE_WITH_BASE) + { + op1->AsField()->gtFieldLookup = fieldInfo.fieldLookup; + } +#endif + + if (fgAddrCouldBeNull(obj)) + { + op1->gtFlags |= GTF_EXCEPT; + } + + if (compIsForInlining() && + impInlineIsGuaranteedThisDerefBeforeAnySideEffects(op2, nullptr, obj, + impInlineInfo->inlArgInfo)) + { + impInlineInfo->thisDereferencedFirst = true; + } + } + break; + + case CORINFO_FIELD_STATIC_TLS: +#ifdef TARGET_X86 + // Legacy TLS access is implemented as intrinsic on x86 only. + op1 = gtNewFieldAddrNode(TYP_I_IMPL, resolvedToken.hField, nullptr, fieldInfo.offset); + op1->gtFlags |= GTF_FLD_TLS; // fgMorphExpandTlsField will handle the transformation. + + if (varTypeIsStruct(lclTyp)) + { + op1 = gtNewObjNode(fieldInfo.structType, op1); + op1->gtFlags |= GTF_IND_NONFAULTING; + } + else + { + op1 = gtNewIndir(lclTyp, op1, GTF_IND_NONFAULTING); + op1->gtFlags |= GTF_GLOB_REF; + } + break; +#else + fieldInfo.fieldAccessor = CORINFO_FIELD_STATIC_ADDR_HELPER; + FALLTHROUGH; +#endif + case CORINFO_FIELD_STATIC_ADDR_HELPER: + case CORINFO_FIELD_INSTANCE_HELPER: + case CORINFO_FIELD_INSTANCE_ADDR_HELPER: + op1 = gtNewRefCOMfield(obj, &resolvedToken, (CORINFO_ACCESS_FLAGS)aflags, &fieldInfo, lclTyp, + clsHnd, op2); + goto SPILL_APPEND; + + case CORINFO_FIELD_STATIC_ADDRESS: + case CORINFO_FIELD_STATIC_RVA_ADDRESS: + case CORINFO_FIELD_STATIC_SHARED_STATIC_HELPER: + case CORINFO_FIELD_STATIC_GENERICS_STATIC_HELPER: + case CORINFO_FIELD_STATIC_READYTORUN_HELPER: + op1 = impImportStaticFieldAccess(&resolvedToken, (CORINFO_ACCESS_FLAGS)aflags, &fieldInfo, + lclTyp); + break; + + default: + assert(!"Unexpected fieldAccessor"); } - else + + if (lclTyp != TYP_STRUCT) { - /* Create the expression "*(array_addr + ArrLenOffs)" */ - op1 = gtNewOperNode(GT_ADD, TYP_BYREF, op1, - gtNewIconNode(OFFSETOF__CORINFO_Array__length, TYP_I_IMPL)); - op1 = gtNewIndir(TYP_INT, op1); - } + assert(op1->OperIs(GT_FIELD, GT_IND)); - /* Push the result back on the stack */ - impPushOnStack(op1, tiRetVal); - break; + if (prefixFlags & PREFIX_VOLATILE) + { + op1->gtFlags |= GTF_ORDER_SIDEEFF; // Prevent this from being reordered + op1->gtFlags |= GTF_IND_VOLATILE; + } + if ((prefixFlags & PREFIX_UNALIGNED) && !varTypeIsByte(lclTyp)) + { + op1->gtFlags |= GTF_IND_UNALIGNED; + } - case CEE_BREAK: - op1 = gtNewHelperCallNode(CORINFO_HELP_USER_BREAKPOINT, TYP_VOID); - goto SPILL_APPEND; + // Currently, *all* TYP_REF statics are stored inside an "object[]" array that itself + // resides on the managed heap, and so we can use an unchecked write barrier for this + // store. Likewise if we're storing to a field of an on-heap object. + if ((lclTyp == TYP_REF) && + (((fieldInfo.fieldFlags & CORINFO_FLG_FIELD_STATIC) != 0) || obj->TypeIs(TYP_REF))) + { + static_assert_no_msg(GTF_FLD_TGT_HEAP == GTF_IND_TGT_HEAP); + op1->gtFlags |= GTF_FLD_TGT_HEAP; + } - case CEE_NOP: - if (opts.compDbgCode) - { - op1 = new (this, GT_NO_OP) GenTree(GT_NO_OP, TYP_VOID); - goto SPILL_APPEND; - } - break; + /* V4.0 allows assignment of i4 constant values to i8 type vars when IL verifier is bypassed (full + trust apps). The reason this works is that JIT stores an i4 constant in Gentree union during + importation and reads from the union as if it were a long during code generation. Though this + can potentially read garbage, one can get lucky to have this working correctly. - /******************************** NYI *******************************/ + This code pattern is generated by Dev10 MC++ compiler while storing to fields when compiled with + /O2 switch (default when compiling retail configs in Dev10) and a customer app has taken a + dependency on it. To be backward compatible, we will explicitly add an upward cast here so that + it works correctly always. + + Note that this is limited to x86 alone as there is no back compat to be addressed for Arm JIT + for V4.0. + */ + CLANG_FORMAT_COMMENT_ANCHOR; + +#ifndef TARGET_64BIT + // In UWP6.0 and beyond (post-.NET Core 2.0), we decided to let this cast from int to long be + // generated for ARM as well as x86, so the following IR will be accepted: + // STMTx (IL 0x... ???) + // * ASG long + // +--* LCL_VAR long + // \--* CNS_INT int 2 + + if ((op1->TypeGet() != op2->TypeGet()) && op2->OperIsConst() && varTypeIsIntOrI(op2->TypeGet()) && + varTypeIsLong(op1->TypeGet())) + { + op2 = gtNewCastNode(op1->TypeGet(), op2, false, op1->TypeGet()); + } +#endif + +#ifdef TARGET_64BIT + // Automatic upcast for a GT_CNS_INT into TYP_I_IMPL + if ((op2->OperGet() == GT_CNS_INT) && varTypeIsI(lclTyp) && !varTypeIsI(op2->gtType)) + { + op2->gtType = TYP_I_IMPL; + } + else + { + // Allow a downcast of op2 from TYP_I_IMPL into a 32-bit Int for x86 JIT compatibility + // + if (varTypeIsI(op2->gtType) && (genActualType(lclTyp) == TYP_INT)) + { + op2 = gtNewCastNode(TYP_INT, op2, false, TYP_INT); + } + // Allow an upcast of op2 from a 32-bit Int into TYP_I_IMPL for x86 JIT compatibility + // + if (varTypeIsI(lclTyp) && (genActualType(op2->gtType) == TYP_INT)) + { + op2 = gtNewCastNode(TYP_I_IMPL, op2, false, TYP_I_IMPL); + } + } +#endif - case 0xCC: - OutputDebugStringA("CLR: Invalid x86 breakpoint in IL stream\n"); - FALLTHROUGH; + // Insert an implicit FLOAT<->DOUBLE cast if needed. + op2 = impImplicitR4orR8Cast(op2, op1->TypeGet()); - case CEE_ILLEGAL: - case CEE_MACRO_END: + op1 = gtNewAssignNode(op1, op2); + } - default: - if (compIsForInlining()) + // Check if the class needs explicit initialization. + if (fieldInfo.fieldFlags & CORINFO_FLG_FIELD_INITCLASS) { - compInlineResult->NoteFatal(InlineObservation::CALLEE_COMPILATION_ERROR); - return; + GenTree* helperNode = impInitClass(&resolvedToken); + if (compDonotInline()) + { + return; + } + if (helperNode != nullptr) + { + impAppendTree(helperNode, CHECK_SPILL_ALL, impCurStmtDI); + } } - BADCODE3("unknown opcode", ": %02X", (int)opcode); - } - - codeAddr += sz; - prevOpcode = opcode; - - prefixFlags = 0; - } - - return; -#undef _impResolveToken -} -#ifdef _PREFAST_ -#pragma warning(pop) -#endif - -// Push a local/argument treeon the operand stack -void Compiler::impPushVar(GenTree* op, typeInfo tiRetVal) -{ - tiRetVal.NormaliseForStack(); - - if (verTrackObjCtorInitState && (verCurrentState.thisInitialized != TIS_Init) && tiRetVal.IsThisPtr()) - { - tiRetVal.SetUninitialisedObjRef(); - } - - impPushOnStack(op, tiRetVal); -} + if (lclTyp == TYP_STRUCT) + { + op1 = impAssignStruct(op1, op2, CHECK_SPILL_ALL); + } + goto SPILL_APPEND; + } -//------------------------------------------------------------------------ -// impCreateLocal: create a GT_LCL_VAR node to access a local that might need to be normalized on load -// -// Arguments: -// lclNum -- The index into lvaTable -// offset -- The offset to associate with the node -// -// Returns: -// The node -// -GenTreeLclVar* Compiler::impCreateLocalNode(unsigned lclNum DEBUGARG(IL_OFFSET offset)) -{ - var_types lclTyp; + case CEE_NEWARR: + { - if (lvaTable[lclNum].lvNormalizeOnLoad()) - { - lclTyp = lvaGetRealType(lclNum); - } - else - { - lclTyp = lvaGetActualType(lclNum); - } + /* Get the class type index operand */ - return gtNewLclvNode(lclNum, lclTyp DEBUGARG(offset)); -} + _impResolveToken(CORINFO_TOKENKIND_Newarr); -// Load a local/argument on the operand stack -// lclNum is an index into lvaTable *NOT* the arg/lcl index in the IL -void Compiler::impLoadVar(unsigned lclNum, IL_OFFSET offset, const typeInfo& tiRetVal) -{ - impPushVar(impCreateLocalNode(lclNum DEBUGARG(offset)), tiRetVal); -} + JITDUMP(" %08X", resolvedToken.token); -// Load an argument on the operand stack -// Shared by the various CEE_LDARG opcodes -// ilArgNum is the argument index as specified in IL. -// It will be mapped to the correct lvaTable index -void Compiler::impLoadArg(unsigned ilArgNum, IL_OFFSET offset) -{ - Verify(ilArgNum < info.compILargsCount, "bad arg num"); + if (!opts.IsReadyToRun()) + { + // Need to restore array classes before creating array objects on the heap + op1 = impTokenToHandle(&resolvedToken, nullptr, true /*mustRestoreHandle*/); + if (op1 == nullptr) + { // compDonotInline() + return; + } + } - if (compIsForInlining()) - { - if (ilArgNum >= info.compArgsCount) - { - compInlineResult->NoteFatal(InlineObservation::CALLEE_BAD_ARGUMENT_NUMBER); - return; - } + tiRetVal = verMakeTypeInfo(resolvedToken.hClass); - impPushVar(impInlineFetchArg(ilArgNum, impInlineInfo->inlArgInfo, impInlineInfo->lclVarInfo), - impInlineInfo->lclVarInfo[ilArgNum].lclVerTypeInfo); - } - else - { - if (ilArgNum >= info.compArgsCount) - { - BADCODE("Bad IL"); - } + accessAllowedResult = + info.compCompHnd->canAccessClass(&resolvedToken, info.compMethodHnd, &calloutHelper); + impHandleAccessAllowed(accessAllowedResult, &calloutHelper); - unsigned lclNum = compMapILargNum(ilArgNum); // account for possible hidden param + /* Form the arglist: array class handle, size */ + op2 = impPopStack().val; + assertImp(genActualTypeIsIntOrI(op2->gtType)); - if (lclNum == info.compThisArg) - { - lclNum = lvaArg0Var; - } +#ifdef TARGET_64BIT + // The array helper takes a native int for array length. + // So if we have an int, explicitly extend it to be a native int. + if (genActualType(op2->TypeGet()) != TYP_I_IMPL) + { + if (op2->IsIntegralConst()) + { + op2->gtType = TYP_I_IMPL; + } + else + { + bool isUnsigned = false; + op2 = gtNewCastNode(TYP_I_IMPL, op2, isUnsigned, TYP_I_IMPL); + } + } +#endif // TARGET_64BIT - impLoadVar(lclNum, offset); - } -} +#ifdef FEATURE_READYTORUN + if (opts.IsReadyToRun()) + { + op1 = impReadyToRunHelperToTree(&resolvedToken, CORINFO_HELP_READYTORUN_NEWARR_1, TYP_REF, nullptr, + op2); + usingReadyToRunHelper = (op1 != nullptr); -// Load a local on the operand stack -// Shared by the various CEE_LDLOC opcodes -// ilLclNum is the local index as specified in IL. -// It will be mapped to the correct lvaTable index -void Compiler::impLoadLoc(unsigned ilLclNum, IL_OFFSET offset) -{ - if (compIsForInlining()) - { - if (ilLclNum >= info.compMethodInfo->locals.numArgs) - { - compInlineResult->NoteFatal(InlineObservation::CALLEE_BAD_LOCAL_NUMBER); - return; - } + if (!usingReadyToRunHelper) + { + // TODO: ReadyToRun: When generic dictionary lookups are necessary, replace the lookup call + // and the newarr call with a single call to a dynamic R2R cell that will: + // 1) Load the context + // 2) Perform the generic dictionary lookup and caching, and generate the appropriate stub + // 3) Allocate the new array + // Reason: performance (today, we'll always use the slow helper for the R2R generics case) - // Get the local type - var_types lclTyp = impInlineInfo->lclVarInfo[ilLclNum + impInlineInfo->argCnt].lclTypeInfo; + // Need to restore array classes before creating array objects on the heap + op1 = impTokenToHandle(&resolvedToken, nullptr, true /*mustRestoreHandle*/); + if (op1 == nullptr) + { // compDonotInline() + return; + } + } + } - typeInfo tiRetVal = impInlineInfo->lclVarInfo[ilLclNum + impInlineInfo->argCnt].lclVerTypeInfo; + if (!usingReadyToRunHelper) +#endif + { + /* Create a call to 'new' */ - /* Have we allocated a temp for this local? */ + // Note that this only works for shared generic code because the same helper is used for all + // reference array types + op1 = + gtNewHelperCallNode(info.compCompHnd->getNewArrHelper(resolvedToken.hClass), TYP_REF, op1, op2); + } - unsigned lclNum = impInlineFetchLocal(ilLclNum DEBUGARG("Inline ldloc first use temp")); + op1->AsCall()->compileTimeHelperArgumentHandle = (CORINFO_GENERIC_HANDLE)resolvedToken.hClass; - // All vars of inlined methods should be !lvNormalizeOnLoad() + // Remember that this function contains 'new' of an SD array. + optMethodFlags |= OMF_HAS_NEWARRAY; - assert(!lvaTable[lclNum].lvNormalizeOnLoad()); - lclTyp = genActualType(lclTyp); + /* Push the result of the call on the stack */ - impPushVar(gtNewLclvNode(lclNum, lclTyp), tiRetVal); - } - else - { - if (ilLclNum >= info.compMethodInfo->locals.numArgs) - { - BADCODE("Bad IL"); - } + impPushOnStack(op1, tiRetVal); - unsigned lclNum = info.compArgsCount + ilLclNum; + callTyp = TYP_REF; + } + break; - impLoadVar(lclNum, offset); - } -} + case CEE_LOCALLOC: + // We don't allow locallocs inside handlers + if (block->hasHndIndex()) + { + BADCODE("Localloc can't be inside handler"); + } -#ifdef TARGET_ARM -/************************************************************************************** - * - * When assigning a vararg call src to a HFA lcl dest, mark that we cannot promote the - * dst struct, because struct promotion will turn it into a float/double variable while - * the rhs will be an int/long variable. We don't code generate assignment of int into - * a float, but there is nothing that might prevent us from doing so. The tree however - * would like: (=, (typ_float, typ_int)) or (GT_TRANSFER, (typ_float, typ_int)) - * - * tmpNum - the lcl dst variable num that is a struct. - * src - the src tree assigned to the dest that is a struct/int (when varargs call.) - * hClass - the type handle for the struct variable. - * - * TODO-ARM-CQ: [301608] This is a rare scenario with varargs and struct promotion coming into play, - * however, we could do a codegen of transferring from int to float registers - * (transfer, not a cast.) - * - */ -void Compiler::impMarkLclDstNotPromotable(unsigned tmpNum, GenTree* src, CORINFO_CLASS_HANDLE hClass) -{ - if (src->gtOper == GT_CALL && src->AsCall()->IsVarargs() && IsHfa(hClass)) - { - int hfaSlots = GetHfaCount(hClass); - var_types hfaType = GetHfaType(hClass); - - // If we have varargs we morph the method's return type to be "int" irrespective of its original - // type: struct/float at importer because the ABI calls out return in integer registers. - // We don't want struct promotion to replace an expression like this: - // lclFld_int = callvar_int() into lclFld_float = callvar_int(); - // This means an int is getting assigned to a float without a cast. Prevent the promotion. - if ((hfaType == TYP_DOUBLE && hfaSlots == sizeof(double) / REGSIZE_BYTES) || - (hfaType == TYP_FLOAT && hfaSlots == sizeof(float) / REGSIZE_BYTES)) - { - // Make sure this struct type stays as struct so we can receive the call in a struct. - lvaTable[tmpNum].lvIsMultiRegRet = true; - } - } -} -#endif // TARGET_ARM + // Get the size to allocate -#if FEATURE_MULTIREG_RET -//------------------------------------------------------------------------ -// impAssignMultiRegTypeToVar: ensure calls that return structs in multiple -// registers return values to suitable temps. -// -// Arguments: -// op -- call returning a struct in registers -// hClass -- class handle for struct -// -// Returns: -// Tree with reference to struct local to use as call return value. + op2 = impPopStack().val; + assertImp(genActualTypeIsIntOrI(op2->gtType)); -GenTree* Compiler::impAssignMultiRegTypeToVar(GenTree* op, - CORINFO_CLASS_HANDLE hClass DEBUGARG(CorInfoCallConvExtension callConv)) -{ - unsigned tmpNum = lvaGrabTemp(true DEBUGARG("Return value temp for multireg return")); - impAssignTempGen(tmpNum, op, hClass, (unsigned)CHECK_SPILL_ALL); - GenTree* ret = gtNewLclvNode(tmpNum, lvaTable[tmpNum].lvType); + if (verCurrentState.esStackDepth != 0) + { + BADCODE("Localloc can only be used when the stack is empty"); + } - // TODO-1stClassStructs: Handle constant propagation and CSE-ing of multireg returns. - ret->gtFlags |= GTF_DONT_CSE; + // If the localloc is not in a loop and its size is a small constant, + // create a new local var of TYP_BLK and return its address. + { + bool convertedToLocal = false; - assert(IsMultiRegReturnedType(hClass, callConv)); + // Need to aggressively fold here, as even fixed-size locallocs + // will have casts in the way. + op2 = gtFoldExpr(op2); - // Mark the var so that fields are not promoted and stay together. - lvaTable[tmpNum].lvIsMultiRegRet = true; + if (op2->IsIntegralConst()) + { + const ssize_t allocSize = op2->AsIntCon()->IconValue(); - return ret; -} -#endif // FEATURE_MULTIREG_RET + bool bbInALoop = impBlockIsInALoop(block); -//------------------------------------------------------------------------ -// impReturnInstruction: import a return or an explicit tail call -// -// Arguments: -// prefixFlags -- active IL prefixes -// opcode -- [in, out] IL opcode -// -// Returns: -// True if import was successful (may fail for some inlinees) -// -bool Compiler::impReturnInstruction(int prefixFlags, OPCODE& opcode) -{ - const bool isTailCall = (prefixFlags & PREFIX_TAILCALL) != 0; + if (allocSize == 0) + { + // Result is nullptr + JITDUMP("Converting stackalloc of 0 bytes to push null unmanaged pointer\n"); + op1 = gtNewIconNode(0, TYP_I_IMPL); + convertedToLocal = true; + } + else if ((allocSize > 0) && !bbInALoop) + { + // Get the size threshold for local conversion + ssize_t maxSize = DEFAULT_MAX_LOCALLOC_TO_LOCAL_SIZE; #ifdef DEBUG - // If we are importing an inlinee and have GC ref locals we always - // need to have a spill temp for the return value. This temp - // should have been set up in advance, over in fgFindBasicBlocks. - if (compIsForInlining() && impInlineInfo->HasGcRefLocals() && (info.compRetType != TYP_VOID)) - { - assert(lvaInlineeReturnSpillTemp != BAD_VAR_NUM); - } + // Optionally allow this to be modified + maxSize = JitConfig.JitStackAllocToLocalSize(); #endif // DEBUG - GenTree* op2 = nullptr; - GenTree* op1 = nullptr; - CORINFO_CLASS_HANDLE retClsHnd = nullptr; + if (allocSize <= maxSize) + { + const unsigned stackallocAsLocal = lvaGrabTemp(false DEBUGARG("stackallocLocal")); + JITDUMP("Converting stackalloc of %zd bytes to new local V%02u\n", allocSize, + stackallocAsLocal); + lvaTable[stackallocAsLocal].lvType = TYP_BLK; + lvaTable[stackallocAsLocal].lvExactSize = (unsigned)allocSize; + lvaTable[stackallocAsLocal].lvHasLdAddrOp = true; + lvaTable[stackallocAsLocal].lvIsUnsafeBuffer = true; + op1 = gtNewLclvNode(stackallocAsLocal, TYP_BLK); + op1 = gtNewOperNode(GT_ADDR, TYP_I_IMPL, op1); + convertedToLocal = true; - if (info.compRetType != TYP_VOID) - { - StackEntry se = impPopStack(); - retClsHnd = se.seTypeInfo.GetClassHandle(); - op2 = se.val; + if (!this->opts.compDbgEnC) + { + // Ensure we have stack security for this method. + // Reorder layout since the converted localloc is treated as an unsafe buffer. + setNeedsGSSecurityCookie(); + compGSReorderStackLayout = true; + } + } + } + } - if (!compIsForInlining()) - { - impBashVarAddrsToI(op2); - op2 = impImplicitIorI4Cast(op2, info.compRetType); - op2 = impImplicitR4orR8Cast(op2, info.compRetType); - // Note that we allow TYP_I_IMPL<->TYP_BYREF transformation, but only TYP_I_IMPL<-TYP_REF. - assertImp((genActualType(op2->TypeGet()) == genActualType(info.compRetType)) || - ((op2->TypeGet() == TYP_I_IMPL) && TypeIs(info.compRetType, TYP_BYREF)) || - (op2->TypeIs(TYP_BYREF, TYP_REF) && (info.compRetType == TYP_I_IMPL)) || - (varTypeIsFloating(op2->gtType) && varTypeIsFloating(info.compRetType)) || - (varTypeIsStruct(op2) && varTypeIsStruct(info.compRetType))); + if (!convertedToLocal) + { + // Bail out if inlining and the localloc was not converted. + // + // Note we might consider allowing the inline, if the call + // site is not in a loop. + if (compIsForInlining()) + { + InlineObservation obs = op2->IsIntegralConst() + ? InlineObservation::CALLEE_LOCALLOC_TOO_LARGE + : InlineObservation::CALLSITE_LOCALLOC_SIZE_UNKNOWN; + compInlineResult->NoteFatal(obs); + return; + } -#ifdef DEBUG - if (!isTailCall && opts.compGcChecks && (info.compRetType == TYP_REF)) + op1 = gtNewOperNode(GT_LCLHEAP, TYP_I_IMPL, op2); + // May throw a stack overflow exception. Obviously, we don't want locallocs to be CSE'd. + op1->gtFlags |= (GTF_EXCEPT | GTF_DONT_CSE); + + // Ensure we have stack security for this method. + setNeedsGSSecurityCookie(); + + /* The FP register may not be back to the original value at the end + of the method, even if the frame size is 0, as localloc may + have modified it. So we will HAVE to reset it */ + compLocallocUsed = true; + } + else + { + compLocallocOptimized = true; + } + } + + impPushOnStack(op1, tiRetVal); + break; + + case CEE_ISINST: { - // DDB 3483 : JIT Stress: early termination of GC ref's life time in exception code path - // VSW 440513: Incorrect gcinfo on the return value under COMPlus_JitGCChecks=1 for methods with - // one-return BB. + /* Get the type token */ + assertImp(sz == sizeof(unsigned)); - assert(op2->gtType == TYP_REF); + _impResolveToken(CORINFO_TOKENKIND_Casting); - // confirm that the argument is a GC pointer (for debugging (GC stress)) - op2 = gtNewHelperCallNode(CORINFO_HELP_CHECK_OBJ, TYP_REF, op2); + JITDUMP(" %08X", resolvedToken.token); - if (verbose) + if (!opts.IsReadyToRun()) { - printf("\ncompGcChecks tree:\n"); - gtDispTree(op2); + op2 = impTokenToHandle(&resolvedToken, nullptr, false); + if (op2 == nullptr) + { // compDonotInline() + return; + } } - } -#endif - } - else - { - if (verCurrentState.esStackDepth != 0) - { - assert(compIsForInlining()); - JITDUMP("CALLSITE_COMPILATION_ERROR: inlinee's stack is not empty."); - compInlineResult->NoteFatal(InlineObservation::CALLSITE_COMPILATION_ERROR); - return false; - } -#ifdef DEBUG - if (verbose) - { - printf("\n\n Inlinee Return expression (before normalization) =>\n"); - gtDispTree(op2); - } -#endif + accessAllowedResult = + info.compCompHnd->canAccessClass(&resolvedToken, info.compMethodHnd, &calloutHelper); + impHandleAccessAllowed(accessAllowedResult, &calloutHelper); - // Make sure the type matches the original call. + op1 = impPopStack().val; - var_types returnType = genActualType(op2->gtType); - var_types originalCallType = impInlineInfo->inlineCandidateInfo->fncRetType; - if ((returnType != originalCallType) && (originalCallType == TYP_STRUCT)) - { - originalCallType = impNormStructType(impInlineInfo->inlineCandidateInfo->methInfo.args.retTypeClass); - } + GenTree* optTree = impOptimizeCastClassOrIsInst(op1, &resolvedToken, false); - if (returnType != originalCallType) - { - // Allow TYP_BYREF to be returned as TYP_I_IMPL and vice versa. - // Allow TYP_REF to be returned as TYP_I_IMPL and NOT vice verse. - if ((TypeIs(returnType, TYP_BYREF, TYP_REF) && (originalCallType == TYP_I_IMPL)) || - ((returnType == TYP_I_IMPL) && TypeIs(originalCallType, TYP_BYREF))) + if (optTree != nullptr) { - JITDUMP("Allowing return type mismatch: have %s, needed %s\n", varTypeName(returnType), - varTypeName(originalCallType)); + impPushOnStack(optTree, tiRetVal); } else { - JITDUMP("Return type mismatch: have %s, needed %s\n", varTypeName(returnType), - varTypeName(originalCallType)); - compInlineResult->NoteFatal(InlineObservation::CALLSITE_RETURN_TYPE_MISMATCH); - return false; + +#ifdef FEATURE_READYTORUN + if (opts.IsReadyToRun()) + { + GenTreeCall* opLookup = + impReadyToRunHelperToTree(&resolvedToken, CORINFO_HELP_READYTORUN_ISINSTANCEOF, TYP_REF, + nullptr, op1); + usingReadyToRunHelper = (opLookup != nullptr); + op1 = (usingReadyToRunHelper ? opLookup : op1); + + if (!usingReadyToRunHelper) + { + // TODO: ReadyToRun: When generic dictionary lookups are necessary, replace the lookup call + // and the isinstanceof_any call with a single call to a dynamic R2R cell that will: + // 1) Load the context + // 2) Perform the generic dictionary lookup and caching, and generate the appropriate + // stub + // 3) Perform the 'is instance' check on the input object + // Reason: performance (today, we'll always use the slow helper for the R2R generics case) + + op2 = impTokenToHandle(&resolvedToken, nullptr, false); + if (op2 == nullptr) + { // compDonotInline() + return; + } + } + } + + if (!usingReadyToRunHelper) +#endif + { + op1 = impCastClassOrIsInstToTree(op1, op2, &resolvedToken, false, opcodeOffs); + } + if (compDonotInline()) + { + return; + } + + impPushOnStack(op1, tiRetVal); + } + break; + } + + case CEE_REFANYVAL: + { + + // get the class handle and make a ICON node out of it + + _impResolveToken(CORINFO_TOKENKIND_Class); + + JITDUMP(" %08X", resolvedToken.token); + + op2 = impTokenToHandle(&resolvedToken); + if (op2 == nullptr) + { // compDonotInline() + return; } + + op1 = impPopStack().val; + // make certain it is normalized; + op1 = impNormStructVal(op1, impGetRefAnyClass(), CHECK_SPILL_ALL); + + // Call helper GETREFANY(classHandle, op1); + GenTreeCall* helperCall = gtNewHelperCallNode(CORINFO_HELP_GETREFANY, TYP_BYREF); + NewCallArg clsHandleArg = NewCallArg::Primitive(op2); + NewCallArg typedRefArg = NewCallArg::Struct(op1, TYP_STRUCT, impGetRefAnyClass()); + helperCall->gtArgs.PushFront(this, clsHandleArg, typedRefArg); + helperCall->gtFlags |= (op1->gtFlags | op2->gtFlags) & GTF_ALL_EFFECT; + op1 = helperCall; + + impPushOnStack(op1, tiRetVal); + break; } - - // Below, we are going to set impInlineInfo->retExpr to the tree with the return - // expression. At this point, retExpr could already be set if there are multiple - // return blocks (meaning fgNeedReturnSpillTemp() == true) and one of - // the other blocks already set it. If there is only a single return block, - // retExpr shouldn't be set. However, this is not true if we reimport a block - // with a return. In that case, retExpr will be set, then the block will be - // reimported, but retExpr won't get cleared as part of setting the block to - // be reimported. The reimported retExpr value should be the same, so even if - // we don't unconditionally overwrite it, it shouldn't matter. - if (info.compRetNativeType != TYP_STRUCT) + case CEE_REFANYTYPE: { - // compRetNativeType is not TYP_STRUCT. - // This implies it could be either a scalar type or SIMD vector type or - // a struct type that can be normalized to a scalar type. + op1 = impPopStack().val; - if (varTypeIsStruct(info.compRetType)) + if (op1->OperIs(GT_MKREFANY)) { - noway_assert(info.compRetBuffArg == BAD_VAR_NUM); - // adjust the type away from struct to integral - // and no normalizing - op2 = impFixupStructReturnType(op2, retClsHnd, info.compCallConv); + // The pointer may have side-effects + if (op1->AsOp()->gtOp1->gtFlags & GTF_SIDE_EFFECT) + { + impAppendTree(op1->AsOp()->gtOp1, CHECK_SPILL_ALL, impCurStmtDI); +#ifdef DEBUG + impNoteLastILoffs(); +#endif + } + + // We already have the class handle + op1 = op1->AsOp()->gtOp2; } else { - // Do we have to normalize? - var_types fncRealRetType = JITtype2varType(info.compMethodInfo->args.retType); - // For RET_EXPR get the type info from the call. Regardless - // of whether it ends up inlined or not normalization will - // happen as part of that function's codegen. - GenTree* returnedTree = op2->OperIs(GT_RET_EXPR) ? op2->AsRetExpr()->gtInlineCandidate : op2; - if ((varTypeIsSmall(returnedTree->TypeGet()) || varTypeIsSmall(fncRealRetType)) && - fgCastNeeded(returnedTree, fncRealRetType)) - { - // Small-typed return values are normalized by the callee - op2 = gtNewCastNode(TYP_INT, op2, false, fncRealRetType); - } + // Get the address of the refany + op1 = impGetStructAddr(op1, impGetRefAnyClass(), CHECK_SPILL_ALL, /* willDeref */ true); + + // Fetch the type from the correct slot + op1 = gtNewOperNode(GT_ADD, TYP_BYREF, op1, + gtNewIconNode(OFFSETOF__CORINFO_TypedReference__type, TYP_I_IMPL)); + op1 = gtNewOperNode(GT_IND, TYP_BYREF, op1); } - if (fgNeedReturnSpillTemp()) - { - assert(info.compRetNativeType != TYP_VOID && - (fgMoreThanOneReturnBlock() || impInlineInfo->HasGcRefLocals())); + // Convert native TypeHandle to RuntimeTypeHandle. + op1 = gtNewHelperCallNode(CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPEHANDLE_MAYBENULL, TYP_STRUCT, op1); - // If this method returns a ref type, track the actual types seen - // in the returns. - if (info.compRetType == TYP_REF) - { - bool isExact = false; - bool isNonNull = false; - CORINFO_CLASS_HANDLE returnClsHnd = gtGetClassHandle(op2, &isExact, &isNonNull); + CORINFO_CLASS_HANDLE classHandle = impGetTypeHandleClass(); - if (impInlineInfo->retExpr == nullptr) - { - // This is the first return, so best known type is the type - // of this return value. - impInlineInfo->retExprClassHnd = returnClsHnd; - impInlineInfo->retExprClassHndIsExact = isExact; - } - else if (impInlineInfo->retExprClassHnd != returnClsHnd) - { - // This return site type differs from earlier seen sites, - // so reset the info and we'll fall back to using the method's - // declared return type for the return spill temp. - impInlineInfo->retExprClassHnd = nullptr; - impInlineInfo->retExprClassHndIsExact = false; - } - } + // The handle struct is returned in register + op1->AsCall()->gtReturnType = GetRuntimeHandleUnderlyingType(); + op1->AsCall()->gtRetClsHnd = classHandle; +#if FEATURE_MULTIREG_RET + op1->AsCall()->InitializeStructReturnType(this, classHandle, op1->AsCall()->GetUnmanagedCallConv()); +#endif - impAssignTempGen(lvaInlineeReturnSpillTemp, op2, se.seTypeInfo.GetClassHandle(), - (unsigned)CHECK_SPILL_ALL); + tiRetVal = typeInfo(TI_STRUCT, classHandle); + impPushOnStack(op1, tiRetVal); + } + break; - var_types lclRetType = lvaGetDesc(lvaInlineeReturnSpillTemp)->lvType; - GenTree* tmpOp2 = gtNewLclvNode(lvaInlineeReturnSpillTemp, lclRetType); + case CEE_LDTOKEN: + { + /* Get the Class index */ + assertImp(sz == sizeof(unsigned)); + lastLoadToken = codeAddr; + _impResolveToken(CORINFO_TOKENKIND_Ldtoken); - op2 = tmpOp2; -#ifdef DEBUG - if (impInlineInfo->retExpr) - { - // Some other block(s) have seen the CEE_RET first. - // Better they spilled to the same temp. - assert(impInlineInfo->retExpr->gtOper == GT_LCL_VAR); - assert(impInlineInfo->retExpr->AsLclVarCommon()->GetLclNum() == - op2->AsLclVarCommon()->GetLclNum()); - } -#endif + tokenType = info.compCompHnd->getTokenTypeAsHandle(&resolvedToken); + + op1 = impTokenToHandle(&resolvedToken, nullptr, true); + if (op1 == nullptr) + { // compDonotInline() + return; } -#ifdef DEBUG - if (verbose) + helper = CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPEHANDLE; + assert(resolvedToken.hClass != nullptr); + + if (resolvedToken.hMethod != nullptr) { - printf("\n\n Inlinee Return expression (after normalization) =>\n"); - gtDispTree(op2); + helper = CORINFO_HELP_METHODDESC_TO_STUBRUNTIMEMETHOD; + } + else if (resolvedToken.hField != nullptr) + { + helper = CORINFO_HELP_FIELDDESC_TO_STUBRUNTIMEFIELD; } + + op1 = gtNewHelperCallNode(helper, TYP_STRUCT, op1); + + // The handle struct is returned in register and + // it could be consumed both as `TYP_STRUCT` and `TYP_REF`. + op1->AsCall()->gtReturnType = GetRuntimeHandleUnderlyingType(); +#if FEATURE_MULTIREG_RET + op1->AsCall()->InitializeStructReturnType(this, tokenType, op1->AsCall()->GetUnmanagedCallConv()); #endif + op1->AsCall()->gtRetClsHnd = tokenType; - // Report the return expression - impInlineInfo->retExpr = op2; + tiRetVal = verMakeTypeInfo(tokenType); + impPushOnStack(op1, tiRetVal); } - else + break; + + case CEE_UNBOX: + case CEE_UNBOX_ANY: { - // compRetNativeType is TYP_STRUCT. - // This implies that struct return via RetBuf arg or multi-reg struct return + /* Get the Class index */ + assertImp(sz == sizeof(unsigned)); - GenTreeCall* iciCall = impInlineInfo->iciCall->AsCall(); + _impResolveToken(CORINFO_TOKENKIND_Class); - // Assign the inlinee return into a spill temp. - // spill temp only exists if there are multiple return points - if (lvaInlineeReturnSpillTemp != BAD_VAR_NUM) - { - // in this case we have to insert multiple struct copies to the temp - // and the retexpr is just the temp. - assert(info.compRetNativeType != TYP_VOID); - assert(fgMoreThanOneReturnBlock() || impInlineInfo->HasGcRefLocals()); + JITDUMP(" %08X", resolvedToken.token); - impAssignTempGen(lvaInlineeReturnSpillTemp, op2, se.seTypeInfo.GetClassHandle(), - (unsigned)CHECK_SPILL_ALL); + bool runtimeLookup; + op2 = impTokenToHandle(&resolvedToken, &runtimeLookup); + if (op2 == nullptr) + { + assert(compDonotInline()); + return; } -#if defined(TARGET_ARM) || defined(UNIX_AMD64_ABI) -#if defined(TARGET_ARM) - // TODO-ARM64-NYI: HFA - // TODO-AMD64-Unix and TODO-ARM once the ARM64 functionality is implemented the - // next ifdefs could be refactored in a single method with the ifdef inside. - if (IsHfa(retClsHnd)) - { -// Same as !IsHfa but just don't bother with impAssignStructPtr. -#else // defined(UNIX_AMD64_ABI) - ReturnTypeDesc retTypeDesc; - retTypeDesc.InitializeStructReturnType(this, retClsHnd, info.compCallConv); - unsigned retRegCount = retTypeDesc.GetReturnRegCount(); - - if (retRegCount != 0) - { - // If single eightbyte, the return type would have been normalized and there won't be a temp var. - // This code will be called only if the struct return has not been normalized (i.e. 2 eightbytes - - // max allowed.) - assert(retRegCount == MAX_RET_REG_COUNT); - // Same as !structDesc.passedInRegisters but just don't bother with impAssignStructPtr. - CLANG_FORMAT_COMMENT_ANCHOR; -#endif // defined(UNIX_AMD64_ABI) + // Run this always so we can get access exceptions even with SkipVerification. + accessAllowedResult = + info.compCompHnd->canAccessClass(&resolvedToken, info.compMethodHnd, &calloutHelper); + impHandleAccessAllowed(accessAllowedResult, &calloutHelper); - if (fgNeedReturnSpillTemp()) - { - if (!impInlineInfo->retExpr) - { -#if defined(TARGET_ARM) - impInlineInfo->retExpr = gtNewLclvNode(lvaInlineeReturnSpillTemp, info.compRetType); -#else // defined(UNIX_AMD64_ABI) - // The inlinee compiler has figured out the type of the temp already. Use it here. - impInlineInfo->retExpr = - gtNewLclvNode(lvaInlineeReturnSpillTemp, lvaTable[lvaInlineeReturnSpillTemp].lvType); -#endif // defined(UNIX_AMD64_ABI) - } - } - else - { - impInlineInfo->retExpr = op2; - } + if (opcode == CEE_UNBOX_ANY && !eeIsValueClass(resolvedToken.hClass)) + { + JITDUMP("\n Importing UNBOX.ANY(refClass) as CASTCLASS\n"); + op1 = impPopStack().val; + goto CASTCLASS; } - else -#elif defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) - ReturnTypeDesc retTypeDesc; - retTypeDesc.InitializeStructReturnType(this, retClsHnd, info.compCallConv); - unsigned retRegCount = retTypeDesc.GetReturnRegCount(); - if (retRegCount != 0) + /* Pop the object and create the unbox helper call */ + /* You might think that for UNBOX_ANY we need to push a different */ + /* (non-byref) type, but here we're making the tiRetVal that is used */ + /* for the intermediate pointer which we then transfer onto the OBJ */ + /* instruction. OBJ then creates the appropriate tiRetVal. */ + + op1 = impPopStack().val; + assertImp(op1->gtType == TYP_REF); + + helper = info.compCompHnd->getUnBoxHelper(resolvedToken.hClass); + assert(helper == CORINFO_HELP_UNBOX || helper == CORINFO_HELP_UNBOX_NULLABLE); + + // Check legality and profitability of inline expansion for unboxing. + const bool canExpandInline = (helper == CORINFO_HELP_UNBOX); + const bool shouldExpandInline = !compCurBB->isRunRarely() && opts.OptimizationEnabled(); + + if (canExpandInline && shouldExpandInline) { - assert(!iciCall->ShouldHaveRetBufArg()); - assert(retRegCount >= 2); - if (fgNeedReturnSpillTemp()) + // See if we know anything about the type of op1, the object being unboxed. + bool isExact = false; + bool isNonNull = false; + CORINFO_CLASS_HANDLE clsHnd = gtGetClassHandle(op1, &isExact, &isNonNull); + + // We can skip the "exact" bit here as we are comparing to a value class. + // compareTypesForEquality should bail on comparisons for shared value classes. + if (clsHnd != NO_CLASS_HANDLE) { - if (!impInlineInfo->retExpr) + const TypeCompareState compare = + info.compCompHnd->compareTypesForEquality(resolvedToken.hClass, clsHnd); + + if (compare == TypeCompareState::Must) { - // The inlinee compiler has figured out the type of the temp already. Use it here. - impInlineInfo->retExpr = - gtNewLclvNode(lvaInlineeReturnSpillTemp, lvaTable[lvaInlineeReturnSpillTemp].lvType); - } - } - else - { - impInlineInfo->retExpr = op2; - } - } - else -#elif defined(TARGET_X86) - ReturnTypeDesc retTypeDesc; - retTypeDesc.InitializeStructReturnType(this, retClsHnd, info.compCallConv); - unsigned retRegCount = retTypeDesc.GetReturnRegCount(); + JITDUMP("\nOptimizing %s (%s) -- type test will succeed\n", + opcode == CEE_UNBOX ? "UNBOX" : "UNBOX.ANY", eeGetClassName(clsHnd)); - if (retRegCount != 0) - { - assert(!iciCall->ShouldHaveRetBufArg()); - assert(retRegCount == MAX_RET_REG_COUNT); - if (fgNeedReturnSpillTemp()) - { - if (!impInlineInfo->retExpr) + // For UNBOX, null check (if necessary), and then leave the box payload byref on the stack. + if (opcode == CEE_UNBOX) + { + GenTree* cloneOperand; + op1 = impCloneExpr(op1, &cloneOperand, NO_CLASS_HANDLE, CHECK_SPILL_ALL, + nullptr DEBUGARG("optimized unbox clone")); + + GenTree* boxPayloadOffset = gtNewIconNode(TARGET_POINTER_SIZE, TYP_I_IMPL); + GenTree* boxPayloadAddress = + gtNewOperNode(GT_ADD, TYP_BYREF, cloneOperand, boxPayloadOffset); + GenTree* nullcheck = gtNewNullCheck(op1, block); + // Add an ordering dependency between the null + // check and forming the byref; the JIT assumes + // in many places that the only legal null + // byref is literally 0, and since the byref + // leaks out here, we need to ensure it is + // nullchecked. + nullcheck->gtFlags |= GTF_ORDER_SIDEEFF; + boxPayloadAddress->gtFlags |= GTF_ORDER_SIDEEFF; + GenTree* result = gtNewOperNode(GT_COMMA, TYP_BYREF, nullcheck, boxPayloadAddress); + impPushOnStack(result, tiRetVal); + break; + } + + // For UNBOX.ANY load the struct from the box payload byref (the load will nullcheck) + assert(opcode == CEE_UNBOX_ANY); + GenTree* boxPayloadOffset = gtNewIconNode(TARGET_POINTER_SIZE, TYP_I_IMPL); + GenTree* boxPayloadAddress = gtNewOperNode(GT_ADD, TYP_BYREF, op1, boxPayloadOffset); + impPushOnStack(boxPayloadAddress, tiRetVal); + oper = GT_OBJ; + goto OBJ; + } + else { - // The inlinee compiler has figured out the type of the temp already. Use it here. - impInlineInfo->retExpr = - gtNewLclvNode(lvaInlineeReturnSpillTemp, lvaTable[lvaInlineeReturnSpillTemp].lvType); + JITDUMP("\nUnable to optimize %s -- can't resolve type comparison\n", + opcode == CEE_UNBOX ? "UNBOX" : "UNBOX.ANY"); } } else { - impInlineInfo->retExpr = op2; + JITDUMP("\nUnable to optimize %s -- class for [%06u] not known\n", + opcode == CEE_UNBOX ? "UNBOX" : "UNBOX.ANY", dspTreeID(op1)); + } + + JITDUMP("\n Importing %s as inline sequence\n", opcode == CEE_UNBOX ? "UNBOX" : "UNBOX.ANY"); + // we are doing normal unboxing + // inline the common case of the unbox helper + // UNBOX(exp) morphs into + // clone = pop(exp); + // ((*clone == typeToken) ? nop : helper(clone, typeToken)); + // push(clone + TARGET_POINTER_SIZE) + // + GenTree* cloneOperand; + op1 = impCloneExpr(op1, &cloneOperand, NO_CLASS_HANDLE, CHECK_SPILL_ALL, + nullptr DEBUGARG("inline UNBOX clone1")); + op1 = gtNewMethodTableLookup(op1); + + GenTree* condBox = gtNewOperNode(GT_EQ, TYP_INT, op1, op2); + + op1 = impCloneExpr(cloneOperand, &cloneOperand, NO_CLASS_HANDLE, CHECK_SPILL_ALL, + nullptr DEBUGARG("inline UNBOX clone2")); + op2 = impTokenToHandle(&resolvedToken); + if (op2 == nullptr) + { // compDonotInline() + return; } + op1 = gtNewHelperCallNode(helper, TYP_VOID, op2, op1); + + op1 = new (this, GT_COLON) GenTreeColon(TYP_VOID, gtNewNothingNode(), op1); + op1 = gtNewQmarkNode(TYP_VOID, condBox, op1->AsColon()); + + // QMARK nodes cannot reside on the evaluation stack. Because there + // may be other trees on the evaluation stack that side-effect the + // sources of the UNBOX operation we must spill the stack. + + impAppendTree(op1, CHECK_SPILL_ALL, impCurStmtDI); + + // Create the address-expression to reference past the object header + // to the beginning of the value-type. Today this means adjusting + // past the base of the objects vtable field which is pointer sized. + + op2 = gtNewIconNode(TARGET_POINTER_SIZE, TYP_I_IMPL); + op1 = gtNewOperNode(GT_ADD, TYP_BYREF, cloneOperand, op2); } else -#endif // defined(TARGET_ARM64) { - assert(iciCall->gtArgs.HasRetBuffer()); - GenTree* dest = gtCloneExpr(iciCall->gtArgs.GetRetBufferArg()->GetEarlyNode()); - // spill temp only exists if there are multiple return points - if (fgNeedReturnSpillTemp()) - { - // if this is the first return we have seen set the retExpr - if (!impInlineInfo->retExpr) - { - impInlineInfo->retExpr = - impAssignStructPtr(dest, gtNewLclvNode(lvaInlineeReturnSpillTemp, info.compRetType), - retClsHnd, (unsigned)CHECK_SPILL_ALL); - } - } - else + JITDUMP("\n Importing %s as helper call because %s\n", opcode == CEE_UNBOX ? "UNBOX" : "UNBOX.ANY", + canExpandInline ? "want smaller code or faster jitting" : "inline expansion not legal"); + + // Don't optimize, just call the helper and be done with it + op1 = gtNewHelperCallNode(helper, + (var_types)((helper == CORINFO_HELP_UNBOX) ? TYP_BYREF : TYP_STRUCT), op2, + op1); + if (op1->gtType == TYP_STRUCT) { - impInlineInfo->retExpr = impAssignStructPtr(dest, op2, retClsHnd, (unsigned)CHECK_SPILL_ALL); + op1->AsCall()->gtRetClsHnd = resolvedToken.hClass; } } - } - if (impInlineInfo->retExpr != nullptr) - { - impInlineInfo->retBB = compCurBB; - } - } - } + assert((helper == CORINFO_HELP_UNBOX && op1->gtType == TYP_BYREF) || // Unbox helper returns a byref. + (helper == CORINFO_HELP_UNBOX_NULLABLE && + varTypeIsStruct(op1)) // UnboxNullable helper returns a struct. + ); - if (compIsForInlining()) - { - return true; - } + /* + ---------------------------------------------------------------------- + | \ helper | | | + | \ | | | + | \ | CORINFO_HELP_UNBOX | CORINFO_HELP_UNBOX_NULLABLE | + | \ | (which returns a BYREF) | (which returns a STRUCT) | | + | opcode \ | | | + |--------------------------------------------------------------------- + | UNBOX | push the BYREF | spill the STRUCT to a local, | + | | | push the BYREF to this local | + |--------------------------------------------------------------------- + | UNBOX_ANY | push a GT_OBJ of | push the STRUCT | + | | the BYREF | For Linux when the | + | | | struct is returned in two | + | | | registers create a temp | + | | | which address is passed to | + | | | the unbox_nullable helper. | + |--------------------------------------------------------------------- + */ - if (info.compRetType == TYP_VOID) - { - // return void - op1 = new (this, GT_RETURN) GenTreeOp(GT_RETURN, TYP_VOID); - } - else if (info.compRetBuffArg != BAD_VAR_NUM) - { - // Assign value to return buff (first param) - GenTree* retBuffAddr = - gtNewLclvNode(info.compRetBuffArg, TYP_BYREF DEBUGARG(impCurStmtDI.GetLocation().GetOffset())); + if (opcode == CEE_UNBOX) + { + if (helper == CORINFO_HELP_UNBOX_NULLABLE) + { + // Unbox nullable helper returns a struct type. + // We need to spill it to a temp so than can take the address of it. + // Here we need unsafe value cls check, since the address of struct is taken to be used + // further along and potetially be exploitable. - op2 = impAssignStructPtr(retBuffAddr, op2, retClsHnd, (unsigned)CHECK_SPILL_ALL); - impAppendTree(op2, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); + unsigned tmp = lvaGrabTemp(true DEBUGARG("UNBOXing a nullable")); + lvaSetStruct(tmp, resolvedToken.hClass, true /* unsafe value cls check */); - // There are cases where the address of the implicit RetBuf should be returned explicitly (in RAX). - CLANG_FORMAT_COMMENT_ANCHOR; + op2 = gtNewLclvNode(tmp, TYP_STRUCT); + op1 = impAssignStruct(op2, op1, CHECK_SPILL_ALL); + assert(op1->gtType == TYP_VOID); // We must be assigning the return struct to the temp. -#if defined(TARGET_AMD64) + op2 = gtNewLclvNode(tmp, TYP_STRUCT); + op2 = gtNewOperNode(GT_ADDR, TYP_BYREF, op2); + op1 = gtNewOperNode(GT_COMMA, TYP_BYREF, op1, op2); + } - // x64 (System V and Win64) calling convention requires to - // return the implicit return buffer explicitly (in RAX). - // Change the return type to be BYREF. - op1 = gtNewOperNode(GT_RETURN, TYP_BYREF, gtNewLclvNode(info.compRetBuffArg, TYP_BYREF)); -#else // !defined(TARGET_AMD64) - // In case of non-AMD64 targets the profiler hook requires to return the implicit RetBuf explicitly (in RAX). - // In such case the return value of the function is changed to BYREF. - // If profiler hook is not needed the return type of the function is TYP_VOID. - if (compIsProfilerHookNeeded()) - { - op1 = gtNewOperNode(GT_RETURN, TYP_BYREF, gtNewLclvNode(info.compRetBuffArg, TYP_BYREF)); - } -#if defined(TARGET_ARM64) - // On ARM64, the native instance calling convention variant - // requires the implicit ByRef to be explicitly returned. - else if (TargetOS::IsWindows && callConvIsInstanceMethodCallConv(info.compCallConv)) - { - op1 = gtNewOperNode(GT_RETURN, TYP_BYREF, gtNewLclvNode(info.compRetBuffArg, TYP_BYREF)); - } -#endif -#if defined(TARGET_X86) - else if (info.compCallConv != CorInfoCallConvExtension::Managed) - { - op1 = gtNewOperNode(GT_RETURN, TYP_BYREF, gtNewLclvNode(info.compRetBuffArg, TYP_BYREF)); - } -#endif - else - { - // return void - op1 = new (this, GT_RETURN) GenTreeOp(GT_RETURN, TYP_VOID); - } -#endif // !defined(TARGET_AMD64) - } - else if (varTypeIsStruct(info.compRetType)) - { -#if !FEATURE_MULTIREG_RET - // For both ARM architectures the HFA native types are maintained as structs. - // Also on System V AMD64 the multireg structs returns are also left as structs. - noway_assert(info.compRetNativeType != TYP_STRUCT); -#endif - op2 = impFixupStructReturnType(op2, retClsHnd, info.compCallConv); - // return op2 - var_types returnType = info.compRetType; - op1 = gtNewOperNode(GT_RETURN, genActualType(returnType), op2); - } - else - { - // return op2 - op1 = gtNewOperNode(GT_RETURN, genActualType(info.compRetType), op2); - } + assert(op1->gtType == TYP_BYREF); + } + else + { + assert(opcode == CEE_UNBOX_ANY); - // We must have imported a tailcall and jumped to RET - if (isTailCall) - { - assert(verCurrentState.esStackDepth == 0 && impOpcodeIsCallOpcode(opcode)); + if (helper == CORINFO_HELP_UNBOX) + { + // Normal unbox helper returns a TYP_BYREF. + impPushOnStack(op1, tiRetVal); + oper = GT_OBJ; + goto OBJ; + } - opcode = CEE_RET; // To prevent trying to spill if CALL_SITE_BOUNDARIES + assert(helper == CORINFO_HELP_UNBOX_NULLABLE && "Make sure the helper is nullable!"); - // impImportCall() would have already appended TYP_VOID calls - if (info.compRetType == TYP_VOID) - { - return true; - } - } +#if FEATURE_MULTIREG_RET - impAppendTree(op1, (unsigned)CHECK_SPILL_NONE, impCurStmtDI); -#ifdef DEBUG - // Remember at which BC offset the tree was finished - impNoteLastILoffs(); -#endif - return true; -} + if (varTypeIsStruct(op1) && + IsMultiRegReturnedType(resolvedToken.hClass, CorInfoCallConvExtension::Managed)) + { + // Unbox nullable helper returns a TYP_STRUCT. + // For the multi-reg case we need to spill it to a temp so that + // we can pass the address to the unbox_nullable jit helper. -/***************************************************************************** - * Mark the block as unimported. - * Note that the caller is responsible for calling impImportBlockPending(), - * with the appropriate stack-state - */ + unsigned tmp = lvaGrabTemp(true DEBUGARG("UNBOXing a register returnable nullable")); + lvaTable[tmp].lvIsMultiRegArg = true; + lvaSetStruct(tmp, resolvedToken.hClass, true /* unsafe value cls check */); -inline void Compiler::impReimportMarkBlock(BasicBlock* block) -{ -#ifdef DEBUG - if (verbose && (block->bbFlags & BBF_IMPORTED)) - { - printf("\n" FMT_BB " will be reimported\n", block->bbNum); - } -#endif + op2 = gtNewLclvNode(tmp, TYP_STRUCT); + op1 = impAssignStruct(op2, op1, CHECK_SPILL_ALL); + assert(op1->gtType == TYP_VOID); // We must be assigning the return struct to the temp. - block->bbFlags &= ~BBF_IMPORTED; -} + op2 = gtNewLclvNode(tmp, TYP_STRUCT); + op2 = gtNewOperNode(GT_ADDR, TYP_BYREF, op2); + op1 = gtNewOperNode(GT_COMMA, TYP_BYREF, op1, op2); -/***************************************************************************** - * Mark the successors of the given block as unimported. - * Note that the caller is responsible for calling impImportBlockPending() - * for all the successors, with the appropriate stack-state. - */ + // In this case the return value of the unbox helper is TYP_BYREF. + // Make sure the right type is placed on the operand type stack. + impPushOnStack(op1, tiRetVal); -void Compiler::impReimportMarkSuccessors(BasicBlock* block) -{ - for (BasicBlock* const succBlock : block->Succs()) - { - impReimportMarkBlock(succBlock); - } -} + // Load the struct. + oper = GT_OBJ; -/***************************************************************************** - * - * Filter wrapper to handle only passed in exception code - * from it). - */ + assert(op1->gtType == TYP_BYREF); -LONG FilterVerificationExceptions(PEXCEPTION_POINTERS pExceptionPointers, LPVOID lpvParam) -{ - if (pExceptionPointers->ExceptionRecord->ExceptionCode == SEH_VERIFICATION_EXCEPTION) - { - return EXCEPTION_EXECUTE_HANDLER; - } + goto OBJ; + } + else - return EXCEPTION_CONTINUE_SEARCH; -} +#endif // !FEATURE_MULTIREG_RET -void Compiler::impVerifyEHBlock(BasicBlock* block, bool isTryStart) -{ - assert(block->hasTryIndex()); - assert(!compIsForInlining()); + { + // If non register passable struct we have it materialized in the RetBuf. + assert(op1->gtType == TYP_STRUCT); + tiRetVal = verMakeTypeInfo(resolvedToken.hClass); + assert(tiRetVal.IsValueClass()); + } + } - unsigned tryIndex = block->getTryIndex(); - EHblkDsc* HBtab = ehGetDsc(tryIndex); + impPushOnStack(op1, tiRetVal); + } + break; - if (isTryStart) - { - assert(block->bbFlags & BBF_TRY_BEG); + case CEE_BOX: + { + /* Get the Class index */ + assertImp(sz == sizeof(unsigned)); - // The Stack must be empty - // - if (block->bbStkDepth != 0) - { - BADCODE("Evaluation stack must be empty on entry into a try block"); - } - } + _impResolveToken(CORINFO_TOKENKIND_Box); - // Save the stack contents, we'll need to restore it later - // - SavedStack blockState; - impSaveStackState(&blockState, false); + JITDUMP(" %08X", resolvedToken.token); - while (HBtab != nullptr) - { - if (isTryStart) - { - // Are we verifying that an instance constructor properly initializes it's 'this' pointer once? - // We do not allow the 'this' pointer to be uninitialized when entering most kinds try regions - // - if (verTrackObjCtorInitState && (verCurrentState.thisInitialized != TIS_Init)) - { - // We trigger an invalid program exception here unless we have a try/fault region. - // - if (HBtab->HasCatchHandler() || HBtab->HasFinallyHandler() || HBtab->HasFilter()) + accessAllowedResult = + info.compCompHnd->canAccessClass(&resolvedToken, info.compMethodHnd, &calloutHelper); + impHandleAccessAllowed(accessAllowedResult, &calloutHelper); + + // Note BOX can be used on things that are not value classes, in which + // case we get a NOP. However the verifier's view of the type on the + // stack changes (in generic code a 'T' becomes a 'boxed T') + if (!eeIsValueClass(resolvedToken.hClass)) { - BADCODE( - "The 'this' pointer of an instance constructor is not initialized upon entry to a try region"); + JITDUMP("\n Importing BOX(refClass) as NOP\n"); + verCurrentState.esStack[verCurrentState.esStackDepth - 1].seTypeInfo = tiRetVal; + break; } - else + + bool isByRefLike = + (info.compCompHnd->getClassAttribs(resolvedToken.hClass) & CORINFO_FLG_BYREF_LIKE) != 0; + if (isByRefLike) { - // Allow a try/fault region to proceed. - assert(HBtab->HasFaultHandler()); + // For ByRefLike types we are required to either fold the + // recognized patterns in impBoxPatternMatch or otherwise + // throw InvalidProgramException at runtime. In either case + // we will need to spill side effects of the expression. + impSpillSideEffects(false, CHECK_SPILL_ALL DEBUGARG("Required for box of ByRefLike type")); } - } - } - - // Recursively process the handler block, if we haven't already done so. - BasicBlock* hndBegBB = HBtab->ebdHndBeg; - - if (((hndBegBB->bbFlags & BBF_IMPORTED) == 0) && (impGetPendingBlockMember(hndBegBB) == 0)) - { - // Construct the proper verification stack state - // either empty or one that contains just - // the Exception Object that we are dealing with - // - verCurrentState.esStackDepth = 0; - if (handlerGetsXcptnObj(hndBegBB->bbCatchTyp)) - { - CORINFO_CLASS_HANDLE clsHnd; + // Look ahead for box idioms + int matched = impBoxPatternMatch(&resolvedToken, codeAddr + sz, codeEndp, + isByRefLike ? BoxPatterns::IsByRefLike : BoxPatterns::None); + if (matched >= 0) + { + // Skip the matched IL instructions + sz += matched; + break; + } - if (HBtab->HasFilter()) + if (isByRefLike) { - clsHnd = impGetObjectClass(); + // ByRefLike types are supported in boxing scenarios when the instruction can be elided + // due to a recognized pattern above. If the pattern is not recognized, the code is invalid. + BADCODE("ByRefLike types cannot be boxed"); } else { - CORINFO_RESOLVED_TOKEN resolvedToken; - - resolvedToken.tokenContext = impTokenLookupContextHandle; - resolvedToken.tokenScope = info.compScopeHnd; - resolvedToken.token = HBtab->ebdTyp; - resolvedToken.tokenType = CORINFO_TOKENKIND_Class; - info.compCompHnd->resolveToken(&resolvedToken); - - clsHnd = resolvedToken.hClass; + impImportAndPushBox(&resolvedToken); + if (compDonotInline()) + { + return; + } } - - // push catch arg the stack, spill to a temp if necessary - // Note: can update HBtab->ebdHndBeg! - hndBegBB = impPushCatchArgOnStack(hndBegBB, clsHnd, false); } + break; - // Queue up the handler for importing - // - impImportBlockPending(hndBegBB); - } + case CEE_SIZEOF: - // Process the filter block, if we haven't already done so. - if (HBtab->HasFilter()) - { - /* @VERIFICATION : Ideally the end of filter state should get - propagated to the catch handler, this is an incompleteness, - but is not a security/compliance issue, since the only - interesting state is the 'thisInit' state. - */ + /* Get the Class index */ + assertImp(sz == sizeof(unsigned)); - BasicBlock* filterBB = HBtab->ebdFilter; + _impResolveToken(CORINFO_TOKENKIND_Class); - if (((filterBB->bbFlags & BBF_IMPORTED) == 0) && (impGetPendingBlockMember(filterBB) == 0)) - { - verCurrentState.esStackDepth = 0; + JITDUMP(" %08X", resolvedToken.token); - // push catch arg the stack, spill to a temp if necessary - // Note: can update HBtab->ebdFilter! - const bool isSingleBlockFilter = (filterBB->bbNext == hndBegBB); - filterBB = impPushCatchArgOnStack(filterBB, impGetObjectClass(), isSingleBlockFilter); + op1 = gtNewIconNode(info.compCompHnd->getClassSize(resolvedToken.hClass)); + impPushOnStack(op1, tiRetVal); + break; - impImportBlockPending(filterBB); - } - } + case CEE_CASTCLASS: - // This seems redundant ....?? - if (verTrackObjCtorInitState && HBtab->HasFaultHandler()) - { - /* Recursively process the handler block */ + /* Get the Class index */ - verCurrentState.esStackDepth = 0; + assertImp(sz == sizeof(unsigned)); - // Queue up the fault handler for importing - // - impImportBlockPending(HBtab->ebdHndBeg); - } + _impResolveToken(CORINFO_TOKENKIND_Casting); - // Now process our enclosing try index (if any) - // - tryIndex = HBtab->ebdEnclosingTryIndex; - if (tryIndex == EHblkDsc::NO_ENCLOSING_INDEX) - { - HBtab = nullptr; - } - else - { - HBtab = ehGetDsc(tryIndex); - } - } + JITDUMP(" %08X", resolvedToken.token); - // Restore the stack contents - impRestoreStackState(&blockState); -} + if (!opts.IsReadyToRun()) + { + op2 = impTokenToHandle(&resolvedToken, nullptr, false); + if (op2 == nullptr) + { // compDonotInline() + return; + } + } -//*************************************************************** -// Import the instructions for the given basic block. Perform -// verification, throwing an exception on failure. Push any successor blocks that are enabled for the first -// time, or whose verification pre-state is changed. + accessAllowedResult = + info.compCompHnd->canAccessClass(&resolvedToken, info.compMethodHnd, &calloutHelper); + impHandleAccessAllowed(accessAllowedResult, &calloutHelper); -#ifdef _PREFAST_ -#pragma warning(push) -#pragma warning(disable : 21000) // Suppress PREFast warning about overly large function -#endif -void Compiler::impImportBlock(BasicBlock* block) -{ - // BBF_INTERNAL blocks only exist during importation due to EH canonicalization. We need to - // handle them specially. In particular, there is no IL to import for them, but we do need - // to mark them as imported and put their successors on the pending import list. - if (block->bbFlags & BBF_INTERNAL) - { - JITDUMP("Marking BBF_INTERNAL block " FMT_BB " as BBF_IMPORTED\n", block->bbNum); - block->bbFlags |= BBF_IMPORTED; + op1 = impPopStack().val; - for (BasicBlock* const succBlock : block->Succs()) - { - impImportBlockPending(succBlock); - } + /* Pop the address and create the 'checked cast' helper call */ - return; - } + // At this point we expect typeRef to contain the token, op1 to contain the value being cast, + // and op2 to contain code that creates the type handle corresponding to typeRef + CASTCLASS: + { + GenTree* optTree = impOptimizeCastClassOrIsInst(op1, &resolvedToken, true); - bool markImport; + if (optTree != nullptr) + { + impPushOnStack(optTree, tiRetVal); + } + else + { - assert(block); +#ifdef FEATURE_READYTORUN + if (opts.IsReadyToRun()) + { + GenTreeCall* opLookup = + impReadyToRunHelperToTree(&resolvedToken, CORINFO_HELP_READYTORUN_CHKCAST, TYP_REF, nullptr, + op1); + usingReadyToRunHelper = (opLookup != nullptr); + op1 = (usingReadyToRunHelper ? opLookup : op1); - /* Make the block globaly available */ + if (!usingReadyToRunHelper) + { + // TODO: ReadyToRun: When generic dictionary lookups are necessary, replace the lookup call + // and the chkcastany call with a single call to a dynamic R2R cell that will: + // 1) Load the context + // 2) Perform the generic dictionary lookup and caching, and generate the appropriate + // stub + // 3) Check the object on the stack for the type-cast + // Reason: performance (today, we'll always use the slow helper for the R2R generics case) - compCurBB = block; + op2 = impTokenToHandle(&resolvedToken, nullptr, false); + if (op2 == nullptr) + { // compDonotInline() + return; + } + } + } -#ifdef DEBUG - /* Initialize the debug variables */ - impCurOpcName = "unknown"; - impCurOpcOffs = block->bbCodeOffs; + if (!usingReadyToRunHelper) #endif + { + op1 = impCastClassOrIsInstToTree(op1, op2, &resolvedToken, true, opcodeOffs); + } + if (compDonotInline()) + { + return; + } - /* Set the current stack state to the merged result */ - verResetCurrentState(block, &verCurrentState); - - /* Now walk the code and import the IL into GenTrees */ - - struct FilterVerificationExceptionsParam - { - Compiler* pThis; - BasicBlock* block; - }; - FilterVerificationExceptionsParam param; - - param.pThis = this; - param.block = block; - - PAL_TRY(FilterVerificationExceptionsParam*, pParam, ¶m) - { - /* @VERIFICATION : For now, the only state propagation from try - to it's handler is "thisInit" state (stack is empty at start of try). - In general, for state that we track in verification, we need to - model the possibility that an exception might happen at any IL - instruction, so we really need to merge all states that obtain - between IL instructions in a try block into the start states of - all handlers. - - However we do not allow the 'this' pointer to be uninitialized when - entering most kinds try regions (only try/fault are allowed to have - an uninitialized this pointer on entry to the try) - - Fortunately, the stack is thrown away when an exception - leads to a handler, so we don't have to worry about that. - We DO, however, have to worry about the "thisInit" state. - But only for the try/fault case. - - The only allowed transition is from TIS_Uninit to TIS_Init. + /* Push the result back on the stack */ + impPushOnStack(op1, tiRetVal); + } + } + break; - So for a try/fault region for the fault handler block - we will merge the start state of the try begin - and the post-state of each block that is part of this try region - */ + case CEE_THROW: - // merge the start state of the try begin - // - if (pParam->block->bbFlags & BBF_TRY_BEG) - { - pParam->pThis->impVerifyEHBlock(pParam->block, true); - } + // Any block with a throw is rarely executed. + block->bbSetRunRarely(); - pParam->pThis->impImportBlockCode(pParam->block); + // Pop the exception object and create the 'throw' helper call + op1 = gtNewHelperCallNode(CORINFO_HELP_THROW, TYP_VOID, impPopStack().val); - // As discussed above: - // merge the post-state of each block that is part of this try region - // - if (pParam->block->hasTryIndex()) - { - pParam->pThis->impVerifyEHBlock(pParam->block, false); - } - } - PAL_EXCEPT_FILTER(FilterVerificationExceptions) - { - verHandleVerificationFailure(block DEBUGARG(false)); - } - PAL_ENDTRY + // Fall through to clear out the eval stack. - if (compDonotInline()) - { - return; - } + EVAL_APPEND: + if (verCurrentState.esStackDepth > 0) + { + impEvalSideEffects(); + } - assert(!compDonotInline()); + assert(verCurrentState.esStackDepth == 0); - markImport = false; + goto APPEND; -SPILLSTACK: + case CEE_RETHROW: - unsigned baseTmp = NO_BASE_TMP; // input temps assigned to successor blocks - bool reimportSpillClique = false; - BasicBlock* tgtBlock = nullptr; + assert(!compIsForInlining()); - /* If the stack is non-empty, we might have to spill its contents */ + if (info.compXcptnsCount == 0) + { + BADCODE("rethrow outside catch"); + } - if (verCurrentState.esStackDepth != 0) - { - impBoxTemp = BAD_VAR_NUM; // if a box temp is used in a block that leaves something - // on the stack, its lifetime is hard to determine, simply - // don't reuse such temps. + /* Create the 'rethrow' helper call */ - Statement* addStmt = nullptr; + op1 = gtNewHelperCallNode(CORINFO_HELP_RETHROW, TYP_VOID); - /* Do the successors of 'block' have any other predecessors ? - We do not want to do some of the optimizations related to multiRef - if we can reimport blocks */ + goto EVAL_APPEND; - unsigned multRef = impCanReimport ? unsigned(~0) : 0; + case CEE_INITOBJ: - switch (block->bbJumpKind) - { - case BBJ_COND: + assertImp(sz == sizeof(unsigned)); - addStmt = impExtractLastStmt(); + _impResolveToken(CORINFO_TOKENKIND_Class); - assert(addStmt->GetRootNode()->gtOper == GT_JTRUE); + JITDUMP(" %08X", resolvedToken.token); - /* Note if the next block has more than one ancestor */ + lclTyp = JITtype2varType(info.compCompHnd->asCorInfoType(resolvedToken.hClass)); + if (lclTyp != TYP_STRUCT) + { + op2 = gtNewZeroConNode(genActualType(lclTyp)); + goto STIND_VALUE; + } - multRef |= block->bbNext->bbRefs; + op1 = impPopStack().val; + op1 = gtNewStructVal(typGetObjLayout(resolvedToken.hClass), op1); + op2 = gtNewIconNode(0); + op1 = gtNewBlkOpNode(op1, op2, (prefixFlags & PREFIX_VOLATILE) != 0); + goto SPILL_APPEND; - /* Does the next block have temps assigned? */ + case CEE_INITBLK: - baseTmp = block->bbNext->bbStkTempsIn; - tgtBlock = block->bbNext; + op3 = impPopStack().val; // Size + op2 = impPopStack().val; // Value + op1 = impPopStack().val; // Dst addr - if (baseTmp != NO_BASE_TMP) + if (op3->IsCnsIntOrI()) { - break; + size = (unsigned)op3->AsIntConCommon()->IconValue(); + op1 = new (this, GT_BLK) GenTreeBlk(GT_BLK, TYP_STRUCT, op1, typGetBlkLayout(size)); + op1 = gtNewBlkOpNode(op1, op2, (prefixFlags & PREFIX_VOLATILE) != 0); } + else + { + if (!op2->IsIntegralConst(0)) + { + op2 = gtNewOperNode(GT_INIT_VAL, TYP_INT, op2); + } - /* Try the target of the jump then */ +#ifdef TARGET_64BIT + // STORE_DYN_BLK takes a native uint size as it turns into call to memset. + op3 = gtNewCastNode(TYP_I_IMPL, op3, /* fromUnsigned */ true, TYP_U_IMPL); +#endif - multRef |= block->bbJumpDest->bbRefs; - baseTmp = block->bbJumpDest->bbStkTempsIn; - tgtBlock = block->bbJumpDest; - break; + op1 = new (this, GT_STORE_DYN_BLK) GenTreeStoreDynBlk(op1, op2, op3); + size = 0; - case BBJ_ALWAYS: - multRef |= block->bbJumpDest->bbRefs; - baseTmp = block->bbJumpDest->bbStkTempsIn; - tgtBlock = block->bbJumpDest; - break; + if ((prefixFlags & PREFIX_VOLATILE) != 0) + { + op1->gtFlags |= GTF_BLK_VOLATILE; + } + } + goto SPILL_APPEND; - case BBJ_NONE: - multRef |= block->bbNext->bbRefs; - baseTmp = block->bbNext->bbStkTempsIn; - tgtBlock = block->bbNext; - break; + case CEE_CPBLK: - case BBJ_SWITCH: - addStmt = impExtractLastStmt(); - assert(addStmt->GetRootNode()->gtOper == GT_SWITCH); + op3 = impPopStack().val; // Size + op2 = impPopStack().val; // Src addr + op1 = impPopStack().val; // Dst addr - for (BasicBlock* const tgtBlock : block->SwitchTargets()) + if (op3->IsCnsIntOrI()) { - multRef |= tgtBlock->bbRefs; + size = static_cast(op3->AsIntConCommon()->IconValue()); - // Thanks to spill cliques, we should have assigned all or none - assert((baseTmp == NO_BASE_TMP) || (baseTmp == tgtBlock->bbStkTempsIn)); - baseTmp = tgtBlock->bbStkTempsIn; - if (multRef > 1) - { - break; - } + op1 = gtNewBlockVal(op1, size); + op2 = gtNewBlockVal(op2, size); + op1 = gtNewBlkOpNode(op1, op2, (prefixFlags & PREFIX_VOLATILE) != 0); } - break; - - case BBJ_CALLFINALLY: - case BBJ_EHCATCHRET: - case BBJ_RETURN: - case BBJ_EHFINALLYRET: - case BBJ_EHFILTERRET: - case BBJ_THROW: - BADCODE("can't have 'unreached' end of BB with non-empty stack"); - break; + else + { + op2 = gtNewOperNode(GT_IND, TYP_STRUCT, op2); - default: - noway_assert(!"Unexpected bbJumpKind"); - break; - } +#ifdef TARGET_64BIT + // STORE_DYN_BLK takes a native uint size as it turns into call to memcpy. + op3 = gtNewCastNode(TYP_I_IMPL, op3, /* fromUnsigned */ true, TYP_U_IMPL); +#endif - assert(multRef >= 1); + op1 = new (this, GT_STORE_DYN_BLK) GenTreeStoreDynBlk(op1, op2, op3); - /* Do we have a base temp number? */ + if ((prefixFlags & PREFIX_VOLATILE) != 0) + { + op1->gtFlags |= GTF_BLK_VOLATILE; + } + } + goto SPILL_APPEND; - bool newTemps = (baseTmp == NO_BASE_TMP); + case CEE_CPOBJ: + { + assertImp(sz == sizeof(unsigned)); - if (newTemps) - { - /* Grab enough temps for the whole stack */ - baseTmp = impGetSpillTmpBase(block); - } + _impResolveToken(CORINFO_TOKENKIND_Class); - /* Spill all stack entries into temps */ - unsigned level, tempNum; + JITDUMP(" %08X", resolvedToken.token); - JITDUMP("\nSpilling stack entries into temps\n"); - for (level = 0, tempNum = baseTmp; level < verCurrentState.esStackDepth; level++, tempNum++) - { - GenTree* tree = verCurrentState.esStack[level].val; + lclTyp = JITtype2varType(info.compCompHnd->asCorInfoType(resolvedToken.hClass)); - /* VC generates code where it pushes a byref from one branch, and an int (ldc.i4 0) from - the other. This should merge to a byref in unverifiable code. - However, if the branch which leaves the TYP_I_IMPL on the stack is imported first, the - successor would be imported assuming there was a TYP_I_IMPL on - the stack. Thus the value would not get GC-tracked. Hence, - change the temp to TYP_BYREF and reimport the successors. - Note: We should only allow this in unverifiable code. - */ - if (tree->gtType == TYP_BYREF && lvaTable[tempNum].lvType == TYP_I_IMPL) - { - lvaTable[tempNum].lvType = TYP_BYREF; - impReimportMarkSuccessors(block); - markImport = true; - } + if (lclTyp != TYP_STRUCT) + { + op1 = impPopStack().val; // address to load from -#ifdef TARGET_64BIT - if (genActualType(tree->gtType) == TYP_I_IMPL && lvaTable[tempNum].lvType == TYP_INT) - { - // Some other block in the spill clique set this to "int", but now we have "native int". - // Change the type and go back to re-import any blocks that used the wrong type. - lvaTable[tempNum].lvType = TYP_I_IMPL; - reimportSpillClique = true; - } - else if (genActualType(tree->gtType) == TYP_INT && lvaTable[tempNum].lvType == TYP_I_IMPL) - { - // Spill clique has decided this should be "native int", but this block only pushes an "int". - // Insert a sign-extension to "native int" so we match the clique. - verCurrentState.esStack[level].val = gtNewCastNode(TYP_I_IMPL, tree, false, TYP_I_IMPL); - } + op1 = gtNewIndir(lclTyp, op1); + op1->gtFlags |= GTF_GLOB_REF; - // Consider the case where one branch left a 'byref' on the stack and the other leaves - // an 'int'. On 32-bit, this is allowed (in non-verifiable code) since they are the same - // size. JIT64 managed to make this work on 64-bit. For compatibility, we support JIT64 - // behavior instead of asserting and then generating bad code (where we save/restore the - // low 32 bits of a byref pointer to an 'int' sized local). If the 'int' side has been - // imported already, we need to change the type of the local and reimport the spill clique. - // If the 'byref' side has imported, we insert a cast from int to 'native int' to match - // the 'byref' size. - if (genActualType(tree->gtType) == TYP_BYREF && lvaTable[tempNum].lvType == TYP_INT) - { - // Some other block in the spill clique set this to "int", but now we have "byref". - // Change the type and go back to re-import any blocks that used the wrong type. - lvaTable[tempNum].lvType = TYP_BYREF; - reimportSpillClique = true; - } - else if (genActualType(tree->gtType) == TYP_INT && lvaTable[tempNum].lvType == TYP_BYREF) - { - // Spill clique has decided this should be "byref", but this block only pushes an "int". - // Insert a sign-extension to "native int" so we match the clique size. - verCurrentState.esStack[level].val = gtNewCastNode(TYP_I_IMPL, tree, false, TYP_I_IMPL); - } + impPushOnStack(op1, typeInfo()); + goto STIND; + } -#endif // TARGET_64BIT + op2 = impPopStack().val; // Src addr + op1 = impPopStack().val; // Dest addr - if (tree->gtType == TYP_DOUBLE && lvaTable[tempNum].lvType == TYP_FLOAT) - { - // Some other block in the spill clique set this to "float", but now we have "double". - // Change the type and go back to re-import any blocks that used the wrong type. - lvaTable[tempNum].lvType = TYP_DOUBLE; - reimportSpillClique = true; + ClassLayout* layout = typGetObjLayout(resolvedToken.hClass); + op1 = gtNewStructVal(layout, op1); + op2 = gtNewStructVal(layout, op2); + if (op1->OperIs(GT_OBJ)) + { + gtSetObjGcInfo(op1->AsObj()); + } + op1 = gtNewBlkOpNode(op1, op2, ((prefixFlags & PREFIX_VOLATILE) != 0)); + goto SPILL_APPEND; } - else if (tree->gtType == TYP_FLOAT && lvaTable[tempNum].lvType == TYP_DOUBLE) + + case CEE_STOBJ: { - // Spill clique has decided this should be "double", but this block only pushes a "float". - // Insert a cast to "double" so we match the clique. - verCurrentState.esStack[level].val = gtNewCastNode(TYP_DOUBLE, tree, false, TYP_DOUBLE); - } + assertImp(sz == sizeof(unsigned)); - /* If addStmt has a reference to tempNum (can only happen if we - are spilling to the temps already used by a previous block), - we need to spill addStmt */ + _impResolveToken(CORINFO_TOKENKIND_Class); - if (addStmt != nullptr && !newTemps && gtHasRef(addStmt->GetRootNode(), tempNum)) - { - GenTree* addTree = addStmt->GetRootNode(); + JITDUMP(" %08X", resolvedToken.token); - if (addTree->gtOper == GT_JTRUE) + lclTyp = JITtype2varType(info.compCompHnd->asCorInfoType(resolvedToken.hClass)); + + if (lclTyp != TYP_STRUCT) { - GenTree* relOp = addTree->AsOp()->gtOp1; - assert(relOp->OperIsCompare()); + goto STIND; + } - var_types type = genActualType(relOp->AsOp()->gtOp1->TypeGet()); + op2 = impPopStack().val; // Value + op1 = impPopStack().val; // Ptr - if (gtHasRef(relOp->AsOp()->gtOp1, tempNum)) - { - unsigned temp = lvaGrabTemp(true DEBUGARG("spill addStmt JTRUE ref Op1")); - impAssignTempGen(temp, relOp->AsOp()->gtOp1, level); - type = genActualType(lvaTable[temp].TypeGet()); - relOp->AsOp()->gtOp1 = gtNewLclvNode(temp, type); - } + assertImp(varTypeIsStruct(op2)); - if (gtHasRef(relOp->AsOp()->gtOp2, tempNum)) - { - unsigned temp = lvaGrabTemp(true DEBUGARG("spill addStmt JTRUE ref Op2")); - impAssignTempGen(temp, relOp->AsOp()->gtOp2, level); - type = genActualType(lvaTable[temp].TypeGet()); - relOp->AsOp()->gtOp2 = gtNewLclvNode(temp, type); - } - } - else - { - assert(addTree->gtOper == GT_SWITCH && genActualTypeIsIntOrI(addTree->AsOp()->gtOp1->TypeGet())); + op1 = impAssignStructPtr(op1, op2, resolvedToken.hClass, CHECK_SPILL_ALL); - unsigned temp = lvaGrabTemp(true DEBUGARG("spill addStmt SWITCH")); - impAssignTempGen(temp, addTree->AsOp()->gtOp1, level); - addTree->AsOp()->gtOp1 = gtNewLclvNode(temp, genActualType(addTree->AsOp()->gtOp1->TypeGet())); + if (op1->OperIsBlkOp() && (prefixFlags & PREFIX_UNALIGNED)) + { + op1->gtFlags |= GTF_BLK_UNALIGNED; } + goto SPILL_APPEND; } - /* Spill the stack entry, and replace with the temp */ + case CEE_MKREFANY: - if (!impSpillStackEntry(level, tempNum -#ifdef DEBUG - , - true, "Spill Stack Entry" -#endif - )) - { - if (markImport) - { - BADCODE("bad stack state"); - } + assert(!compIsForInlining()); - // Oops. Something went wrong when spilling. Bad code. - verHandleVerificationFailure(block DEBUGARG(true)); + // Being lazy here. Refanys are tricky in terms of gc tracking. + // Since it is uncommon, just don't perform struct promotion in any method that contains mkrefany. - goto SPILLSTACK; - } - } + JITDUMP("disabling struct promotion because of mkrefany\n"); + fgNoStructPromotion = true; - /* Put back the 'jtrue'/'switch' if we removed it earlier */ + oper = GT_MKREFANY; + assertImp(sz == sizeof(unsigned)); - if (addStmt != nullptr) - { - impAppendStmt(addStmt, (unsigned)CHECK_SPILL_NONE); - } - } + _impResolveToken(CORINFO_TOKENKIND_Class); - // Some of the append/spill logic works on compCurBB + JITDUMP(" %08X", resolvedToken.token); - assert(compCurBB == block); + op2 = impTokenToHandle(&resolvedToken, nullptr, true); + if (op2 == nullptr) + { // compDonotInline() + return; + } - /* Save the tree list in the block */ - impEndTreeList(block); + accessAllowedResult = + info.compCompHnd->canAccessClass(&resolvedToken, info.compMethodHnd, &calloutHelper); + impHandleAccessAllowed(accessAllowedResult, &calloutHelper); - // impEndTreeList sets BBF_IMPORTED on the block - // We do *NOT* want to set it later than this because - // impReimportSpillClique might clear it if this block is both a - // predecessor and successor in the current spill clique - assert(block->bbFlags & BBF_IMPORTED); + op1 = impPopStack().val; - // If we had a int/native int, or float/double collision, we need to re-import - if (reimportSpillClique) - { - // This will re-import all the successors of block (as well as each of their predecessors) - impReimportSpillClique(block); + // @SPECVIOLATION: TYP_INT should not be allowed here by a strict reading of the spec. + // But JIT32 allowed it, so we continue to allow it. + assertImp(op1->TypeGet() == TYP_BYREF || op1->TypeGet() == TYP_I_IMPL || op1->TypeGet() == TYP_INT); - // For blocks that haven't been imported yet, we still need to mark them as pending import. - for (BasicBlock* const succ : block->Succs()) - { - if ((succ->bbFlags & BBF_IMPORTED) == 0) + // MKREFANY returns a struct. op2 is the class token. + op1 = gtNewOperNode(oper, TYP_STRUCT, op1, op2); + + impPushOnStack(op1, verMakeTypeInfo(impGetRefAnyClass())); + break; + + case CEE_LDOBJ: { - impImportBlockPending(succ); + oper = GT_OBJ; + assertImp(sz == sizeof(unsigned)); + + _impResolveToken(CORINFO_TOKENKIND_Class); + + JITDUMP(" %08X", resolvedToken.token); + + OBJ: + lclTyp = JITtype2varType(info.compCompHnd->asCorInfoType(resolvedToken.hClass)); + tiRetVal = verMakeTypeInfo(resolvedToken.hClass); + + if (lclTyp != TYP_STRUCT) + { + goto LDIND; + } + + op1 = impPopStack().val; + + assertImp((genActualType(op1) == TYP_I_IMPL) || op1->TypeIs(TYP_BYREF)); + + op1 = gtNewObjNode(resolvedToken.hClass, op1); + op1->gtFlags |= GTF_EXCEPT; + + if (prefixFlags & PREFIX_UNALIGNED) + { + op1->gtFlags |= GTF_IND_UNALIGNED; + } + + impPushOnStack(op1, tiRetVal); + break; } - } - } - else // the normal case - { - // otherwise just import the successors of block - /* Does this block jump to any other blocks? */ - for (BasicBlock* const succ : block->Succs()) - { - impImportBlockPending(succ); - } - } -} -#ifdef _PREFAST_ -#pragma warning(pop) -#endif + case CEE_LDLEN: + op1 = impPopStack().val; + if (opts.OptimizationEnabled()) + { + /* Use GT_ARR_LENGTH operator so rng check opts see this */ + GenTreeArrLen* arrLen = gtNewArrLen(TYP_INT, op1, OFFSETOF__CORINFO_Array__length, block); -/*****************************************************************************/ -// -// Ensures that "block" is a member of the list of BBs waiting to be imported, pushing it on the list if -// necessary (and ensures that it is a member of the set of BB's on the list, by setting its byte in -// impPendingBlockMembers). Merges the current verification state into the verification state of "block" -// (its "pre-state"). + op1 = arrLen; + } + else + { + /* Create the expression "*(array_addr + ArrLenOffs)" */ + op1 = gtNewOperNode(GT_ADD, TYP_BYREF, op1, + gtNewIconNode(OFFSETOF__CORINFO_Array__length, TYP_I_IMPL)); + op1 = gtNewIndir(TYP_INT, op1); + } -void Compiler::impImportBlockPending(BasicBlock* block) -{ -#ifdef DEBUG - if (verbose) - { - printf("\nimpImportBlockPending for " FMT_BB "\n", block->bbNum); - } -#endif + /* Push the result back on the stack */ + impPushOnStack(op1, tiRetVal); + break; - // We will add a block to the pending set if it has not already been imported (or needs to be re-imported), - // or if it has, but merging in a predecessor's post-state changes the block's pre-state. - // (When we're doing verification, we always attempt the merge to detect verification errors.) + case CEE_BREAK: + op1 = gtNewHelperCallNode(CORINFO_HELP_USER_BREAKPOINT, TYP_VOID); + goto SPILL_APPEND; - // If the block has not been imported, add to pending set. - bool addToPending = ((block->bbFlags & BBF_IMPORTED) == 0); + case CEE_NOP: + if (opts.compDbgCode) + { + op1 = new (this, GT_NO_OP) GenTree(GT_NO_OP, TYP_VOID); + goto SPILL_APPEND; + } + break; - // Initialize bbEntryState just the first time we try to add this block to the pending list - // Just because bbEntryState is NULL, doesn't mean the pre-state wasn't previously set - // We use NULL to indicate the 'common' state to avoid memory allocation - if ((block->bbEntryState == nullptr) && ((block->bbFlags & (BBF_IMPORTED | BBF_FAILED_VERIFICATION)) == 0) && - (impGetPendingBlockMember(block) == 0)) - { - verInitBBEntryState(block, &verCurrentState); - assert(block->bbStkDepth == 0); - block->bbStkDepth = static_cast(verCurrentState.esStackDepth); - assert(addToPending); - assert(impGetPendingBlockMember(block) == 0); - } - else - { - // The stack should have the same height on entry to the block from all its predecessors. - if (block->bbStkDepth != verCurrentState.esStackDepth) - { -#ifdef DEBUG - char buffer[400]; - sprintf_s(buffer, sizeof(buffer), - "Block at offset %4.4x to %4.4x in %0.200s entered with different stack depths.\n" - "Previous depth was %d, current depth is %d", - block->bbCodeOffs, block->bbCodeOffsEnd, info.compFullName, block->bbStkDepth, - verCurrentState.esStackDepth); - buffer[400 - 1] = 0; - NO_WAY(buffer); -#else - NO_WAY("Block entered with different stack depths"); -#endif - } + /******************************** NYI *******************************/ - if (!addToPending) - { - return; - } + case 0xCC: + OutputDebugStringA("CLR: Invalid x86 breakpoint in IL stream\n"); + FALLTHROUGH; + + case CEE_ILLEGAL: + case CEE_MACRO_END: + + default: + if (compIsForInlining()) + { + compInlineResult->NoteFatal(InlineObservation::CALLEE_COMPILATION_ERROR); + return; + } - if (block->bbStkDepth > 0) - { - // We need to fix the types of any spill temps that might have changed: - // int->native int, float->double, int->byref, etc. - impRetypeEntryStateTemps(block); + BADCODE3("unknown opcode", ": %02X", (int)opcode); } - // OK, we must add to the pending list, if it's not already in it. - if (impGetPendingBlockMember(block) != 0) - { - return; - } + codeAddr += sz; + prevOpcode = opcode; + + prefixFlags = 0; } - // Get an entry to add to the pending list + return; +#undef _impResolveToken +} +#ifdef _PREFAST_ +#pragma warning(pop) +#endif - PendingDsc* dsc; +// Push a local/argument treeon the operand stack +void Compiler::impPushVar(GenTree* op, typeInfo tiRetVal) +{ + tiRetVal.NormaliseForStack(); + impPushOnStack(op, tiRetVal); +} - if (impPendingFree) +//------------------------------------------------------------------------ +// impCreateLocal: create a GT_LCL_VAR node to access a local that might need to be normalized on load +// +// Arguments: +// lclNum -- The index into lvaTable +// offset -- The offset to associate with the node +// +// Returns: +// The node +// +GenTreeLclVar* Compiler::impCreateLocalNode(unsigned lclNum DEBUGARG(IL_OFFSET offset)) +{ + var_types lclTyp; + + if (lvaTable[lclNum].lvNormalizeOnLoad()) { - // We can reuse one of the freed up dscs. - dsc = impPendingFree; - impPendingFree = dsc->pdNext; + lclTyp = lvaGetRealType(lclNum); } else { - // We have to create a new dsc - dsc = new (this, CMK_Unknown) PendingDsc; + lclTyp = lvaGetActualType(lclNum); } - dsc->pdBB = block; - dsc->pdSavedStack.ssDepth = verCurrentState.esStackDepth; - dsc->pdThisPtrInit = verCurrentState.thisInitialized; + return gtNewLclvNode(lclNum, lclTyp DEBUGARG(offset)); +} - // Save the stack trees for later +// Load a local/argument on the operand stack +// lclNum is an index into lvaTable *NOT* the arg/lcl index in the IL +void Compiler::impLoadVar(unsigned lclNum, IL_OFFSET offset) +{ + impPushVar(impCreateLocalNode(lclNum DEBUGARG(offset)), verMakeTypeInfoForLocal(lclNum)); +} - if (verCurrentState.esStackDepth) +// Load an argument on the operand stack +// Shared by the various CEE_LDARG opcodes +// ilArgNum is the argument index as specified in IL. +// It will be mapped to the correct lvaTable index +void Compiler::impLoadArg(unsigned ilArgNum, IL_OFFSET offset) +{ + Verify(ilArgNum < info.compILargsCount, "bad arg num"); + + if (compIsForInlining()) { - impSaveStackState(&dsc->pdSavedStack, false); - } + if (ilArgNum >= info.compArgsCount) + { + compInlineResult->NoteFatal(InlineObservation::CALLEE_BAD_ARGUMENT_NUMBER); + return; + } - // Add the entry to the pending list + impPushVar(impInlineFetchArg(ilArgNum, impInlineInfo->inlArgInfo, impInlineInfo->lclVarInfo), + impInlineInfo->lclVarInfo[ilArgNum].lclVerTypeInfo); + } + else + { + if (ilArgNum >= info.compArgsCount) + { + BADCODE("Bad IL"); + } - dsc->pdNext = impPendingList; - impPendingList = dsc; - impSetPendingBlockMember(block, 1); // And indicate that it's now a member of the set. + unsigned lclNum = compMapILargNum(ilArgNum); // account for possible hidden param - // Various assertions require us to now to consider the block as not imported (at least for - // the final time...) - block->bbFlags &= ~BBF_IMPORTED; + if (lclNum == info.compThisArg) + { + lclNum = lvaArg0Var; + } -#ifdef DEBUG - if (verbose && 0) - { - printf("Added PendingDsc - %08p for " FMT_BB "\n", dspPtr(dsc), block->bbNum); + impLoadVar(lclNum, offset); } -#endif } -/*****************************************************************************/ -// -// Ensures that "block" is a member of the list of BBs waiting to be imported, pushing it on the list if -// necessary (and ensures that it is a member of the set of BB's on the list, by setting its byte in -// impPendingBlockMembers). Does *NOT* change the existing "pre-state" of the block. - -void Compiler::impReimportBlockPending(BasicBlock* block) +// Load a local on the operand stack +// Shared by the various CEE_LDLOC opcodes +// ilLclNum is the local index as specified in IL. +// It will be mapped to the correct lvaTable index +void Compiler::impLoadLoc(unsigned ilLclNum, IL_OFFSET offset) { - JITDUMP("\nimpReimportBlockPending for " FMT_BB, block->bbNum); + if (compIsForInlining()) + { + if (ilLclNum >= info.compMethodInfo->locals.numArgs) + { + compInlineResult->NoteFatal(InlineObservation::CALLEE_BAD_LOCAL_NUMBER); + return; + } - assert(block->bbFlags & BBF_IMPORTED); + // Get the local type + var_types lclTyp = impInlineInfo->lclVarInfo[ilLclNum + impInlineInfo->argCnt].lclTypeInfo; - // OK, we must add to the pending list, if it's not already in it. - if (impGetPendingBlockMember(block) != 0) - { - return; - } + typeInfo tiRetVal = impInlineInfo->lclVarInfo[ilLclNum + impInlineInfo->argCnt].lclVerTypeInfo; - // Get an entry to add to the pending list + /* Have we allocated a temp for this local? */ - PendingDsc* dsc; + unsigned lclNum = impInlineFetchLocal(ilLclNum DEBUGARG("Inline ldloc first use temp")); - if (impPendingFree) - { - // We can reuse one of the freed up dscs. - dsc = impPendingFree; - impPendingFree = dsc->pdNext; - } - else - { - // We have to create a new dsc - dsc = new (this, CMK_ImpStack) PendingDsc; - } + // All vars of inlined methods should be !lvNormalizeOnLoad() - dsc->pdBB = block; + assert(!lvaTable[lclNum].lvNormalizeOnLoad()); + lclTyp = genActualType(lclTyp); - if (block->bbEntryState) - { - dsc->pdThisPtrInit = block->bbEntryState->thisInitialized; - dsc->pdSavedStack.ssDepth = block->bbEntryState->esStackDepth; - dsc->pdSavedStack.ssTrees = block->bbEntryState->esStack; + impPushVar(gtNewLclvNode(lclNum, lclTyp), tiRetVal); } else { - dsc->pdThisPtrInit = TIS_Bottom; - dsc->pdSavedStack.ssDepth = 0; - dsc->pdSavedStack.ssTrees = nullptr; - } - - // Add the entry to the pending list - - dsc->pdNext = impPendingList; - impPendingList = dsc; - impSetPendingBlockMember(block, 1); // And indicate that it's now a member of the set. + if (ilLclNum >= info.compMethodInfo->locals.numArgs) + { + BADCODE("Bad IL"); + } - // Various assertions require us to now to consider the block as not imported (at least for - // the final time...) - block->bbFlags &= ~BBF_IMPORTED; + unsigned lclNum = info.compArgsCount + ilLclNum; -#ifdef DEBUG - if (verbose && 0) - { - printf("Added PendingDsc - %08p for " FMT_BB "\n", dspPtr(dsc), block->bbNum); + impLoadVar(lclNum, offset); } -#endif } -void* Compiler::BlockListNode::operator new(size_t sz, Compiler* comp) -{ - if (comp->impBlockListNodeFreeList == nullptr) - { - return comp->getAllocator(CMK_BasicBlock).allocate(1); - } - else - { - BlockListNode* res = comp->impBlockListNodeFreeList; - comp->impBlockListNodeFreeList = res->m_next; - return res; - } -} +//------------------------------------------------------------------------ +// impAssignMultiRegTypeToVar: ensure calls that return structs in multiple +// registers return values to suitable temps. +// +// Arguments: +// op -- call returning a struct in registers +// hClass -- class handle for struct +// +// Returns: +// Tree with reference to struct local to use as call return value. -void Compiler::FreeBlockListNode(Compiler::BlockListNode* node) +GenTree* Compiler::impAssignMultiRegTypeToVar(GenTree* op, + CORINFO_CLASS_HANDLE hClass DEBUGARG(CorInfoCallConvExtension callConv)) { - node->m_next = impBlockListNodeFreeList; - impBlockListNodeFreeList = node; + unsigned tmpNum = lvaGrabTemp(true DEBUGARG("Return value temp for multireg return")); + impAssignTempGen(tmpNum, op, hClass, CHECK_SPILL_ALL); + GenTree* ret = gtNewLclvNode(tmpNum, lvaTable[tmpNum].lvType); + + // TODO-1stClassStructs: Handle constant propagation and CSE-ing of multireg returns. + ret->gtFlags |= GTF_DONT_CSE; + + assert(IsMultiRegReturnedType(hClass, callConv) || op->IsMultiRegNode()); + + // Set "lvIsMultiRegRet" to block promotion under "!lvaEnregMultiRegVars". + lvaTable[tmpNum].lvIsMultiRegRet = true; + + return ret; } -void Compiler::impWalkSpillCliqueFromPred(BasicBlock* block, SpillCliqueWalker* callback) +//------------------------------------------------------------------------ +// impReturnInstruction: import a return or an explicit tail call +// +// Arguments: +// prefixFlags -- active IL prefixes +// opcode -- [in, out] IL opcode +// +// Returns: +// True if import was successful (may fail for some inlinees) +// +bool Compiler::impReturnInstruction(int prefixFlags, OPCODE& opcode) { - bool toDo = true; + const bool isTailCall = (prefixFlags & PREFIX_TAILCALL) != 0; - noway_assert(!fgComputePredsDone); - if (!fgCheapPredsValid) +#ifdef DEBUG + // If we are importing an inlinee and have GC ref locals we always + // need to have a spill temp for the return value. This temp + // should have been set up in advance, over in fgFindBasicBlocks. + if (compIsForInlining() && impInlineInfo->HasGcRefLocals() && (info.compRetType != TYP_VOID)) { - fgComputeCheapPreds(); + assert(lvaInlineeReturnSpillTemp != BAD_VAR_NUM); } +#endif // DEBUG - BlockListNode* succCliqueToDo = nullptr; - BlockListNode* predCliqueToDo = new (this) BlockListNode(block); - while (toDo) + GenTree* op2 = nullptr; + GenTree* op1 = nullptr; + CORINFO_CLASS_HANDLE retClsHnd = nullptr; + + if (info.compRetType != TYP_VOID) { - toDo = false; - // Look at the successors of every member of the predecessor to-do list. - while (predCliqueToDo != nullptr) + StackEntry se = impPopStack(); + retClsHnd = se.seTypeInfo.GetClassHandle(); + op2 = se.val; + + if (!compIsForInlining()) { - BlockListNode* node = predCliqueToDo; - predCliqueToDo = node->m_next; - BasicBlock* blk = node->m_blk; - FreeBlockListNode(node); + impBashVarAddrsToI(op2); + op2 = impImplicitIorI4Cast(op2, info.compRetType); + op2 = impImplicitR4orR8Cast(op2, info.compRetType); - for (BasicBlock* const succ : blk->Succs()) + assertImp((genActualType(op2->TypeGet()) == genActualType(info.compRetType)) || + ((op2->TypeGet() == TYP_I_IMPL) && (info.compRetType == TYP_BYREF)) || + ((op2->TypeGet() == TYP_BYREF) && (info.compRetType == TYP_I_IMPL)) || + (varTypeIsFloating(op2->gtType) && varTypeIsFloating(info.compRetType)) || + (varTypeIsStruct(op2) && varTypeIsStruct(info.compRetType))); + +#ifdef DEBUG + if (!isTailCall && opts.compGcChecks && (info.compRetType == TYP_REF)) { - // If it's not already in the clique, add it, and also add it - // as a member of the successor "toDo" set. - if (impSpillCliqueGetMember(SpillCliqueSucc, succ) == 0) + // DDB 3483 : JIT Stress: early termination of GC ref's life time in exception code path + // VSW 440513: Incorrect gcinfo on the return value under COMPlus_JitGCChecks=1 for methods with + // one-return BB. + + assert(op2->gtType == TYP_REF); + + // confirm that the argument is a GC pointer (for debugging (GC stress)) + op2 = gtNewHelperCallNode(CORINFO_HELP_CHECK_OBJ, TYP_REF, op2); + + if (verbose) { - callback->Visit(SpillCliqueSucc, succ); - impSpillCliqueSetMember(SpillCliqueSucc, succ, 1); - succCliqueToDo = new (this) BlockListNode(succ, succCliqueToDo); - toDo = true; + printf("\ncompGcChecks tree:\n"); + gtDispTree(op2); } } +#endif } - // Look at the predecessors of every member of the successor to-do list. - while (succCliqueToDo != nullptr) + else { - BlockListNode* node = succCliqueToDo; - succCliqueToDo = node->m_next; - BasicBlock* blk = node->m_blk; - FreeBlockListNode(node); + if (verCurrentState.esStackDepth != 0) + { + assert(compIsForInlining()); + JITDUMP("CALLSITE_COMPILATION_ERROR: inlinee's stack is not empty."); + compInlineResult->NoteFatal(InlineObservation::CALLSITE_COMPILATION_ERROR); + return false; + } - for (BasicBlockList* pred = blk->bbCheapPreds; pred != nullptr; pred = pred->next) +#ifdef DEBUG + if (verbose) { - BasicBlock* predBlock = pred->block; - // If it's not already in the clique, add it, and also add it - // as a member of the predecessor "toDo" set. - if (impSpillCliqueGetMember(SpillCliquePred, predBlock) == 0) - { - callback->Visit(SpillCliquePred, predBlock); - impSpillCliqueSetMember(SpillCliquePred, predBlock, 1); - predCliqueToDo = new (this) BlockListNode(predBlock, predCliqueToDo); - toDo = true; - } + printf("\n\n Inlinee Return expression (before normalization) =>\n"); + gtDispTree(op2); } - } - } +#endif - // If this fails, it means we didn't walk the spill clique properly and somehow managed - // miss walking back to include the predecessor we started from. - // This most likely cause: missing or out of date bbPreds - assert(impSpillCliqueGetMember(SpillCliquePred, block) != 0); -} + InlineCandidateInfo* inlCandInfo = impInlineInfo->inlineCandidateInfo; + GenTreeRetExpr* inlRetExpr = inlCandInfo->retExpr; + // Make sure the type matches the original call. -void Compiler::SetSpillTempsBase::Visit(SpillCliqueDir predOrSucc, BasicBlock* blk) -{ - if (predOrSucc == SpillCliqueSucc) - { - assert(blk->bbStkTempsIn == NO_BASE_TMP); // Should not already be a member of a clique as a successor. - blk->bbStkTempsIn = m_baseTmp; - } - else - { - assert(predOrSucc == SpillCliquePred); - assert(blk->bbStkTempsOut == NO_BASE_TMP); // Should not already be a member of a clique as a predecessor. - blk->bbStkTempsOut = m_baseTmp; - } -} + var_types returnType = genActualType(op2->gtType); + var_types originalCallType = inlCandInfo->fncRetType; + if ((returnType != originalCallType) && (originalCallType == TYP_STRUCT)) + { + originalCallType = impNormStructType(inlCandInfo->methInfo.args.retTypeClass); + } -void Compiler::ReimportSpillClique::Visit(SpillCliqueDir predOrSucc, BasicBlock* blk) -{ - // For Preds we could be a little smarter and just find the existing store - // and re-type it/add a cast, but that is complicated and hopefully very rare, so - // just re-import the whole block (just like we do for successors) + if (returnType != originalCallType) + { + // Allow TYP_BYREF to be returned as TYP_I_IMPL and vice versa. + if (((returnType == TYP_BYREF) && (originalCallType == TYP_I_IMPL)) || + ((returnType == TYP_I_IMPL) && (originalCallType == TYP_BYREF))) + { + JITDUMP("Allowing return type mismatch: have %s, needed %s\n", varTypeName(returnType), + varTypeName(originalCallType)); + } + else + { + JITDUMP("Return type mismatch: have %s, needed %s\n", varTypeName(returnType), + varTypeName(originalCallType)); + compInlineResult->NoteFatal(InlineObservation::CALLSITE_RETURN_TYPE_MISMATCH); + return false; + } + } - if (((blk->bbFlags & BBF_IMPORTED) == 0) && (m_pComp->impGetPendingBlockMember(blk) == 0)) - { - // If we haven't imported this block and we're not going to (because it isn't on - // the pending list) then just ignore it for now. + // Below, we are going to set impInlineInfo->retExpr to the tree with the return + // expression. At this point, retExpr could already be set if there are multiple + // return blocks (meaning fgNeedReturnSpillTemp() == true) and one of + // the other blocks already set it. If there is only a single return block, + // retExpr shouldn't be set. However, this is not true if we reimport a block + // with a return. In that case, retExpr will be set, then the block will be + // reimported, but retExpr won't get cleared as part of setting the block to + // be reimported. The reimported retExpr value should be the same, so even if + // we don't unconditionally overwrite it, it shouldn't matter. + if (info.compRetNativeType != TYP_STRUCT) + { + // compRetNativeType is not TYP_STRUCT. + // This implies it could be either a scalar type or SIMD vector type or + // a struct type that can be normalized to a scalar type. - // This block has either never been imported (EntryState == NULL) or it failed - // verification. Neither state requires us to force it to be imported now. - assert((blk->bbEntryState == nullptr) || (blk->bbFlags & BBF_FAILED_VERIFICATION)); - return; - } + if (varTypeIsStruct(info.compRetType)) + { + noway_assert(info.compRetBuffArg == BAD_VAR_NUM); + // Handle calls with "fake" return buffers. + op2 = impFixupStructReturnType(op2); + } + else + { + // Do we have to normalize? + var_types fncRealRetType = JITtype2varType(info.compMethodInfo->args.retType); + // For RET_EXPR get the type info from the call. Regardless + // of whether it ends up inlined or not normalization will + // happen as part of that function's codegen. + GenTree* returnedTree = op2->OperIs(GT_RET_EXPR) ? op2->AsRetExpr()->gtInlineCandidate : op2; + if ((varTypeIsSmall(returnedTree->TypeGet()) || varTypeIsSmall(fncRealRetType)) && + fgCastNeeded(returnedTree, fncRealRetType)) + { + // Small-typed return values are normalized by the callee + op2 = gtNewCastNode(TYP_INT, op2, false, fncRealRetType); + } + } - // For successors we have a valid verCurrentState, so just mark them for reimport - // the 'normal' way - // Unlike predecessors, we *DO* need to reimport the current block because the - // initial import had the wrong entry state types. - // Similarly, blocks that are currently on the pending list, still need to call - // impImportBlockPending to fixup their entry state. - if (predOrSucc == SpillCliqueSucc) - { - m_pComp->impReimportMarkBlock(blk); + if (fgNeedReturnSpillTemp()) + { + assert(info.compRetNativeType != TYP_VOID && + (fgMoreThanOneReturnBlock() || impInlineInfo->HasGcRefLocals())); - // Set the current stack state to that of the blk->bbEntryState - m_pComp->verResetCurrentState(blk, &m_pComp->verCurrentState); - assert(m_pComp->verCurrentState.thisInitialized == blk->bbThisOnEntry()); + // If this method returns a ref type, track the actual types seen in the returns. + if (info.compRetType == TYP_REF) + { + bool isExact = false; + bool isNonNull = false; + CORINFO_CLASS_HANDLE returnClsHnd = gtGetClassHandle(op2, &isExact, &isNonNull); - m_pComp->impImportBlockPending(blk); - } - else if ((blk != m_pComp->compCurBB) && ((blk->bbFlags & BBF_IMPORTED) != 0)) - { - // As described above, we are only visiting predecessors so they can - // add the appropriate casts, since we have already done that for the current - // block, it does not need to be reimported. - // Nor do we need to reimport blocks that are still pending, but not yet - // imported. - // - // For predecessors, we have no state to seed the EntryState, so we just have - // to assume the existing one is correct. - // If the block is also a successor, it will get the EntryState properly - // updated when it is visited as a successor in the above "if" block. - assert(predOrSucc == SpillCliquePred); - m_pComp->impReimportBlockPending(blk); - } -} + if (inlRetExpr->gtSubstExpr == nullptr) + { + // This is the first return, so best known type is the type + // of this return value. + impInlineInfo->retExprClassHnd = returnClsHnd; + impInlineInfo->retExprClassHndIsExact = isExact; + } + else if (impInlineInfo->retExprClassHnd != returnClsHnd) + { + // This return site type differs from earlier seen sites, + // so reset the info and we'll fall back to using the method's + // declared return type for the return spill temp. + impInlineInfo->retExprClassHnd = nullptr; + impInlineInfo->retExprClassHndIsExact = false; + } + } -// Re-type the incoming lclVar nodes to match the varDsc. -void Compiler::impRetypeEntryStateTemps(BasicBlock* blk) -{ - if (blk->bbEntryState != nullptr) - { - EntryState* es = blk->bbEntryState; - for (unsigned level = 0; level < es->esStackDepth; level++) - { - GenTree* tree = es->esStack[level].val; - if ((tree->gtOper == GT_LCL_VAR) || (tree->gtOper == GT_LCL_FLD)) - { - es->esStack[level].val->gtType = lvaGetDesc(tree->AsLclVarCommon())->TypeGet(); - } - } - } -} + impAssignTempGen(lvaInlineeReturnSpillTemp, op2, se.seTypeInfo.GetClassHandle(), CHECK_SPILL_ALL); -unsigned Compiler::impGetSpillTmpBase(BasicBlock* block) -{ - if (block->bbStkTempsOut != NO_BASE_TMP) - { - return block->bbStkTempsOut; - } + var_types lclRetType = lvaGetDesc(lvaInlineeReturnSpillTemp)->lvType; + GenTree* tmpOp2 = gtNewLclvNode(lvaInlineeReturnSpillTemp, lclRetType); + op2 = tmpOp2; #ifdef DEBUG - if (verbose) - { - printf("\n*************** In impGetSpillTmpBase(" FMT_BB ")\n", block->bbNum); - } -#endif // DEBUG - - // Otherwise, choose one, and propagate to all members of the spill clique. - // Grab enough temps for the whole stack. - unsigned baseTmp = lvaGrabTemps(verCurrentState.esStackDepth DEBUGARG("IL Stack Entries")); - SetSpillTempsBase callback(baseTmp); - - // We do *NOT* need to reset the SpillClique*Members because a block can only be the predecessor - // to one spill clique, and similarly can only be the successor to one spill clique - impWalkSpillCliqueFromPred(block, &callback); - - return baseTmp; -} + if (inlRetExpr->gtSubstExpr != nullptr) + { + // Some other block(s) have seen the CEE_RET first. + // Better they spilled to the same temp. + assert(inlRetExpr->gtSubstExpr->gtOper == GT_LCL_VAR); + assert(inlRetExpr->gtSubstExpr->AsLclVarCommon()->GetLclNum() == + op2->AsLclVarCommon()->GetLclNum()); + } +#endif + } -void Compiler::impReimportSpillClique(BasicBlock* block) -{ #ifdef DEBUG - if (verbose) - { - printf("\n*************** In impReimportSpillClique(" FMT_BB ")\n", block->bbNum); - } -#endif // DEBUG - - // If we get here, it is because this block is already part of a spill clique - // and one predecessor had an outgoing live stack slot of type int, and this - // block has an outgoing live stack slot of type native int. - // We need to reset these before traversal because they have already been set - // by the previous walk to determine all the members of the spill clique. - impInlineRoot()->impSpillCliquePredMembers.Reset(); - impInlineRoot()->impSpillCliqueSuccMembers.Reset(); + if (verbose) + { + printf("\n\n Inlinee Return expression (after normalization) =>\n"); + gtDispTree(op2); + } +#endif - ReimportSpillClique callback(this); + // Report the return expression + inlRetExpr->gtSubstExpr = op2; + } + else + { + // compRetNativeType is TYP_STRUCT. + // This implies that struct return via RetBuf arg or multi-reg struct return. - impWalkSpillCliqueFromPred(block, &callback); -} + GenTreeCall* iciCall = impInlineInfo->iciCall->AsCall(); -// Set the pre-state of "block" (which should not have a pre-state allocated) to -// a copy of "srcState", cloning tree pointers as required. -void Compiler::verInitBBEntryState(BasicBlock* block, EntryState* srcState) -{ - if (srcState->esStackDepth == 0 && srcState->thisInitialized == TIS_Bottom) - { - block->bbEntryState = nullptr; - return; - } + // Assign the inlinee return into a spill temp. + if (fgNeedReturnSpillTemp()) + { + // in this case we have to insert multiple struct copies to the temp + // and the retexpr is just the temp. + assert(info.compRetNativeType != TYP_VOID); + assert(fgMoreThanOneReturnBlock() || impInlineInfo->HasGcRefLocals()); - block->bbEntryState = getAllocator(CMK_Unknown).allocate(1); + impAssignTempGen(lvaInlineeReturnSpillTemp, op2, se.seTypeInfo.GetClassHandle(), CHECK_SPILL_ALL); + } - // block->bbEntryState.esRefcount = 1; + if (compMethodReturnsMultiRegRetType()) + { + assert(!iciCall->ShouldHaveRetBufArg()); - block->bbEntryState->esStackDepth = srcState->esStackDepth; - block->bbEntryState->thisInitialized = TIS_Bottom; + if (fgNeedReturnSpillTemp()) + { + if (inlRetExpr->gtSubstExpr == nullptr) + { + // The inlinee compiler has figured out the type of the temp already. Use it here. + inlRetExpr->gtSubstExpr = + gtNewLclvNode(lvaInlineeReturnSpillTemp, lvaTable[lvaInlineeReturnSpillTemp].lvType); + } + } + else + { + inlRetExpr->gtSubstExpr = op2; + } + } + else // The struct was to be returned via a return buffer. + { + assert(iciCall->gtArgs.HasRetBuffer()); + GenTree* dest = gtCloneExpr(iciCall->gtArgs.GetRetBufferArg()->GetEarlyNode()); - if (srcState->esStackDepth > 0) - { - block->bbSetStack(new (this, CMK_Unknown) StackEntry[srcState->esStackDepth]); - unsigned stackSize = srcState->esStackDepth * sizeof(StackEntry); + if (fgNeedReturnSpillTemp()) + { + // If this is the first return we have seen set the retExpr. + if (inlRetExpr->gtSubstExpr == nullptr) + { + inlRetExpr->gtSubstExpr = + impAssignStructPtr(dest, gtNewLclvNode(lvaInlineeReturnSpillTemp, info.compRetType), + retClsHnd, CHECK_SPILL_ALL); + } + } + else + { + inlRetExpr->gtSubstExpr = impAssignStructPtr(dest, op2, retClsHnd, CHECK_SPILL_ALL); + } + } + } - memcpy(block->bbEntryState->esStack, srcState->esStack, stackSize); - for (unsigned level = 0; level < srcState->esStackDepth; level++) - { - GenTree* tree = srcState->esStack[level].val; - block->bbEntryState->esStack[level].val = gtCloneExpr(tree); + // If gtSubstExpr is an arbitrary tree then we may need to + // propagate mandatory "IR presence" flags (e.g. BBF_HAS_IDX_LEN) + // to the BB it ends up in. + inlRetExpr->gtSubstBB = fgNeedReturnSpillTemp() ? nullptr : compCurBB; } } - if (verTrackObjCtorInitState) + if (compIsForInlining()) { - verSetThisInit(block, srcState->thisInitialized); + return true; } - return; -} - -void Compiler::verSetThisInit(BasicBlock* block, ThisInitState tis) -{ - assert(tis != TIS_Bottom); // Precondition. - if (block->bbEntryState == nullptr) + if (info.compRetBuffArg != BAD_VAR_NUM) { - block->bbEntryState = new (this, CMK_Unknown) EntryState(); - } - - block->bbEntryState->thisInitialized = tis; -} + // Assign value to return buff (first param) + GenTree* retBuffAddr = + gtNewLclvNode(info.compRetBuffArg, TYP_BYREF DEBUGARG(impCurStmtDI.GetLocation().GetOffset())); -/* - * Resets the current state to the state at the start of the basic block - */ -void Compiler::verResetCurrentState(BasicBlock* block, EntryState* destState) -{ + op2 = impAssignStructPtr(retBuffAddr, op2, retClsHnd, CHECK_SPILL_ALL); + impAppendTree(op2, CHECK_SPILL_NONE, impCurStmtDI); - if (block->bbEntryState == nullptr) - { - destState->esStackDepth = 0; - destState->thisInitialized = TIS_Bottom; - return; + // There are cases where the address of the implicit RetBuf should be returned explicitly. + // + if (compMethodReturnsRetBufAddr()) + { + op1 = gtNewOperNode(GT_RETURN, TYP_BYREF, gtNewLclvNode(info.compRetBuffArg, TYP_BYREF)); + } + else + { + op1 = new (this, GT_RETURN) GenTreeOp(GT_RETURN, TYP_VOID); + } } - - destState->esStackDepth = block->bbEntryState->esStackDepth; - - if (destState->esStackDepth > 0) + else if (varTypeIsStruct(info.compRetType)) { - unsigned stackSize = destState->esStackDepth * sizeof(StackEntry); - - memcpy(destState->esStack, block->bbStackOnEntry(), stackSize); +#if !FEATURE_MULTIREG_RET + // For both ARM architectures the HFA native types are maintained as structs. + // Also on System V AMD64 the multireg structs returns are also left as structs. + noway_assert(info.compRetNativeType != TYP_STRUCT); +#endif + op2 = impFixupStructReturnType(op2); + op1 = gtNewOperNode(GT_RETURN, genActualType(info.compRetType), op2); } - - destState->thisInitialized = block->bbThisOnEntry(); - - return; -} - -ThisInitState BasicBlock::bbThisOnEntry() const -{ - return bbEntryState ? bbEntryState->thisInitialized : TIS_Bottom; -} - -unsigned BasicBlock::bbStackDepthOnEntry() const -{ - return (bbEntryState ? bbEntryState->esStackDepth : 0); -} - -void BasicBlock::bbSetStack(void* stackBuffer) -{ - assert(bbEntryState); - assert(stackBuffer); - bbEntryState->esStack = (StackEntry*)stackBuffer; -} - -StackEntry* BasicBlock::bbStackOnEntry() const -{ - assert(bbEntryState); - return bbEntryState->esStack; -} - -void Compiler::verInitCurrentState() -{ - verTrackObjCtorInitState = false; - verCurrentState.thisInitialized = TIS_Bottom; - - // initialize stack info - - verCurrentState.esStackDepth = 0; - assert(verCurrentState.esStack != nullptr); - - // copy current state to entry state of first BB - verInitBBEntryState(fgFirstBB, &verCurrentState); -} - -Compiler* Compiler::impInlineRoot() -{ - if (impInlineInfo == nullptr) + else if (info.compRetType != TYP_VOID) { - return this; + op1 = gtNewOperNode(GT_RETURN, genActualType(info.compRetType), op2); } else { - return impInlineInfo->InlineRoot; + op1 = new (this, GT_RETURN) GenTreeOp(GT_RETURN, TYP_VOID); } -} -BYTE Compiler::impSpillCliqueGetMember(SpillCliqueDir predOrSucc, BasicBlock* blk) -{ - if (predOrSucc == SpillCliquePred) - { - return impInlineRoot()->impSpillCliquePredMembers.Get(blk->bbInd()); - } - else + // We must have imported a tailcall and jumped to RET + if (isTailCall) { - assert(predOrSucc == SpillCliqueSucc); - return impInlineRoot()->impSpillCliqueSuccMembers.Get(blk->bbInd()); - } -} + assert(verCurrentState.esStackDepth == 0 && impOpcodeIsCallOpcode(opcode)); -void Compiler::impSpillCliqueSetMember(SpillCliqueDir predOrSucc, BasicBlock* blk, BYTE val) -{ - if (predOrSucc == SpillCliquePred) - { - impInlineRoot()->impSpillCliquePredMembers.Set(blk->bbInd(), val); - } - else - { - assert(predOrSucc == SpillCliqueSucc); - impInlineRoot()->impSpillCliqueSuccMembers.Set(blk->bbInd(), val); + opcode = CEE_RET; // To prevent trying to spill if CALL_SITE_BOUNDARIES + + // impImportCall() would have already appended TYP_VOID calls + if (info.compRetType == TYP_VOID) + { + return true; + } } + + impAppendTree(op1, CHECK_SPILL_NONE, impCurStmtDI); +#ifdef DEBUG + // Remember at which BC offset the tree was finished + impNoteLastILoffs(); +#endif + return true; } /***************************************************************************** - * - * Convert the instrs ("import") into our internal format (trees). The - * basic flowgraph has already been constructed and is passed in. + * Mark the block as unimported. + * Note that the caller is responsible for calling impImportBlockPending(), + * with the appropriate stack-state */ -void Compiler::impImport() +inline void Compiler::impReimportMarkBlock(BasicBlock* block) { #ifdef DEBUG - if (verbose) + if (verbose && (block->bbFlags & BBF_IMPORTED)) { - printf("*************** In impImport() for %s\n", info.compFullName); + printf("\n" FMT_BB " will be reimported\n", block->bbNum); } #endif - Compiler* inlineRoot = impInlineRoot(); - - if (info.compMaxStack <= SMALL_STACK_SIZE) - { - impStkSize = SMALL_STACK_SIZE; - } - else - { - impStkSize = info.compMaxStack; - } + block->bbFlags &= ~BBF_IMPORTED; +} - if (this == inlineRoot) - { - // Allocate the stack contents - verCurrentState.esStack = new (this, CMK_ImpStack) StackEntry[impStkSize]; - } - else - { - // This is the inlinee compiler, steal the stack from the inliner compiler - // (after ensuring that it is large enough). - if (inlineRoot->impStkSize < impStkSize) - { - inlineRoot->impStkSize = impStkSize; - inlineRoot->verCurrentState.esStack = new (this, CMK_ImpStack) StackEntry[impStkSize]; - } +/***************************************************************************** + * Mark the successors of the given block as unimported. + * Note that the caller is responsible for calling impImportBlockPending() + * for all the successors, with the appropriate stack-state. + */ - verCurrentState.esStack = inlineRoot->verCurrentState.esStack; +void Compiler::impReimportMarkSuccessors(BasicBlock* block) +{ + for (BasicBlock* const succBlock : block->Succs()) + { + impReimportMarkBlock(succBlock); } +} - // initialize the entry state at start of method - verInitCurrentState(); +/***************************************************************************** + * + * Filter wrapper to handle only passed in exception code + * from it). + */ - // Initialize stuff related to figuring "spill cliques" (see spec comment for impGetSpillTmpBase). - if (this == inlineRoot) // These are only used on the root of the inlining tree. +LONG FilterVerificationExceptions(PEXCEPTION_POINTERS pExceptionPointers, LPVOID lpvParam) +{ + if (pExceptionPointers->ExceptionRecord->ExceptionCode == SEH_VERIFICATION_EXCEPTION) { - // We have initialized these previously, but to size 0. Make them larger. - impPendingBlockMembers.Init(getAllocator(), fgBBNumMax * 2); - impSpillCliquePredMembers.Init(getAllocator(), fgBBNumMax * 2); - impSpillCliqueSuccMembers.Init(getAllocator(), fgBBNumMax * 2); + return EXCEPTION_EXECUTE_HANDLER; } - inlineRoot->impPendingBlockMembers.Reset(fgBBNumMax * 2); - inlineRoot->impSpillCliquePredMembers.Reset(fgBBNumMax * 2); - inlineRoot->impSpillCliqueSuccMembers.Reset(fgBBNumMax * 2); - impBlockListNodeFreeList = nullptr; -#ifdef DEBUG - impLastILoffsStmt = nullptr; - impNestedStackSpill = false; -#endif - impBoxTemp = BAD_VAR_NUM; + return EXCEPTION_CONTINUE_SEARCH; +} - impPendingList = impPendingFree = nullptr; +void Compiler::impVerifyEHBlock(BasicBlock* block, bool isTryStart) +{ + assert(block->hasTryIndex()); + assert(!compIsForInlining()); - // Skip leading internal blocks. - // These can arise from needing a leading scratch BB, from EH normalization, and from OSR entry redirects. - // - BasicBlock* entryBlock = fgFirstBB; + unsigned tryIndex = block->getTryIndex(); + EHblkDsc* HBtab = ehGetDsc(tryIndex); - while (entryBlock->bbFlags & BBF_INTERNAL) + if (isTryStart) { - JITDUMP("Marking leading BBF_INTERNAL block " FMT_BB " as BBF_IMPORTED\n", entryBlock->bbNum); - entryBlock->bbFlags |= BBF_IMPORTED; + assert(block->bbFlags & BBF_TRY_BEG); - if (entryBlock->bbJumpKind == BBJ_NONE) - { - entryBlock = entryBlock->bbNext; - } - else if (opts.IsOSR() && (entryBlock->bbJumpKind == BBJ_ALWAYS)) - { - entryBlock = entryBlock->bbJumpDest; - } - else + // The Stack must be empty + // + if (block->bbStkDepth != 0) { - assert(!"unexpected bbJumpKind in entry sequence"); + BADCODE("Evaluation stack must be empty on entry into a try block"); } } - // Note for OSR we'd like to be able to verify this block must be - // stack empty, but won't know that until we've imported...so instead - // we'll BADCODE out if we mess up. - // - // (the concern here is that the runtime asks us to OSR a - // different IL version than the one that matched the method that - // triggered OSR). This should not happen but I might have the - // IL versioning stuff wrong. + // Save the stack contents, we'll need to restore it later // - // TODO: we also currently expect this block to be a join point, - // which we should verify over when we find jump targets. - impImportBlockPending(entryBlock); - - /* Import blocks in the worker-list until there are no more */ + SavedStack blockState; + impSaveStackState(&blockState, false); - while (impPendingList) + while (HBtab != nullptr) { - /* Remove the entry at the front of the list */ - - PendingDsc* dsc = impPendingList; - impPendingList = impPendingList->pdNext; - impSetPendingBlockMember(dsc->pdBB, 0); - - /* Restore the stack state */ + // Recursively process the handler block, if we haven't already done so. + BasicBlock* hndBegBB = HBtab->ebdHndBeg; - verCurrentState.thisInitialized = dsc->pdThisPtrInit; - verCurrentState.esStackDepth = dsc->pdSavedStack.ssDepth; - if (verCurrentState.esStackDepth) + if (((hndBegBB->bbFlags & BBF_IMPORTED) == 0) && (impGetPendingBlockMember(hndBegBB) == 0)) { - impRestoreStackState(&dsc->pdSavedStack); - } + // Construct the proper verification stack state + // either empty or one that contains just + // the Exception Object that we are dealing with + // + verCurrentState.esStackDepth = 0; - /* Add the entry to the free list for reuse */ + if (handlerGetsXcptnObj(hndBegBB->bbCatchTyp)) + { + CORINFO_CLASS_HANDLE clsHnd; - dsc->pdNext = impPendingFree; - impPendingFree = dsc; + if (HBtab->HasFilter()) + { + clsHnd = impGetObjectClass(); + } + else + { + CORINFO_RESOLVED_TOKEN resolvedToken; - /* Now import the block */ + resolvedToken.tokenContext = impTokenLookupContextHandle; + resolvedToken.tokenScope = info.compScopeHnd; + resolvedToken.token = HBtab->ebdTyp; + resolvedToken.tokenType = CORINFO_TOKENKIND_Class; + info.compCompHnd->resolveToken(&resolvedToken); - if (dsc->pdBB->bbFlags & BBF_FAILED_VERIFICATION) - { - verConvertBBToThrowVerificationException(dsc->pdBB DEBUGARG(true)); - impEndTreeList(dsc->pdBB); - } - else - { - impImportBlock(dsc->pdBB); + clsHnd = resolvedToken.hClass; + } - if (compDonotInline()) - { - return; - } - if (compIsForImportOnly()) - { - return; + // push catch arg the stack, spill to a temp if necessary + // Note: can update HBtab->ebdHndBeg! + hndBegBB = impPushCatchArgOnStack(hndBegBB, clsHnd, false); } + + // Queue up the handler for importing + // + impImportBlockPending(hndBegBB); } - } -#ifdef DEBUG - if (verbose && info.compXcptnsCount) - { - printf("\nAfter impImport() added block for try,catch,finally"); - fgDispBasicBlocks(); - printf("\n"); - } + // Process the filter block, if we haven't already done so. + if (HBtab->HasFilter()) + { + /* @VERIFICATION : Ideally the end of filter state should get + propagated to the catch handler, this is an incompleteness, + but is not a security/compliance issue, since the only + interesting state is the 'thisInit' state. + */ - // Used in impImportBlockPending() for STRESS_CHK_REIMPORT - for (BasicBlock* const block : Blocks()) - { - block->bbFlags &= ~BBF_VISITED; - } -#endif -} + BasicBlock* filterBB = HBtab->ebdFilter; -// Checks if a typeinfo (usually stored in the type stack) is a struct. -// The invariant here is that if it's not a ref or a method and has a class handle -// it's a valuetype -bool Compiler::impIsValueType(typeInfo* pTypeInfo) -{ - if (pTypeInfo && pTypeInfo->IsValueClassWithClsHnd()) - { - return true; - } - else - { - return false; - } -} + if (((filterBB->bbFlags & BBF_IMPORTED) == 0) && (impGetPendingBlockMember(filterBB) == 0)) + { + verCurrentState.esStackDepth = 0; -//------------------------------------------------------------------------ -// impIsInvariant: check if a tree (created during import) is invariant. -// -// Arguments: -// tree -- The tree -// -// Returns: -// true if it is invariant -// -// Remarks: -// This is a variant of GenTree::IsInvariant that is more suitable for use -// during import. Unlike that function, this one handles GT_FIELD nodes. -// -bool Compiler::impIsInvariant(const GenTree* tree) -{ - return tree->OperIsConst() || impIsAddressInLocal(tree); -} + // push catch arg the stack, spill to a temp if necessary + // Note: can update HBtab->ebdFilter! + const bool isSingleBlockFilter = (filterBB->bbNext == hndBegBB); + filterBB = impPushCatchArgOnStack(filterBB, impGetObjectClass(), isSingleBlockFilter); -//------------------------------------------------------------------------ -// impIsAddressInLocal: -// Check to see if the tree is the address of a local or -// the address of a field in a local. -// Arguments: -// tree -- The tree -// lclVarTreeOut -- [out] the local that this points into -// -// Returns: -// true if it points into a local -// -// Remarks: -// This is a variant of GenTree::IsLocalAddrExpr that is more suitable for -// use during import. Unlike that function, this one handles GT_FIELD nodes. -// -bool Compiler::impIsAddressInLocal(const GenTree* tree, GenTree** lclVarTreeOut) -{ - if (tree->gtOper != GT_ADDR) - { - return false; - } + impImportBlockPending(filterBB); + } + } - GenTree* op = tree->AsOp()->gtOp1; - while (op->gtOper == GT_FIELD) - { - op = op->AsField()->GetFldObj(); - if (op && op->gtOper == GT_ADDR) // Skip static fields where op will be NULL. + // Now process our enclosing try index (if any) + // + tryIndex = HBtab->ebdEnclosingTryIndex; + if (tryIndex == EHblkDsc::NO_ENCLOSING_INDEX) { - op = op->AsOp()->gtOp1; + HBtab = nullptr; } else { - return false; + HBtab = ehGetDsc(tryIndex); } } - if (op->gtOper == GT_LCL_VAR) + // Restore the stack contents + impRestoreStackState(&blockState); +} + +//*************************************************************** +// Import the instructions for the given basic block. Perform +// verification, throwing an exception on failure. Push any successor blocks that are enabled for the first +// time, or whose verification pre-state is changed. + +#ifdef _PREFAST_ +#pragma warning(push) +#pragma warning(disable : 21000) // Suppress PREFast warning about overly large function +#endif +void Compiler::impImportBlock(BasicBlock* block) +{ + // BBF_INTERNAL blocks only exist during importation due to EH canonicalization. We need to + // handle them specially. In particular, there is no IL to import for them, but we do need + // to mark them as imported and put their successors on the pending import list. + if (block->bbFlags & BBF_INTERNAL) { - if (lclVarTreeOut != nullptr) + JITDUMP("Marking BBF_INTERNAL block " FMT_BB " as BBF_IMPORTED\n", block->bbNum); + block->bbFlags |= BBF_IMPORTED; + + for (BasicBlock* const succBlock : block->Succs()) { - *lclVarTreeOut = op; + impImportBlockPending(succBlock); } - return true; - } - else - { - return false; + + return; } -} -//------------------------------------------------------------------------ -// impMakeDiscretionaryInlineObservations: make observations that help -// determine the profitability of a discretionary inline -// -// Arguments: -// pInlineInfo -- InlineInfo for the inline, or null for the prejit root -// inlineResult -- InlineResult accumulating information about this inline -// -// Notes: -// If inlining or prejitting the root, this method also makes -// various observations about the method that factor into inline -// decisions. It sets `compNativeSizeEstimate` as a side effect. + bool markImport; -void Compiler::impMakeDiscretionaryInlineObservations(InlineInfo* pInlineInfo, InlineResult* inlineResult) -{ - assert((pInlineInfo != nullptr && compIsForInlining()) || // Perform the actual inlining. - (pInlineInfo == nullptr && !compIsForInlining()) // Calculate the static inlining hint for ngen. - ); + assert(block); - // If we're really inlining, we should just have one result in play. - assert((pInlineInfo == nullptr) || (inlineResult == pInlineInfo->inlineResult)); + /* Make the block globally available */ - // If this is a "forceinline" method, the JIT probably shouldn't have gone - // to the trouble of estimating the native code size. Even if it did, it - // shouldn't be relying on the result of this method. - assert(inlineResult->GetObservation() == InlineObservation::CALLEE_IS_DISCRETIONARY_INLINE); + compCurBB = block; - // Note if the caller contains NEWOBJ or NEWARR. - Compiler* rootCompiler = impInlineRoot(); +#ifdef DEBUG + /* Initialize the debug variables */ + impCurOpcName = "unknown"; + impCurOpcOffs = block->bbCodeOffs; +#endif - if ((rootCompiler->optMethodFlags & OMF_HAS_NEWARRAY) != 0) - { - inlineResult->Note(InlineObservation::CALLER_HAS_NEWARRAY); - } + /* Set the current stack state to the merged result */ + verResetCurrentState(block, &verCurrentState); - if ((rootCompiler->optMethodFlags & OMF_HAS_NEWOBJ) != 0) + /* Now walk the code and import the IL into GenTrees */ + + struct FilterVerificationExceptionsParam { - inlineResult->Note(InlineObservation::CALLER_HAS_NEWOBJ); - } + Compiler* pThis; + BasicBlock* block; + }; + FilterVerificationExceptionsParam param; - bool calleeIsStatic = (info.compFlags & CORINFO_FLG_STATIC) != 0; - bool isSpecialMethod = (info.compFlags & CORINFO_FLG_CONSTRUCTOR) != 0; + param.pThis = this; + param.block = block; - if (isSpecialMethod) + PAL_TRY(FilterVerificationExceptionsParam*, pParam, ¶m) { - if (calleeIsStatic) + /* @VERIFICATION : For now, the only state propagation from try + to it's handler is "thisInit" state (stack is empty at start of try). + In general, for state that we track in verification, we need to + model the possibility that an exception might happen at any IL + instruction, so we really need to merge all states that obtain + between IL instructions in a try block into the start states of + all handlers. + + However we do not allow the 'this' pointer to be uninitialized when + entering most kinds try regions (only try/fault are allowed to have + an uninitialized this pointer on entry to the try) + + Fortunately, the stack is thrown away when an exception + leads to a handler, so we don't have to worry about that. + We DO, however, have to worry about the "thisInit" state. + But only for the try/fault case. + + The only allowed transition is from TIS_Uninit to TIS_Init. + + So for a try/fault region for the fault handler block + we will merge the start state of the try begin + and the post-state of each block that is part of this try region + */ + + // merge the start state of the try begin + // + if (pParam->block->bbFlags & BBF_TRY_BEG) { - inlineResult->Note(InlineObservation::CALLEE_IS_CLASS_CTOR); + pParam->pThis->impVerifyEHBlock(pParam->block, true); } - else + + pParam->pThis->impImportBlockCode(pParam->block); + + // As discussed above: + // merge the post-state of each block that is part of this try region + // + if (pParam->block->hasTryIndex()) { - inlineResult->Note(InlineObservation::CALLEE_IS_INSTANCE_CTOR); + pParam->pThis->impVerifyEHBlock(pParam->block, false); } } - else if (!calleeIsStatic) + PAL_EXCEPT_FILTER(FilterVerificationExceptions) { - // Callee is an instance method. - // - // Check if the callee has the same 'this' as the root. - if (pInlineInfo != nullptr) - { - GenTree* thisArg = pInlineInfo->iciCall->AsCall()->gtArgs.GetThisArg()->GetNode(); - assert(thisArg); - bool isSameThis = impIsThis(thisArg); - inlineResult->NoteBool(InlineObservation::CALLSITE_IS_SAME_THIS, isSameThis); - } + verHandleVerificationFailure(block DEBUGARG(false)); } + PAL_ENDTRY - bool callsiteIsGeneric = (rootCompiler->info.compMethodInfo->args.sigInst.methInstCount != 0) || - (rootCompiler->info.compMethodInfo->args.sigInst.classInstCount != 0); - - bool calleeIsGeneric = (info.compMethodInfo->args.sigInst.methInstCount != 0) || - (info.compMethodInfo->args.sigInst.classInstCount != 0); - - if (!callsiteIsGeneric && calleeIsGeneric) + if (compDonotInline()) { - inlineResult->Note(InlineObservation::CALLSITE_NONGENERIC_CALLS_GENERIC); + return; } - // Inspect callee's arguments (and the actual values at the callsite for them) - CORINFO_SIG_INFO sig = info.compMethodInfo->args; - CORINFO_ARG_LIST_HANDLE sigArg = sig.args; + assert(!compDonotInline()); + + markImport = false; - CallArg* argUse = pInlineInfo == nullptr ? nullptr : &*pInlineInfo->iciCall->AsCall()->gtArgs.Args().begin(); +SPILLSTACK: - for (unsigned i = 0; i < info.compMethodInfo->args.numArgs; i++) + unsigned baseTmp = NO_BASE_TMP; // input temps assigned to successor blocks + bool reimportSpillClique = false; + BasicBlock* tgtBlock = nullptr; + + /* If the stack is non-empty, we might have to spill its contents */ + + if (verCurrentState.esStackDepth != 0) { - if ((argUse != nullptr) && (argUse->GetWellKnownArg() == WellKnownArg::ThisPointer)) - { - argUse = argUse->GetNext(); - } + impBoxTemp = BAD_VAR_NUM; // if a box temp is used in a block that leaves something + // on the stack, its lifetime is hard to determine, simply + // don't reuse such temps. - CORINFO_CLASS_HANDLE sigClass; - CorInfoType corType = strip(info.compCompHnd->getArgType(&sig, sigArg, &sigClass)); - GenTree* argNode = argUse == nullptr ? nullptr : argUse->GetEarlyNode(); + Statement* addStmt = nullptr; - if (corType == CORINFO_TYPE_CLASS) - { - sigClass = info.compCompHnd->getArgClass(&sig, sigArg); - } - else if (corType == CORINFO_TYPE_VALUECLASS) - { - inlineResult->Note(InlineObservation::CALLEE_ARG_STRUCT); - } - else if (corType == CORINFO_TYPE_BYREF) - { - sigClass = info.compCompHnd->getArgClass(&sig, sigArg); - corType = info.compCompHnd->getChildType(sigClass, &sigClass); - } + /* Do the successors of 'block' have any other predecessors ? + We do not want to do some of the optimizations related to multiRef + if we can reimport blocks */ - if (argNode != nullptr) + unsigned multRef = impCanReimport ? unsigned(~0) : 0; + + switch (block->bbJumpKind) { - bool isExact = false; - bool isNonNull = false; - CORINFO_CLASS_HANDLE argCls = gtGetClassHandle(argNode, &isExact, &isNonNull); - if (argCls != nullptr) - { - const bool isArgValueType = eeIsValueClass(argCls); - // Exact class of the arg is known - if (isExact && !isArgValueType) - { - inlineResult->Note(InlineObservation::CALLSITE_ARG_EXACT_CLS); - if ((argCls != sigClass) && (sigClass != nullptr)) - { - // .. but the signature accepts a less concrete type. - inlineResult->Note(InlineObservation::CALLSITE_ARG_EXACT_CLS_SIG_IS_NOT); - } - } - // Arg is a reference type in the signature and a boxed value type was passed. - else if (isArgValueType && (corType == CORINFO_TYPE_CLASS)) + case BBJ_COND: + + addStmt = impExtractLastStmt(); + + assert(addStmt->GetRootNode()->gtOper == GT_JTRUE); + + /* Note if the next block has more than one ancestor */ + + multRef |= block->bbNext->bbRefs; + + /* Does the next block have temps assigned? */ + + baseTmp = block->bbNext->bbStkTempsIn; + tgtBlock = block->bbNext; + + if (baseTmp != NO_BASE_TMP) { - inlineResult->Note(InlineObservation::CALLSITE_ARG_BOXED); + break; } - } - if (argNode->OperIsConst()) - { - inlineResult->Note(InlineObservation::CALLSITE_ARG_CONST); - } - argUse = argUse->GetNext(); - } - sigArg = info.compCompHnd->getArgNext(sigArg); - } + /* Try the target of the jump then */ - // Note if the callee's return type is a value type - if (info.compMethodInfo->args.retType == CORINFO_TYPE_VALUECLASS) - { - inlineResult->Note(InlineObservation::CALLEE_RETURNS_STRUCT); - } + multRef |= block->bbJumpDest->bbRefs; + baseTmp = block->bbJumpDest->bbStkTempsIn; + tgtBlock = block->bbJumpDest; + break; + + case BBJ_ALWAYS: + multRef |= block->bbJumpDest->bbRefs; + baseTmp = block->bbJumpDest->bbStkTempsIn; + tgtBlock = block->bbJumpDest; + break; + + case BBJ_NONE: + multRef |= block->bbNext->bbRefs; + baseTmp = block->bbNext->bbStkTempsIn; + tgtBlock = block->bbNext; + break; + + case BBJ_SWITCH: + addStmt = impExtractLastStmt(); + assert(addStmt->GetRootNode()->gtOper == GT_SWITCH); + + for (BasicBlock* const tgtBlock : block->SwitchTargets()) + { + multRef |= tgtBlock->bbRefs; + + // Thanks to spill cliques, we should have assigned all or none + assert((baseTmp == NO_BASE_TMP) || (baseTmp == tgtBlock->bbStkTempsIn)); + baseTmp = tgtBlock->bbStkTempsIn; + if (multRef > 1) + { + break; + } + } + break; - // Note if the callee's class is a promotable struct - if ((info.compClassAttr & CORINFO_FLG_VALUECLASS) != 0) - { - assert(structPromotionHelper != nullptr); - if (structPromotionHelper->CanPromoteStructType(info.compClassHnd)) - { - inlineResult->Note(InlineObservation::CALLEE_CLASS_PROMOTABLE); - } - inlineResult->Note(InlineObservation::CALLEE_CLASS_VALUETYPE); - } + case BBJ_CALLFINALLY: + case BBJ_EHCATCHRET: + case BBJ_RETURN: + case BBJ_EHFINALLYRET: + case BBJ_EHFILTERRET: + case BBJ_THROW: + BADCODE("can't have 'unreached' end of BB with non-empty stack"); + break; -#ifdef FEATURE_SIMD + default: + noway_assert(!"Unexpected bbJumpKind"); + break; + } - // Note if this method is has SIMD args or return value - if (pInlineInfo != nullptr && pInlineInfo->hasSIMDTypeArgLocalOrReturn) - { - inlineResult->Note(InlineObservation::CALLEE_HAS_SIMD); - } + assert(multRef >= 1); -#endif // FEATURE_SIMD + /* Do we have a base temp number? */ - // Roughly classify callsite frequency. - InlineCallsiteFrequency frequency = InlineCallsiteFrequency::UNUSED; + bool newTemps = (baseTmp == NO_BASE_TMP); - // If this is a prejit root, or a maximally hot block... - if ((pInlineInfo == nullptr) || (pInlineInfo->iciBlock->isMaxBBWeight())) - { - frequency = InlineCallsiteFrequency::HOT; - } - // No training data. Look for loop-like things. - // We consider a recursive call loop-like. Do not give the inlining boost to the method itself. - // However, give it to things nearby. - else if ((pInlineInfo->iciBlock->bbFlags & BBF_BACKWARD_JUMP) && - (pInlineInfo->fncHandle != pInlineInfo->inlineCandidateInfo->ilCallerHandle)) - { - frequency = InlineCallsiteFrequency::LOOP; - } - else if (pInlineInfo->iciBlock->hasProfileWeight() && (pInlineInfo->iciBlock->bbWeight > BB_ZERO_WEIGHT)) - { - frequency = InlineCallsiteFrequency::WARM; - } - // Now modify the multiplier based on where we're called from. - else if (pInlineInfo->iciBlock->isRunRarely() || ((info.compFlags & FLG_CCTOR) == FLG_CCTOR)) - { - frequency = InlineCallsiteFrequency::RARE; - } - else - { - frequency = InlineCallsiteFrequency::BORING; - } + if (newTemps) + { + /* Grab enough temps for the whole stack */ + baseTmp = impGetSpillTmpBase(block); + } - // Also capture the block weight of the call site. - // - // In the prejit root case, assume at runtime there might be a hot call site - // for this method, so we won't prematurely conclude this method should never - // be inlined. - // - weight_t weight = 0; + /* Spill all stack entries into temps */ + unsigned level, tempNum; - if (pInlineInfo != nullptr) - { - weight = pInlineInfo->iciBlock->bbWeight; - } - else - { - const weight_t prejitHotCallerWeight = 1000000.0; - weight = prejitHotCallerWeight; - } + JITDUMP("\nSpilling stack entries into temps\n"); + for (level = 0, tempNum = baseTmp; level < verCurrentState.esStackDepth; level++, tempNum++) + { + GenTree* tree = verCurrentState.esStack[level].val; - inlineResult->NoteInt(InlineObservation::CALLSITE_FREQUENCY, static_cast(frequency)); - inlineResult->NoteInt(InlineObservation::CALLSITE_WEIGHT, (int)(weight)); + /* VC generates code where it pushes a byref from one branch, and an int (ldc.i4 0) from + the other. This should merge to a byref in unverifiable code. + However, if the branch which leaves the TYP_I_IMPL on the stack is imported first, the + successor would be imported assuming there was a TYP_I_IMPL on + the stack. Thus the value would not get GC-tracked. Hence, + change the temp to TYP_BYREF and reimport the successors. + Note: We should only allow this in unverifiable code. + */ + if (tree->gtType == TYP_BYREF && lvaTable[tempNum].lvType == TYP_I_IMPL) + { + lvaTable[tempNum].lvType = TYP_BYREF; + impReimportMarkSuccessors(block); + markImport = true; + } - bool hasProfile = false; - double profileFreq = 0.0; +#ifdef TARGET_64BIT + if (genActualType(tree->gtType) == TYP_I_IMPL && lvaTable[tempNum].lvType == TYP_INT) + { + // Some other block in the spill clique set this to "int", but now we have "native int". + // Change the type and go back to re-import any blocks that used the wrong type. + lvaTable[tempNum].lvType = TYP_I_IMPL; + reimportSpillClique = true; + } + else if (genActualType(tree->gtType) == TYP_INT && lvaTable[tempNum].lvType == TYP_I_IMPL) + { + // Spill clique has decided this should be "native int", but this block only pushes an "int". + // Insert a sign-extension to "native int" so we match the clique. + verCurrentState.esStack[level].val = gtNewCastNode(TYP_I_IMPL, tree, false, TYP_I_IMPL); + } - // If the call site has profile data, report the relative frequency of the site. - // - if ((pInlineInfo != nullptr) && rootCompiler->fgHaveSufficientProfileData()) - { - const weight_t callSiteWeight = pInlineInfo->iciBlock->bbWeight; - const weight_t entryWeight = rootCompiler->fgFirstBB->bbWeight; - profileFreq = fgProfileWeightsEqual(entryWeight, 0.0) ? 0.0 : callSiteWeight / entryWeight; - hasProfile = true; + // Consider the case where one branch left a 'byref' on the stack and the other leaves + // an 'int'. On 32-bit, this is allowed (in non-verifiable code) since they are the same + // size. JIT64 managed to make this work on 64-bit. For compatibility, we support JIT64 + // behavior instead of asserting and then generating bad code (where we save/restore the + // low 32 bits of a byref pointer to an 'int' sized local). If the 'int' side has been + // imported already, we need to change the type of the local and reimport the spill clique. + // If the 'byref' side has imported, we insert a cast from int to 'native int' to match + // the 'byref' size. + if (genActualType(tree->gtType) == TYP_BYREF && lvaTable[tempNum].lvType == TYP_INT) + { + // Some other block in the spill clique set this to "int", but now we have "byref". + // Change the type and go back to re-import any blocks that used the wrong type. + lvaTable[tempNum].lvType = TYP_BYREF; + reimportSpillClique = true; + } + else if (genActualType(tree->gtType) == TYP_INT && lvaTable[tempNum].lvType == TYP_BYREF) + { + // Spill clique has decided this should be "byref", but this block only pushes an "int". + // Insert a sign-extension to "native int" so we match the clique size. + verCurrentState.esStack[level].val = gtNewCastNode(TYP_I_IMPL, tree, false, TYP_I_IMPL); + } - assert(callSiteWeight >= 0); - assert(entryWeight >= 0); - } - else if (pInlineInfo == nullptr) - { - // Simulate a hot callsite for PrejitRoot mode. - hasProfile = true; - profileFreq = 1.0; - } +#endif // TARGET_64BIT - inlineResult->NoteBool(InlineObservation::CALLSITE_HAS_PROFILE, hasProfile); - inlineResult->NoteDouble(InlineObservation::CALLSITE_PROFILE_FREQUENCY, profileFreq); -} + if (tree->gtType == TYP_DOUBLE && lvaTable[tempNum].lvType == TYP_FLOAT) + { + // Some other block in the spill clique set this to "float", but now we have "double". + // Change the type and go back to re-import any blocks that used the wrong type. + lvaTable[tempNum].lvType = TYP_DOUBLE; + reimportSpillClique = true; + } + else if (tree->gtType == TYP_FLOAT && lvaTable[tempNum].lvType == TYP_DOUBLE) + { + // Spill clique has decided this should be "double", but this block only pushes a "float". + // Insert a cast to "double" so we match the clique. + verCurrentState.esStack[level].val = gtNewCastNode(TYP_DOUBLE, tree, false, TYP_DOUBLE); + } -/***************************************************************************** - This method makes STATIC inlining decision based on the IL code. - It should not make any inlining decision based on the context. - If forceInline is true, then the inlining decision should not depend on - performance heuristics (code size, etc.). - */ + /* If addStmt has a reference to tempNum (can only happen if we + are spilling to the temps already used by a previous block), + we need to spill addStmt */ -void Compiler::impCanInlineIL(CORINFO_METHOD_HANDLE fncHandle, - CORINFO_METHOD_INFO* methInfo, - bool forceInline, - InlineResult* inlineResult) -{ - unsigned codeSize = methInfo->ILCodeSize; + if (addStmt != nullptr && !newTemps && gtHasRef(addStmt->GetRootNode(), tempNum)) + { + GenTree* addTree = addStmt->GetRootNode(); - // We shouldn't have made up our minds yet... - assert(!inlineResult->IsDecided()); + if (addTree->gtOper == GT_JTRUE) + { + GenTree* relOp = addTree->AsOp()->gtOp1; + assert(relOp->OperIsCompare()); - if (methInfo->EHcount) - { - inlineResult->NoteFatal(InlineObservation::CALLEE_HAS_EH); - return; - } + var_types type = genActualType(relOp->AsOp()->gtOp1->TypeGet()); - if ((methInfo->ILCode == nullptr) || (codeSize == 0)) - { - inlineResult->NoteFatal(InlineObservation::CALLEE_HAS_NO_BODY); - return; - } + if (gtHasRef(relOp->AsOp()->gtOp1, tempNum)) + { + unsigned temp = lvaGrabTemp(true DEBUGARG("spill addStmt JTRUE ref Op1")); + impAssignTempGen(temp, relOp->AsOp()->gtOp1, level); + type = genActualType(lvaTable[temp].TypeGet()); + relOp->AsOp()->gtOp1 = gtNewLclvNode(temp, type); + } - // For now we don't inline varargs (import code can't handle it) + if (gtHasRef(relOp->AsOp()->gtOp2, tempNum)) + { + unsigned temp = lvaGrabTemp(true DEBUGARG("spill addStmt JTRUE ref Op2")); + impAssignTempGen(temp, relOp->AsOp()->gtOp2, level); + type = genActualType(lvaTable[temp].TypeGet()); + relOp->AsOp()->gtOp2 = gtNewLclvNode(temp, type); + } + } + else + { + assert(addTree->gtOper == GT_SWITCH && genActualTypeIsIntOrI(addTree->AsOp()->gtOp1->TypeGet())); - if (methInfo->args.isVarArg()) - { - inlineResult->NoteFatal(InlineObservation::CALLEE_HAS_MANAGED_VARARGS); - return; - } + unsigned temp = lvaGrabTemp(true DEBUGARG("spill addStmt SWITCH")); + impAssignTempGen(temp, addTree->AsOp()->gtOp1, level); + addTree->AsOp()->gtOp1 = gtNewLclvNode(temp, genActualType(addTree->AsOp()->gtOp1->TypeGet())); + } + } - // Reject if it has too many locals. - // This is currently an implementation limit due to fixed-size arrays in the - // inline info, rather than a performance heuristic. + /* Spill the stack entry, and replace with the temp */ - inlineResult->NoteInt(InlineObservation::CALLEE_NUMBER_OF_LOCALS, methInfo->locals.numArgs); + if (!impSpillStackEntry(level, tempNum +#ifdef DEBUG + , + true, "Spill Stack Entry" +#endif + )) + { + if (markImport) + { + BADCODE("bad stack state"); + } - if (methInfo->locals.numArgs > MAX_INL_LCLS) - { - inlineResult->NoteFatal(InlineObservation::CALLEE_TOO_MANY_LOCALS); - return; - } + // Oops. Something went wrong when spilling. Bad code. + verHandleVerificationFailure(block DEBUGARG(true)); - // Make sure there aren't too many arguments. - // This is currently an implementation limit due to fixed-size arrays in the - // inline info, rather than a performance heuristic. + goto SPILLSTACK; + } + } - inlineResult->NoteInt(InlineObservation::CALLEE_NUMBER_OF_ARGUMENTS, methInfo->args.numArgs); + /* Put back the 'jtrue'/'switch' if we removed it earlier */ - if (methInfo->args.numArgs > MAX_INL_ARGS) - { - inlineResult->NoteFatal(InlineObservation::CALLEE_TOO_MANY_ARGUMENTS); - return; + if (addStmt != nullptr) + { + impAppendStmt(addStmt, CHECK_SPILL_NONE); + } } - // Note force inline state + // Some of the append/spill logic works on compCurBB - inlineResult->NoteBool(InlineObservation::CALLEE_IS_FORCE_INLINE, forceInline); + assert(compCurBB == block); - // Note IL code size + /* Save the tree list in the block */ + impEndTreeList(block); - inlineResult->NoteInt(InlineObservation::CALLEE_IL_CODE_SIZE, codeSize); + // impEndTreeList sets BBF_IMPORTED on the block + // We do *NOT* want to set it later than this because + // impReimportSpillClique might clear it if this block is both a + // predecessor and successor in the current spill clique + assert(block->bbFlags & BBF_IMPORTED); - if (inlineResult->IsFailure()) + // If we had a int/native int, or float/double collision, we need to re-import + if (reimportSpillClique) { - return; - } - - // Make sure maxstack is not too big - - inlineResult->NoteInt(InlineObservation::CALLEE_MAXSTACK, methInfo->maxStack); + // This will re-import all the successors of block (as well as each of their predecessors) + impReimportSpillClique(block); - if (inlineResult->IsFailure()) + // For blocks that haven't been imported yet, we still need to mark them as pending import. + for (BasicBlock* const succ : block->Succs()) + { + if ((succ->bbFlags & BBF_IMPORTED) == 0) + { + impImportBlockPending(succ); + } + } + } + else // the normal case { - return; + // otherwise just import the successors of block + + /* Does this block jump to any other blocks? */ + for (BasicBlock* const succ : block->Succs()) + { + impImportBlockPending(succ); + } } } +#ifdef _PREFAST_ +#pragma warning(pop) +#endif -/***************************************************************************** - */ +/*****************************************************************************/ +// +// Ensures that "block" is a member of the list of BBs waiting to be imported, pushing it on the list if +// necessary (and ensures that it is a member of the set of BB's on the list, by setting its byte in +// impPendingBlockMembers). Merges the current verification state into the verification state of "block" +// (its "pre-state"). -void Compiler::impCheckCanInline(GenTreeCall* call, - CORINFO_METHOD_HANDLE fncHandle, - unsigned methAttr, - CORINFO_CONTEXT_HANDLE exactContextHnd, - InlineCandidateInfo** ppInlineCandidateInfo, - InlineResult* inlineResult) +void Compiler::impImportBlockPending(BasicBlock* block) { - // Either EE or JIT might throw exceptions below. - // If that happens, just don't inline the method. - - struct Param - { - Compiler* pThis; - GenTreeCall* call; - CORINFO_METHOD_HANDLE fncHandle; - unsigned methAttr; - CORINFO_CONTEXT_HANDLE exactContextHnd; - InlineResult* result; - InlineCandidateInfo** ppInlineCandidateInfo; - } param; - memset(¶m, 0, sizeof(param)); - - param.pThis = this; - param.call = call; - param.fncHandle = fncHandle; - param.methAttr = methAttr; - param.exactContextHnd = (exactContextHnd != nullptr) ? exactContextHnd : MAKE_METHODCONTEXT(fncHandle); - param.result = inlineResult; - param.ppInlineCandidateInfo = ppInlineCandidateInfo; - - bool success = eeRunWithErrorTrap( - [](Param* pParam) { - CorInfoInitClassResult initClassResult; - #ifdef DEBUG - const char* methodName; - const char* className; - methodName = pParam->pThis->eeGetMethodName(pParam->fncHandle, &className); - - if (JitConfig.JitNoInline()) - { - pParam->result->NoteFatal(InlineObservation::CALLEE_IS_JIT_NOINLINE); - goto _exit; - } + if (verbose) + { + printf("\nimpImportBlockPending for " FMT_BB "\n", block->bbNum); + } #endif - /* Try to get the code address/size for the method */ - - CORINFO_METHOD_INFO methInfo; - if (!pParam->pThis->info.compCompHnd->getMethodInfo(pParam->fncHandle, &methInfo)) - { - pParam->result->NoteFatal(InlineObservation::CALLEE_NO_METHOD_INFO); - goto _exit; - } + // We will add a block to the pending set if it has not already been imported (or needs to be re-imported), + // or if it has, but merging in a predecessor's post-state changes the block's pre-state. + // (When we're doing verification, we always attempt the merge to detect verification errors.) - // Profile data allows us to avoid early "too many IL bytes" outs. - pParam->result->NoteBool(InlineObservation::CALLSITE_HAS_PROFILE, - pParam->pThis->fgHaveSufficientProfileData()); + // If the block has not been imported, add to pending set. + bool addToPending = ((block->bbFlags & BBF_IMPORTED) == 0); - bool forceInline; - forceInline = !!(pParam->methAttr & CORINFO_FLG_FORCEINLINE); + // Initialize bbEntryState just the first time we try to add this block to the pending list + // Just because bbEntryState is NULL, doesn't mean the pre-state wasn't previously set + // We use NULL to indicate the 'common' state to avoid memory allocation + if ((block->bbEntryState == nullptr) && ((block->bbFlags & (BBF_IMPORTED | BBF_FAILED_VERIFICATION)) == 0) && + (impGetPendingBlockMember(block) == 0)) + { + verInitBBEntryState(block, &verCurrentState); + assert(block->bbStkDepth == 0); + block->bbStkDepth = static_cast(verCurrentState.esStackDepth); + assert(addToPending); + assert(impGetPendingBlockMember(block) == 0); + } + else + { + // The stack should have the same height on entry to the block from all its predecessors. + if (block->bbStkDepth != verCurrentState.esStackDepth) + { +#ifdef DEBUG + char buffer[400]; + sprintf_s(buffer, sizeof(buffer), + "Block at offset %4.4x to %4.4x in %0.200s entered with different stack depths.\n" + "Previous depth was %d, current depth is %d", + block->bbCodeOffs, block->bbCodeOffsEnd, info.compFullName, block->bbStkDepth, + verCurrentState.esStackDepth); + buffer[400 - 1] = 0; + NO_WAY(buffer); +#else + NO_WAY("Block entered with different stack depths"); +#endif + } - pParam->pThis->impCanInlineIL(pParam->fncHandle, &methInfo, forceInline, pParam->result); + if (!addToPending) + { + return; + } - if (pParam->result->IsFailure()) - { - assert(pParam->result->IsNever()); - goto _exit; - } + if (block->bbStkDepth > 0) + { + // We need to fix the types of any spill temps that might have changed: + // int->native int, float->double, int->byref, etc. + impRetypeEntryStateTemps(block); + } - // Speculatively check if initClass() can be done. - // If it can be done, we will try to inline the method. - initClassResult = - pParam->pThis->info.compCompHnd->initClass(nullptr /* field */, pParam->fncHandle /* method */, - pParam->exactContextHnd /* context */); + // OK, we must add to the pending list, if it's not already in it. + if (impGetPendingBlockMember(block) != 0) + { + return; + } + } - if (initClassResult & CORINFO_INITCLASS_DONT_INLINE) - { - pParam->result->NoteFatal(InlineObservation::CALLSITE_CANT_CLASS_INIT); - goto _exit; - } + // Get an entry to add to the pending list - // Given the EE the final say in whether to inline or not. - // This should be last since for verifiable code, this can be expensive + PendingDsc* dsc; - /* VM Inline check also ensures that the method is verifiable if needed */ - CorInfoInline vmResult; - vmResult = pParam->pThis->info.compCompHnd->canInline(pParam->pThis->info.compMethodHnd, pParam->fncHandle); + if (impPendingFree) + { + // We can reuse one of the freed up dscs. + dsc = impPendingFree; + impPendingFree = dsc->pdNext; + } + else + { + // We have to create a new dsc + dsc = new (this, CMK_Unknown) PendingDsc; + } - if (vmResult == INLINE_FAIL) - { - pParam->result->NoteFatal(InlineObservation::CALLSITE_IS_VM_NOINLINE); - } - else if (vmResult == INLINE_NEVER) - { - pParam->result->NoteFatal(InlineObservation::CALLEE_IS_VM_NOINLINE); - } + dsc->pdBB = block; + dsc->pdSavedStack.ssDepth = verCurrentState.esStackDepth; - if (pParam->result->IsFailure()) - { - // Do not report this as a failure. Instead report as a "VM failure" - pParam->result->SetVMFailure(); - goto _exit; - } + // Save the stack trees for later - /* Get the method properties */ + if (verCurrentState.esStackDepth) + { + impSaveStackState(&dsc->pdSavedStack, false); + } - CORINFO_CLASS_HANDLE clsHandle; - clsHandle = pParam->pThis->info.compCompHnd->getMethodClass(pParam->fncHandle); - unsigned clsAttr; - clsAttr = pParam->pThis->info.compCompHnd->getClassAttribs(clsHandle); + // Add the entry to the pending list - /* Get the return type */ + dsc->pdNext = impPendingList; + impPendingList = dsc; + impSetPendingBlockMember(block, 1); // And indicate that it's now a member of the set. - var_types fncRetType; - fncRetType = pParam->call->TypeGet(); + // Various assertions require us to now to consider the block as not imported (at least for + // the final time...) + block->bbFlags &= ~BBF_IMPORTED; #ifdef DEBUG - var_types fncRealRetType; - fncRealRetType = JITtype2varType(methInfo.args.retType); - - assert((genActualType(fncRealRetType) == genActualType(fncRetType)) || - // VSW 288602 - // In case of IJW, we allow to assign a native pointer to a BYREF. - (fncRetType == TYP_BYREF && methInfo.args.retType == CORINFO_TYPE_PTR) || - (varTypeIsStruct(fncRetType) && (fncRealRetType == TYP_STRUCT))); -#endif - - // Allocate an InlineCandidateInfo structure, - // - // Or, reuse the existing GuardedDevirtualizationCandidateInfo, - // which was pre-allocated to have extra room. - // - InlineCandidateInfo* pInfo; - - if (pParam->call->IsGuardedDevirtualizationCandidate()) - { - pInfo = pParam->call->gtInlineCandidateInfo; - } - else - { - pInfo = new (pParam->pThis, CMK_Inlining) InlineCandidateInfo; - - // Null out bits we don't use when we're just inlining - pInfo->guardedClassHandle = nullptr; - pInfo->guardedMethodHandle = nullptr; - pInfo->guardedMethodUnboxedEntryHandle = nullptr; - pInfo->likelihood = 0; - pInfo->requiresInstMethodTableArg = false; - } - - pInfo->methInfo = methInfo; - pInfo->ilCallerHandle = pParam->pThis->info.compMethodHnd; - pInfo->clsHandle = clsHandle; - pInfo->exactContextHnd = pParam->exactContextHnd; - pInfo->retExpr = nullptr; - pInfo->preexistingSpillTemp = BAD_VAR_NUM; - pInfo->clsAttr = clsAttr; - pInfo->methAttr = pParam->methAttr; - pInfo->initClassResult = initClassResult; - pInfo->fncRetType = fncRetType; - pInfo->exactContextNeedsRuntimeLookup = false; - pInfo->inlinersContext = pParam->pThis->compInlineContext; - - // Note exactContextNeedsRuntimeLookup is reset later on, - // over in impMarkInlineCandidate. - - *(pParam->ppInlineCandidateInfo) = pInfo; - - _exit:; - }, - ¶m); - if (!success) + if (verbose && 0) { - param.result->NoteFatal(InlineObservation::CALLSITE_COMPILATION_ERROR); + printf("Added PendingDsc - %08p for " FMT_BB "\n", dspPtr(dsc), block->bbNum); } +#endif } -//------------------------------------------------------------------------ -// impInlineRecordArgInfo: record information about an inline candidate argument -// -// Arguments: -// pInlineInfo - inline info for the inline candidate -// arg - the caller argument -// argNum - logical index of this argument -// inlineResult - result of ongoing inline evaluation -// -// Notes: +/*****************************************************************************/ // -// Checks for various inline blocking conditions and makes notes in -// the inline info arg table about the properties of the actual. These -// properties are used later by impInlineFetchArg to determine how best to -// pass the argument into the inlinee. +// Ensures that "block" is a member of the list of BBs waiting to be imported, pushing it on the list if +// necessary (and ensures that it is a member of the set of BB's on the list, by setting its byte in +// impPendingBlockMembers). Does *NOT* change the existing "pre-state" of the block. -void Compiler::impInlineRecordArgInfo(InlineInfo* pInlineInfo, - CallArg* arg, - unsigned argNum, - InlineResult* inlineResult) +void Compiler::impReimportBlockPending(BasicBlock* block) { - InlArgInfo* inlCurArgInfo = &pInlineInfo->inlArgInfo[argNum]; - - inlCurArgInfo->arg = arg; - GenTree* curArgVal = arg->GetNode(); + JITDUMP("\nimpReimportBlockPending for " FMT_BB, block->bbNum); - assert(!curArgVal->OperIs(GT_RET_EXPR)); + assert(block->bbFlags & BBF_IMPORTED); - if (curArgVal->gtOper == GT_MKREFANY) + // OK, we must add to the pending list, if it's not already in it. + if (impGetPendingBlockMember(block) != 0) { - inlineResult->NoteFatal(InlineObservation::CALLSITE_ARG_IS_MKREFANY); return; } - GenTree* lclVarTree; + // Get an entry to add to the pending list - const bool isAddressInLocal = impIsAddressInLocal(curArgVal, &lclVarTree); - if (isAddressInLocal && varTypeIsStruct(lclVarTree)) + PendingDsc* dsc; + + if (impPendingFree) { - inlCurArgInfo->argIsByRefToStructLocal = true; -#ifdef FEATURE_SIMD - if (lvaTable[lclVarTree->AsLclVarCommon()->GetLclNum()].lvSIMDType) - { - pInlineInfo->hasSIMDTypeArgLocalOrReturn = true; - } -#endif // FEATURE_SIMD + // We can reuse one of the freed up dscs. + dsc = impPendingFree; + impPendingFree = dsc->pdNext; + } + else + { + // We have to create a new dsc + dsc = new (this, CMK_ImpStack) PendingDsc; } - if (curArgVal->gtFlags & GTF_ALL_EFFECT) + dsc->pdBB = block; + + if (block->bbEntryState) { - inlCurArgInfo->argHasGlobRef = (curArgVal->gtFlags & GTF_GLOB_REF) != 0; - inlCurArgInfo->argHasSideEff = (curArgVal->gtFlags & (GTF_ALL_EFFECT & ~GTF_GLOB_REF)) != 0; + dsc->pdSavedStack.ssDepth = block->bbEntryState->esStackDepth; + dsc->pdSavedStack.ssTrees = block->bbEntryState->esStack; + } + else + { + dsc->pdSavedStack.ssDepth = 0; + dsc->pdSavedStack.ssTrees = nullptr; } - if (curArgVal->gtOper == GT_LCL_VAR) - { - inlCurArgInfo->argIsLclVar = true; + // Add the entry to the pending list + + dsc->pdNext = impPendingList; + impPendingList = dsc; + impSetPendingBlockMember(block, 1); // And indicate that it's now a member of the set. + + // Various assertions require us to now to consider the block as not imported (at least for + // the final time...) + block->bbFlags &= ~BBF_IMPORTED; - /* Remember the "original" argument number */ - INDEBUG(curArgVal->AsLclVar()->gtLclILoffs = argNum;) +#ifdef DEBUG + if (verbose && 0) + { + printf("Added PendingDsc - %08p for " FMT_BB "\n", dspPtr(dsc), block->bbNum); } +#endif +} - if (impIsInvariant(curArgVal)) +void* Compiler::BlockListNode::operator new(size_t sz, Compiler* comp) +{ + if (comp->impBlockListNodeFreeList == nullptr) { - inlCurArgInfo->argIsInvariant = true; - if (inlCurArgInfo->argIsThis && (curArgVal->gtOper == GT_CNS_INT) && (curArgVal->AsIntCon()->gtIconVal == 0)) - { - // Abort inlining at this call site - inlineResult->NoteFatal(InlineObservation::CALLSITE_ARG_HAS_NULL_THIS); - return; - } + return comp->getAllocator(CMK_BasicBlock).allocate(1); } else { - if (curArgVal->IsHelperCall() && gtIsTypeHandleToRuntimeTypeHelper(curArgVal->AsCall()) && - (gtGetHelperArgClassHandle(curArgVal->AsCall()->gtArgs.GetArgByIndex(0)->GetEarlyNode()) != - NO_CLASS_HANDLE)) - { - inlCurArgInfo->argIsInvariant = true; - inlCurArgInfo->argHasSideEff = false; - } + BlockListNode* res = comp->impBlockListNodeFreeList; + comp->impBlockListNodeFreeList = res->m_next; + return res; } +} - bool isExact = false; - bool isNonNull = false; - inlCurArgInfo->argIsExact = (gtGetClassHandle(curArgVal, &isExact, &isNonNull) != NO_CLASS_HANDLE) && isExact; +void Compiler::FreeBlockListNode(Compiler::BlockListNode* node) +{ + node->m_next = impBlockListNodeFreeList; + impBlockListNodeFreeList = node; +} - // If the arg is a local that is address-taken, we can't safely - // directly substitute it into the inlinee. - // - // Previously we'd accomplish this by setting "argHasLdargaOp" but - // that has a stronger meaning: that the arg value can change in - // the method body. Using that flag prevents type propagation, - // which is safe in this case. - // - // Instead mark the arg as having a caller local ref. - if (!inlCurArgInfo->argIsInvariant && gtHasLocalsWithAddrOp(curArgVal)) +void Compiler::impWalkSpillCliqueFromPred(BasicBlock* block, SpillCliqueWalker* callback) +{ + bool toDo = true; + + noway_assert(!fgComputePredsDone); + if (!fgCheapPredsValid) { - inlCurArgInfo->argHasCallerLocalRef = true; + fgComputeCheapPreds(); } -#ifdef DEBUG - if (verbose) + BlockListNode* succCliqueToDo = nullptr; + BlockListNode* predCliqueToDo = new (this) BlockListNode(block); + while (toDo) { - if (inlCurArgInfo->argIsThis) - { - printf("thisArg:"); - } - else - { - printf("\nArgument #%u:", argNum); - } - if (inlCurArgInfo->argIsLclVar) - { - printf(" is a local var"); - } - if (inlCurArgInfo->argIsInvariant) - { - printf(" is a constant or invariant"); - } - if (inlCurArgInfo->argHasGlobRef) - { - printf(" has global refs"); - } - if (inlCurArgInfo->argHasCallerLocalRef) - { - printf(" has caller local ref"); - } - if (inlCurArgInfo->argHasSideEff) - { - printf(" has side effects"); - } - if (inlCurArgInfo->argHasLdargaOp) - { - printf(" has ldarga effect"); - } - if (inlCurArgInfo->argHasStargOp) + toDo = false; + // Look at the successors of every member of the predecessor to-do list. + while (predCliqueToDo != nullptr) { - printf(" has starg effect"); + BlockListNode* node = predCliqueToDo; + predCliqueToDo = node->m_next; + BasicBlock* blk = node->m_blk; + FreeBlockListNode(node); + + for (BasicBlock* const succ : blk->Succs()) + { + // If it's not already in the clique, add it, and also add it + // as a member of the successor "toDo" set. + if (impSpillCliqueGetMember(SpillCliqueSucc, succ) == 0) + { + callback->Visit(SpillCliqueSucc, succ); + impSpillCliqueSetMember(SpillCliqueSucc, succ, 1); + succCliqueToDo = new (this) BlockListNode(succ, succCliqueToDo); + toDo = true; + } + } } - if (inlCurArgInfo->argIsByRefToStructLocal) + // Look at the predecessors of every member of the successor to-do list. + while (succCliqueToDo != nullptr) { - printf(" is byref to a struct local"); - } + BlockListNode* node = succCliqueToDo; + succCliqueToDo = node->m_next; + BasicBlock* blk = node->m_blk; + FreeBlockListNode(node); - printf("\n"); - gtDispTree(curArgVal); - printf("\n"); + for (BasicBlockList* pred = blk->bbCheapPreds; pred != nullptr; pred = pred->next) + { + BasicBlock* predBlock = pred->block; + // If it's not already in the clique, add it, and also add it + // as a member of the predecessor "toDo" set. + if (impSpillCliqueGetMember(SpillCliquePred, predBlock) == 0) + { + callback->Visit(SpillCliquePred, predBlock); + impSpillCliqueSetMember(SpillCliquePred, predBlock, 1); + predCliqueToDo = new (this) BlockListNode(predBlock, predCliqueToDo); + toDo = true; + } + } + } } -#endif -} -//------------------------------------------------------------------------ -// impInlineInitVars: setup inline information for inlinee args and locals -// -// Arguments: -// pInlineInfo - inline info for the inline candidate -// -// Notes: -// This method primarily adds caller-supplied info to the inlArgInfo -// and sets up the lclVarInfo table. -// -// For args, the inlArgInfo records properties of the actual argument -// including the tree node that produces the arg value. This node is -// usually the tree node present at the call, but may also differ in -// various ways: -// - when the call arg is a GT_RET_EXPR, we search back through the ret -// expr chain for the actual node. Note this will either be the original -// call (which will be a failed inline by this point), or the return -// expression from some set of inlines. -// - when argument type casting is needed the necessary casts are added -// around the argument node. -// - if an argument can be simplified by folding then the node here is the -// folded value. -// -// The method may make observations that lead to marking this candidate as -// a failed inline. If this happens the initialization is abandoned immediately -// to try and reduce the jit time cost for a failed inline. + // If this fails, it means we didn't walk the spill clique properly and somehow managed + // miss walking back to include the predecessor we started from. + // This most likely cause: missing or out of date bbPreds + assert(impSpillCliqueGetMember(SpillCliquePred, block) != 0); +} -void Compiler::impInlineInitVars(InlineInfo* pInlineInfo) +void Compiler::SetSpillTempsBase::Visit(SpillCliqueDir predOrSucc, BasicBlock* blk) { - assert(!compIsForInlining()); - - GenTreeCall* call = pInlineInfo->iciCall; - CORINFO_METHOD_INFO* methInfo = &pInlineInfo->inlineCandidateInfo->methInfo; - unsigned clsAttr = pInlineInfo->inlineCandidateInfo->clsAttr; - InlArgInfo* inlArgInfo = pInlineInfo->inlArgInfo; - InlLclVarInfo* lclVarInfo = pInlineInfo->lclVarInfo; - InlineResult* inlineResult = pInlineInfo->inlineResult; + if (predOrSucc == SpillCliqueSucc) + { + assert(blk->bbStkTempsIn == NO_BASE_TMP); // Should not already be a member of a clique as a successor. + blk->bbStkTempsIn = m_baseTmp; + } + else + { + assert(predOrSucc == SpillCliquePred); + assert(blk->bbStkTempsOut == NO_BASE_TMP); // Should not already be a member of a clique as a predecessor. + blk->bbStkTempsOut = m_baseTmp; + } +} - // Inlined methods always use the managed calling convention - const bool hasRetBuffArg = impMethodInfo_hasRetBuffArg(methInfo, CorInfoCallConvExtension::Managed); +void Compiler::ReimportSpillClique::Visit(SpillCliqueDir predOrSucc, BasicBlock* blk) +{ + // For Preds we could be a little smarter and just find the existing store + // and re-type it/add a cast, but that is complicated and hopefully very rare, so + // just re-import the whole block (just like we do for successors) - /* init the argument stuct */ + if (((blk->bbFlags & BBF_IMPORTED) == 0) && (m_pComp->impGetPendingBlockMember(blk) == 0)) + { + // If we haven't imported this block and we're not going to (because it isn't on + // the pending list) then just ignore it for now. - memset(inlArgInfo, 0, (MAX_INL_ARGS + 1) * sizeof(inlArgInfo[0])); + // This block has either never been imported (EntryState == NULL) or it failed + // verification. Neither state requires us to force it to be imported now. + assert((blk->bbEntryState == nullptr) || (blk->bbFlags & BBF_FAILED_VERIFICATION)); + return; + } - unsigned ilArgCnt = 0; - for (CallArg& arg : call->gtArgs.Args()) + // For successors we have a valid verCurrentState, so just mark them for reimport + // the 'normal' way + // Unlike predecessors, we *DO* need to reimport the current block because the + // initial import had the wrong entry state types. + // Similarly, blocks that are currently on the pending list, still need to call + // impImportBlockPending to fixup their entry state. + if (predOrSucc == SpillCliqueSucc) { - switch (arg.GetWellKnownArg()) - { - case WellKnownArg::ThisPointer: - inlArgInfo[ilArgCnt].argIsThis = true; - break; - case WellKnownArg::RetBuffer: - case WellKnownArg::InstParam: - // These do not appear in the table of inline arg info; do not include them - continue; - default: - break; - } - - arg.SetEarlyNode(gtFoldExpr(arg.GetEarlyNode())); - impInlineRecordArgInfo(pInlineInfo, &arg, ilArgCnt, inlineResult); + m_pComp->impReimportMarkBlock(blk); - if (inlineResult->IsFailure()) - { - return; - } + // Set the current stack state to that of the blk->bbEntryState + m_pComp->verResetCurrentState(blk, &m_pComp->verCurrentState); - ilArgCnt++; + m_pComp->impImportBlockPending(blk); + } + else if ((blk != m_pComp->compCurBB) && ((blk->bbFlags & BBF_IMPORTED) != 0)) + { + // As described above, we are only visiting predecessors so they can + // add the appropriate casts, since we have already done that for the current + // block, it does not need to be reimported. + // Nor do we need to reimport blocks that are still pending, but not yet + // imported. + // + // For predecessors, we have no state to seed the EntryState, so we just have + // to assume the existing one is correct. + // If the block is also a successor, it will get the EntryState properly + // updated when it is visited as a successor in the above "if" block. + assert(predOrSucc == SpillCliquePred); + m_pComp->impReimportBlockPending(blk); } +} - /* Make sure we got the arg number right */ - assert(ilArgCnt == methInfo->args.totalILArgs()); - -#ifdef FEATURE_SIMD - bool foundSIMDType = pInlineInfo->hasSIMDTypeArgLocalOrReturn; -#endif // FEATURE_SIMD - - /* We have typeless opcodes, get type information from the signature */ - - CallArg* thisArg = call->gtArgs.GetThisArg(); - if (thisArg != nullptr) +// Re-type the incoming lclVar nodes to match the varDsc. +void Compiler::impRetypeEntryStateTemps(BasicBlock* blk) +{ + if (blk->bbEntryState != nullptr) { - lclVarInfo[0].lclVerTypeInfo = verMakeTypeInfo(pInlineInfo->inlineCandidateInfo->clsHandle); - lclVarInfo[0].lclHasLdlocaOp = false; - -#ifdef FEATURE_SIMD - // We always want to check isSIMDClass, since we want to set foundSIMDType (to increase - // the inlining multiplier) for anything in that assembly. - // But we only need to normalize it if it is a TYP_STRUCT - // (which we need to do even if we have already set foundSIMDType). - if (!foundSIMDType && isSIMDorHWSIMDClass(&(lclVarInfo[0].lclVerTypeInfo))) + EntryState* es = blk->bbEntryState; + for (unsigned level = 0; level < es->esStackDepth; level++) { - foundSIMDType = true; + GenTree* tree = es->esStack[level].val; + if ((tree->gtOper == GT_LCL_VAR) || (tree->gtOper == GT_LCL_FLD)) + { + es->esStack[level].val->gtType = lvaGetDesc(tree->AsLclVarCommon())->TypeGet(); + } } -#endif // FEATURE_SIMD - - var_types sigType = ((clsAttr & CORINFO_FLG_VALUECLASS) != 0) ? TYP_BYREF : TYP_REF; - lclVarInfo[0].lclTypeInfo = sigType; + } +} - GenTree* thisArgNode = thisArg->GetEarlyNode(); +unsigned Compiler::impGetSpillTmpBase(BasicBlock* block) +{ + if (block->bbStkTempsOut != NO_BASE_TMP) + { + return block->bbStkTempsOut; + } - assert(varTypeIsGC(thisArgNode->TypeGet()) || // "this" is managed - ((thisArgNode->TypeGet() == TYP_I_IMPL) && // "this" is unmgd but the method's class doesnt care - (clsAttr & CORINFO_FLG_VALUECLASS))); +#ifdef DEBUG + if (verbose) + { + printf("\n*************** In impGetSpillTmpBase(" FMT_BB ")\n", block->bbNum); + } +#endif // DEBUG - if (genActualType(thisArgNode->TypeGet()) != genActualType(sigType)) - { - if (sigType == TYP_REF) - { - /* The argument cannot be bashed into a ref (see bug 750871) */ - inlineResult->NoteFatal(InlineObservation::CALLSITE_ARG_NO_BASH_TO_REF); - return; - } + // Otherwise, choose one, and propagate to all members of the spill clique. + // Grab enough temps for the whole stack. + unsigned baseTmp = lvaGrabTemps(verCurrentState.esStackDepth DEBUGARG("IL Stack Entries")); + SetSpillTempsBase callback(baseTmp); - /* This can only happen with byrefs <-> ints/shorts */ + // We do *NOT* need to reset the SpillClique*Members because a block can only be the predecessor + // to one spill clique, and similarly can only be the successor to one spill clique + impWalkSpillCliqueFromPred(block, &callback); - assert(sigType == TYP_BYREF); - assert((genActualType(thisArgNode->TypeGet()) == TYP_I_IMPL) || (thisArgNode->TypeGet() == TYP_BYREF)); + return baseTmp; +} - lclVarInfo[0].lclVerTypeInfo = typeInfo(varType2tiType(TYP_I_IMPL)); - } +void Compiler::impReimportSpillClique(BasicBlock* block) +{ +#ifdef DEBUG + if (verbose) + { + printf("\n*************** In impReimportSpillClique(" FMT_BB ")\n", block->bbNum); } +#endif // DEBUG - /* Init the types of the arguments and make sure the types - * from the trees match the types in the signature */ + // If we get here, it is because this block is already part of a spill clique + // and one predecessor had an outgoing live stack slot of type int, and this + // block has an outgoing live stack slot of type native int. + // We need to reset these before traversal because they have already been set + // by the previous walk to determine all the members of the spill clique. + impInlineRoot()->impSpillCliquePredMembers.Reset(); + impInlineRoot()->impSpillCliqueSuccMembers.Reset(); - CORINFO_ARG_LIST_HANDLE argLst; - argLst = methInfo->args.args; + ReimportSpillClique callback(this); - // TODO-ARGS: We can presumably just use type info stored in CallArgs - // instead of reiterating the signature. - unsigned i; - for (i = (thisArg ? 1 : 0); i < ilArgCnt; i++, argLst = info.compCompHnd->getArgNext(argLst)) - { - var_types sigType = (var_types)eeGetArgType(argLst, &methInfo->args); + impWalkSpillCliqueFromPred(block, &callback); +} - lclVarInfo[i].lclVerTypeInfo = verParseArgSigToTypeInfo(&methInfo->args, argLst); +// Set the pre-state of "block" (which should not have a pre-state allocated) to +// a copy of "srcState", cloning tree pointers as required. +void Compiler::verInitBBEntryState(BasicBlock* block, EntryState* srcState) +{ + if (srcState->esStackDepth == 0) + { + block->bbEntryState = nullptr; + return; + } -#ifdef FEATURE_SIMD - if ((!foundSIMDType || (sigType == TYP_STRUCT)) && isSIMDorHWSIMDClass(&(lclVarInfo[i].lclVerTypeInfo))) - { - // If this is a SIMD class (i.e. in the SIMD assembly), then we will consider that we've - // found a SIMD type, even if this may not be a type we recognize (the assumption is that - // it is likely to use a SIMD type, and therefore we want to increase the inlining multiplier). - foundSIMDType = true; - if (sigType == TYP_STRUCT) - { - var_types structType = impNormStructType(lclVarInfo[i].lclVerTypeInfo.GetClassHandle()); - sigType = structType; - } - } -#endif // FEATURE_SIMD + block->bbEntryState = getAllocator(CMK_Unknown).allocate(1); - lclVarInfo[i].lclTypeInfo = sigType; - lclVarInfo[i].lclHasLdlocaOp = false; + // block->bbEntryState.esRefcount = 1; - // Does the tree type match the signature type? + block->bbEntryState->esStackDepth = srcState->esStackDepth; - GenTree* inlArgNode = inlArgInfo[i].arg->GetNode(); + if (srcState->esStackDepth > 0) + { + block->bbSetStack(new (this, CMK_Unknown) StackEntry[srcState->esStackDepth]); + unsigned stackSize = srcState->esStackDepth * sizeof(StackEntry); - if (sigType == inlArgNode->gtType) + memcpy(block->bbEntryState->esStack, srcState->esStack, stackSize); + for (unsigned level = 0; level < srcState->esStackDepth; level++) { - continue; + GenTree* tree = srcState->esStack[level].val; + block->bbEntryState->esStack[level].val = gtCloneExpr(tree); } + } +} - assert(impCheckImplicitArgumentCoercion(sigType, inlArgNode->gtType)); - assert(!varTypeIsStruct(inlArgNode->gtType) && !varTypeIsStruct(sigType)); +/* + * Resets the current state to the state at the start of the basic block + */ +void Compiler::verResetCurrentState(BasicBlock* block, EntryState* destState) +{ + if (block->bbEntryState == nullptr) + { + destState->esStackDepth = 0; + return; + } - // In valid IL, this can only happen for short integer types or byrefs <-> [native] ints, - // but in bad IL cases with caller-callee signature mismatches we can see other types. - // Intentionally reject cases with mismatches so the jit is more flexible when - // encountering bad IL. + destState->esStackDepth = block->bbEntryState->esStackDepth; - bool isPlausibleTypeMatch = (genActualType(sigType) == genActualType(inlArgNode->gtType)) || - (genActualTypeIsIntOrI(sigType) && inlArgNode->gtType == TYP_BYREF) || - (sigType == TYP_BYREF && genActualTypeIsIntOrI(inlArgNode->gtType)); + if (destState->esStackDepth > 0) + { + unsigned stackSize = destState->esStackDepth * sizeof(StackEntry); - if (!isPlausibleTypeMatch) - { - inlineResult->NoteFatal(InlineObservation::CALLSITE_ARG_TYPES_INCOMPATIBLE); - return; - } + memcpy(destState->esStack, block->bbStackOnEntry(), stackSize); + } +} - // The same size but different type of the arguments. - GenTree** pInlArgNode = &inlArgInfo[i].arg->EarlyNodeRef(); +unsigned BasicBlock::bbStackDepthOnEntry() const +{ + return (bbEntryState ? bbEntryState->esStackDepth : 0); +} - // Is it a narrowing or widening cast? - // Widening casts are ok since the value computed is already - // normalized to an int (on the IL stack) - if (genTypeSize(inlArgNode->gtType) >= genTypeSize(sigType)) - { - if (sigType == TYP_BYREF) - { - lclVarInfo[i].lclVerTypeInfo = typeInfo(varType2tiType(TYP_I_IMPL)); - } - else if (inlArgNode->gtType == TYP_BYREF) - { - assert(varTypeIsIntOrI(sigType)); +void BasicBlock::bbSetStack(void* stackBuffer) +{ + assert(bbEntryState); + assert(stackBuffer); + bbEntryState->esStack = (StackEntry*)stackBuffer; +} - /* If possible bash the BYREF to an int */ - if (inlArgNode->IsLocalAddrExpr() != nullptr) - { - inlArgNode->gtType = TYP_I_IMPL; - lclVarInfo[i].lclVerTypeInfo = typeInfo(varType2tiType(TYP_I_IMPL)); - } - else - { - // Arguments 'int <- byref' cannot be changed - inlineResult->NoteFatal(InlineObservation::CALLSITE_ARG_NO_BASH_TO_INT); - return; - } - } - else if (genTypeSize(sigType) < TARGET_POINTER_SIZE) - { - // Narrowing cast. - if (inlArgNode->OperIs(GT_LCL_VAR)) - { - const unsigned lclNum = inlArgNode->AsLclVarCommon()->GetLclNum(); - if (!lvaTable[lclNum].lvNormalizeOnLoad() && sigType == lvaGetRealType(lclNum)) - { - // We don't need to insert a cast here as the variable - // was assigned a normalized value of the right type. - continue; - } - } +StackEntry* BasicBlock::bbStackOnEntry() const +{ + assert(bbEntryState); + return bbEntryState->esStack; +} - inlArgNode = gtNewCastNode(TYP_INT, inlArgNode, false, sigType); +void Compiler::verInitCurrentState() +{ + // initialize stack info + verCurrentState.esStackDepth = 0; + assert(verCurrentState.esStack != nullptr); - inlArgInfo[i].argIsLclVar = false; - // Try to fold the node in case we have constant arguments. - if (inlArgInfo[i].argIsInvariant) - { - inlArgNode = gtFoldExpr(inlArgNode); - } - *pInlArgNode = inlArgNode; - } -#ifdef TARGET_64BIT - else if (genTypeSize(genActualType(inlArgNode->gtType)) < genTypeSize(sigType)) - { - // This should only happen for int -> native int widening - inlArgNode = gtNewCastNode(genActualType(sigType), inlArgNode, false, sigType); + // copy current state to entry state of first BB + verInitBBEntryState(fgFirstBB, &verCurrentState); +} - inlArgInfo[i].argIsLclVar = false; +Compiler* Compiler::impInlineRoot() +{ + if (impInlineInfo == nullptr) + { + return this; + } + else + { + return impInlineInfo->InlineRoot; + } +} - // Try to fold the node in case we have constant arguments. - if (inlArgInfo[i].argIsInvariant) - { - inlArgNode = gtFoldExpr(inlArgNode); - } - *pInlArgNode = inlArgNode; - } -#endif // TARGET_64BIT - } +BYTE Compiler::impSpillCliqueGetMember(SpillCliqueDir predOrSucc, BasicBlock* blk) +{ + if (predOrSucc == SpillCliquePred) + { + return impInlineRoot()->impSpillCliquePredMembers.Get(blk->bbInd()); + } + else + { + assert(predOrSucc == SpillCliqueSucc); + return impInlineRoot()->impSpillCliqueSuccMembers.Get(blk->bbInd()); } +} - /* Init the types of the local variables */ +void Compiler::impSpillCliqueSetMember(SpillCliqueDir predOrSucc, BasicBlock* blk, BYTE val) +{ + if (predOrSucc == SpillCliquePred) + { + impInlineRoot()->impSpillCliquePredMembers.Set(blk->bbInd(), val); + } + else + { + assert(predOrSucc == SpillCliqueSucc); + impInlineRoot()->impSpillCliqueSuccMembers.Set(blk->bbInd(), val); + } +} - CORINFO_ARG_LIST_HANDLE localsSig; - localsSig = methInfo->locals.args; +/***************************************************************************** + * + * Convert the instrs ("import") into our internal format (trees). The + * basic flowgraph has already been constructed and is passed in. + */ - for (i = 0; i < methInfo->locals.numArgs; i++) +void Compiler::impImport() +{ +#ifdef DEBUG + if (verbose) { - bool isPinned; - var_types type = (var_types)eeGetArgType(localsSig, &methInfo->locals, &isPinned); - - lclVarInfo[i + ilArgCnt].lclHasLdlocaOp = false; - lclVarInfo[i + ilArgCnt].lclTypeInfo = type; + printf("*************** In impImport() for %s\n", info.compFullName); + } +#endif - if (varTypeIsGC(type)) - { - if (isPinned) - { - JITDUMP("Inlinee local #%02u is pinned\n", i); - lclVarInfo[i + ilArgCnt].lclIsPinned = true; + Compiler* inlineRoot = impInlineRoot(); - // Pinned locals may cause inlines to fail. - inlineResult->Note(InlineObservation::CALLEE_HAS_PINNED_LOCALS); - if (inlineResult->IsFailure()) - { - return; - } - } + if (info.compMaxStack <= SMALL_STACK_SIZE) + { + impStkSize = SMALL_STACK_SIZE; + } + else + { + impStkSize = info.compMaxStack; + } - pInlineInfo->numberOfGcRefLocals++; - } - else if (isPinned) + if (this == inlineRoot) + { + // Allocate the stack contents + verCurrentState.esStack = new (this, CMK_ImpStack) StackEntry[impStkSize]; + } + else + { + // This is the inlinee compiler, steal the stack from the inliner compiler + // (after ensuring that it is large enough). + if (inlineRoot->impStkSize < impStkSize) { - JITDUMP("Ignoring pin on inlinee local #%02u -- not a GC type\n", i); + inlineRoot->impStkSize = impStkSize; + inlineRoot->verCurrentState.esStack = new (this, CMK_ImpStack) StackEntry[impStkSize]; } - lclVarInfo[i + ilArgCnt].lclVerTypeInfo = verParseArgSigToTypeInfo(&methInfo->locals, localsSig); + verCurrentState.esStack = inlineRoot->verCurrentState.esStack; + } - // If this local is a struct type with GC fields, inform the inliner. It may choose to bail - // out on the inline. - if (type == TYP_STRUCT) - { - CORINFO_CLASS_HANDLE lclHandle = lclVarInfo[i + ilArgCnt].lclVerTypeInfo.GetClassHandle(); - DWORD typeFlags = info.compCompHnd->getClassAttribs(lclHandle); - if ((typeFlags & CORINFO_FLG_CONTAINS_GC_PTR) != 0) - { - inlineResult->Note(InlineObservation::CALLEE_HAS_GC_STRUCT); - if (inlineResult->IsFailure()) - { - return; - } + // initialize the entry state at start of method + verInitCurrentState(); - // Do further notification in the case where the call site is rare; some policies do - // not track the relative hotness of call sites for "always" inline cases. - if (pInlineInfo->iciBlock->isRunRarely()) - { - inlineResult->Note(InlineObservation::CALLSITE_RARE_GC_STRUCT); - if (inlineResult->IsFailure()) - { + // Initialize stuff related to figuring "spill cliques" (see spec comment for impGetSpillTmpBase). + if (this == inlineRoot) // These are only used on the root of the inlining tree. + { + // We have initialized these previously, but to size 0. Make them larger. + impPendingBlockMembers.Init(getAllocator(), fgBBNumMax * 2); + impSpillCliquePredMembers.Init(getAllocator(), fgBBNumMax * 2); + impSpillCliqueSuccMembers.Init(getAllocator(), fgBBNumMax * 2); + } + inlineRoot->impPendingBlockMembers.Reset(fgBBNumMax * 2); + inlineRoot->impSpillCliquePredMembers.Reset(fgBBNumMax * 2); + inlineRoot->impSpillCliqueSuccMembers.Reset(fgBBNumMax * 2); + impBlockListNodeFreeList = nullptr; - return; - } - } - } - } +#ifdef DEBUG + impLastILoffsStmt = nullptr; + impNestedStackSpill = false; +#endif + impBoxTemp = BAD_VAR_NUM; - localsSig = info.compCompHnd->getArgNext(localsSig); + impPendingList = impPendingFree = nullptr; -#ifdef FEATURE_SIMD - if ((!foundSIMDType || (type == TYP_STRUCT)) && isSIMDorHWSIMDClass(&(lclVarInfo[i + ilArgCnt].lclVerTypeInfo))) - { - foundSIMDType = true; - if (type == TYP_STRUCT) - { - var_types structType = impNormStructType(lclVarInfo[i + ilArgCnt].lclVerTypeInfo.GetClassHandle()); - lclVarInfo[i + ilArgCnt].lclTypeInfo = structType; - } - } -#endif // FEATURE_SIMD - } + // Skip leading internal blocks. + // These can arise from needing a leading scratch BB, from EH normalization, and from OSR entry redirects. + // + BasicBlock* entryBlock = fgFirstBB; -#ifdef FEATURE_SIMD - if (!foundSIMDType && (call->AsCall()->gtRetClsHnd != nullptr) && isSIMDorHWSIMDClass(call->AsCall()->gtRetClsHnd)) + while (entryBlock->bbFlags & BBF_INTERNAL) { - foundSIMDType = true; - } - pInlineInfo->hasSIMDTypeArgLocalOrReturn = foundSIMDType; -#endif // FEATURE_SIMD -} + JITDUMP("Marking leading BBF_INTERNAL block " FMT_BB " as BBF_IMPORTED\n", entryBlock->bbNum); + entryBlock->bbFlags |= BBF_IMPORTED; -//------------------------------------------------------------------------ -// impInlineFetchLocal: get a local var that represents an inlinee local -// -// Arguments: -// lclNum -- number of the inlinee local -// reason -- debug string describing purpose of the local var -// -// Returns: -// Number of the local to use -// -// Notes: -// This method is invoked only for locals actually used in the -// inlinee body. -// -// Allocates a new temp if necessary, and copies key properties -// over from the inlinee local var info. + if (entryBlock->bbJumpKind == BBJ_NONE) + { + entryBlock = entryBlock->bbNext; + } + else if (opts.IsOSR() && (entryBlock->bbJumpKind == BBJ_ALWAYS)) + { + entryBlock = entryBlock->bbJumpDest; + } + else + { + assert(!"unexpected bbJumpKind in entry sequence"); + } + } -unsigned Compiler::impInlineFetchLocal(unsigned lclNum DEBUGARG(const char* reason)) -{ - assert(compIsForInlining()); + // Note for OSR we'd like to be able to verify this block must be + // stack empty, but won't know that until we've imported...so instead + // we'll BADCODE out if we mess up. + // + // (the concern here is that the runtime asks us to OSR a + // different IL version than the one that matched the method that + // triggered OSR). This should not happen but I might have the + // IL versioning stuff wrong. + // + // TODO: we also currently expect this block to be a join point, + // which we should verify over when we find jump targets. + impImportBlockPending(entryBlock); - unsigned tmpNum = impInlineInfo->lclTmpNum[lclNum]; + /* Import blocks in the worker-list until there are no more */ - if (tmpNum == BAD_VAR_NUM) + while (impPendingList) { - const InlLclVarInfo& inlineeLocal = impInlineInfo->lclVarInfo[lclNum + impInlineInfo->argCnt]; - const var_types lclTyp = inlineeLocal.lclTypeInfo; - - // The lifetime of this local might span multiple BBs. - // So it is a long lifetime local. - impInlineInfo->lclTmpNum[lclNum] = tmpNum = lvaGrabTemp(false DEBUGARG(reason)); - - // Copy over key info - lvaTable[tmpNum].lvType = lclTyp; - lvaTable[tmpNum].lvHasLdAddrOp = inlineeLocal.lclHasLdlocaOp; - lvaTable[tmpNum].lvPinned = inlineeLocal.lclIsPinned; - lvaTable[tmpNum].lvHasILStoreOp = inlineeLocal.lclHasStlocOp; - lvaTable[tmpNum].lvHasMultipleILStoreOp = inlineeLocal.lclHasMultipleStlocOp; + /* Remove the entry at the front of the list */ - // Copy over class handle for ref types. Note this may be a - // shared type -- someday perhaps we can get the exact - // signature and pass in a more precise type. - if (lclTyp == TYP_REF) - { - assert(lvaTable[tmpNum].lvSingleDef == 0); + PendingDsc* dsc = impPendingList; + impPendingList = impPendingList->pdNext; + impSetPendingBlockMember(dsc->pdBB, 0); - lvaTable[tmpNum].lvSingleDef = !inlineeLocal.lclHasMultipleStlocOp && !inlineeLocal.lclHasLdlocaOp; - if (lvaTable[tmpNum].lvSingleDef) - { - JITDUMP("Marked V%02u as a single def temp\n", tmpNum); - } + /* Restore the stack state */ - lvaSetClass(tmpNum, inlineeLocal.lclVerTypeInfo.GetClassHandleForObjRef()); + verCurrentState.esStackDepth = dsc->pdSavedStack.ssDepth; + if (verCurrentState.esStackDepth) + { + impRestoreStackState(&dsc->pdSavedStack); } - if (inlineeLocal.lclVerTypeInfo.IsStruct()) + /* Add the entry to the free list for reuse */ + + dsc->pdNext = impPendingFree; + impPendingFree = dsc; + + /* Now import the block */ + + if (dsc->pdBB->bbFlags & BBF_FAILED_VERIFICATION) { - if (varTypeIsStruct(lclTyp)) + verConvertBBToThrowVerificationException(dsc->pdBB DEBUGARG(true)); + impEndTreeList(dsc->pdBB); + } + else + { + impImportBlock(dsc->pdBB); + + if (compDonotInline()) { - lvaSetStruct(tmpNum, inlineeLocal.lclVerTypeInfo.GetClassHandle(), true /* unsafe value cls check */); + return; } - else + if (compIsForImportOnly()) { - // This is a wrapped primitive. Make sure the verstate knows that - lvaTable[tmpNum].lvVerTypeInfo = inlineeLocal.lclVerTypeInfo; + return; } } + } #ifdef DEBUG - // Sanity check that we're properly prepared for gc ref locals. - if (varTypeIsGC(lclTyp)) - { - // Since there are gc locals we should have seen them earlier - // and if there was a return value, set up the spill temp. - assert(impInlineInfo->HasGcRefLocals()); - assert((info.compRetNativeType == TYP_VOID) || fgNeedReturnSpillTemp()); - } - else - { - // Make sure all pinned locals count as gc refs. - assert(!inlineeLocal.lclIsPinned); - } -#endif // DEBUG + if (verbose && info.compXcptnsCount) + { + printf("\nAfter impImport() added block for try,catch,finally"); + fgDispBasicBlocks(); + printf("\n"); } - return tmpNum; + // Used in impImportBlockPending() for STRESS_CHK_REIMPORT + for (BasicBlock* const block : Blocks()) + { + block->bbFlags &= ~BBF_VISITED; + } +#endif } //------------------------------------------------------------------------ -// impInlineFetchArg: return tree node for argument value in an inlinee +// impIsInvariant: check if a tree (created during import) is invariant. // // Arguments: -// lclNum -- argument number in inlinee IL -// inlArgInfo -- argument info for inlinee -// lclVarInfo -- var info for inlinee +// tree -- The tree // // Returns: -// Tree for the argument's value. Often an inlinee-scoped temp -// GT_LCL_VAR but can be other tree kinds, if the argument -// expression from the caller can be directly substituted into the -// inlinee body. +// true if it is invariant // -// Notes: -// Must be used only for arguments -- use impInlineFetchLocal for -// inlinee locals. +// Remarks: +// This is a variant of GenTree::IsInvariant that is more suitable for use +// during import. Unlike that function, this one handles GT_FIELD nodes. // -// Direct substitution is performed when the formal argument cannot -// change value in the inlinee body (no starg or ldarga), and the -// actual argument expression's value cannot be changed if it is -// substituted it into the inlinee body. +bool Compiler::impIsInvariant(const GenTree* tree) +{ + return tree->OperIsConst() || impIsAddressInLocal(tree); +} + +//------------------------------------------------------------------------ +// impIsAddressInLocal: +// Check to see if the tree is the address of a local or +// the address of a field in a local. +// Arguments: +// tree -- The tree +// lclVarTreeOut -- [out] the local that this points into // -// Even if an inlinee-scoped temp is returned here, it may later be -// "bashed" to a caller-supplied tree when arguments are actually -// passed (see fgInlinePrependStatements). Bashing can happen if -// the argument ends up being single use and other conditions are -// met. So the contents of the tree returned here may not end up -// being the ones ultimately used for the argument. +// Returns: +// true if it points into a local // -// This method will side effect inlArgInfo. It should only be called -// for actual uses of the argument in the inlinee. - -GenTree* Compiler::impInlineFetchArg(unsigned lclNum, InlArgInfo* inlArgInfo, InlLclVarInfo* lclVarInfo) +// Remarks: +// This is a variant of GenTree::IsLocalAddrExpr that is more suitable for +// use during import. Unlike that function, this one handles field nodes. +// +bool Compiler::impIsAddressInLocal(const GenTree* tree, GenTree** lclVarTreeOut) { - // Cache the relevant arg and lcl info for this argument. - // We will modify argInfo but not lclVarInfo. - InlArgInfo& argInfo = inlArgInfo[lclNum]; - const InlLclVarInfo& lclInfo = lclVarInfo[lclNum]; - const bool argCanBeModified = argInfo.argHasLdargaOp || argInfo.argHasStargOp; - const var_types lclTyp = lclInfo.lclTypeInfo; - GenTree* op1 = nullptr; - - GenTree* argNode = argInfo.arg->GetNode(); - assert(!argNode->OperIs(GT_RET_EXPR)); - - if (argInfo.argIsInvariant && !argCanBeModified) + const GenTree* op = tree; + while (op->OperIs(GT_FIELD_ADDR) && op->AsField()->IsInstance()) { - // Directly substitute constants or addresses of locals - // - // Clone the constant. Note that we cannot directly use - // argNode in the trees even if !argInfo.argIsUsed as this - // would introduce aliasing between inlArgInfo[].argNode and - // impInlineExpr. Then gtFoldExpr() could change it, causing - // further references to the argument working off of the - // bashed copy. - op1 = gtCloneExpr(argNode); - PREFIX_ASSUME(op1 != nullptr); - argInfo.argTmpNum = BAD_VAR_NUM; + op = op->AsField()->GetFldObj(); + } - // We may need to retype to ensure we match the callee's view of the type. - // Otherwise callee-pass throughs of arguments can create return type - // mismatches that block inlining. - // - // Note argument type mismatches that prevent inlining should - // have been caught in impInlineInitVars. - if (op1->TypeGet() != lclTyp) + if (op->OperIs(GT_LCL_VAR_ADDR)) + { + if (lclVarTreeOut != nullptr) { - op1->gtType = genActualType(lclTyp); + *lclVarTreeOut = const_cast(op); } + + return true; } - else if (argInfo.argIsLclVar && !argCanBeModified && !argInfo.argHasCallerLocalRef) - { - // Directly substitute unaliased caller locals for args that cannot be modified - // - // Use the caller-supplied node if this is the first use. - op1 = argNode; - unsigned argLclNum = op1->AsLclVarCommon()->GetLclNum(); - argInfo.argTmpNum = argLclNum; - // Use an equivalent copy if this is the second or subsequent - // use. - // - // Note argument type mismatches that prevent inlining should - // have been caught in impInlineInitVars. If inlining is not prevented - // but a cast is necessary, we similarly expect it to have been inserted then. - // So here we may have argument type mismatches that are benign, for instance - // passing a TYP_SHORT local (eg. normalized-on-load) as a TYP_INT arg. - // The exception is when the inlining means we should start tracking the argument. - if (argInfo.argIsUsed || ((lclTyp == TYP_BYREF) && (op1->TypeGet() != TYP_BYREF))) - { - assert(op1->gtOper == GT_LCL_VAR); - assert(lclNum == op1->AsLclVar()->gtLclILoffs); + return false; +} - // Create a new lcl var node - remember the argument lclNum - op1 = impCreateLocalNode(argLclNum DEBUGARG(op1->AsLclVar()->gtLclILoffs)); - // Start tracking things as a byref if the parameter is a byref. - if (lclTyp == TYP_BYREF) - { - op1->gtType = TYP_BYREF; - } - } +//------------------------------------------------------------------------ +// impMakeDiscretionaryInlineObservations: make observations that help +// determine the profitability of a discretionary inline +// +// Arguments: +// pInlineInfo -- InlineInfo for the inline, or null for the prejit root +// inlineResult -- InlineResult accumulating information about this inline +// +// Notes: +// If inlining or prejitting the root, this method also makes +// various observations about the method that factor into inline +// decisions. It sets `compNativeSizeEstimate` as a side effect. + +void Compiler::impMakeDiscretionaryInlineObservations(InlineInfo* pInlineInfo, InlineResult* inlineResult) +{ + assert((pInlineInfo != nullptr && compIsForInlining()) || // Perform the actual inlining. + (pInlineInfo == nullptr && !compIsForInlining()) // Calculate the static inlining hint for ngen. + ); + + // If we're really inlining, we should just have one result in play. + assert((pInlineInfo == nullptr) || (inlineResult == pInlineInfo->inlineResult)); + + // If this is a "forceinline" method, the JIT probably shouldn't have gone + // to the trouble of estimating the native code size. Even if it did, it + // shouldn't be relying on the result of this method. + assert(inlineResult->GetObservation() == InlineObservation::CALLEE_IS_DISCRETIONARY_INLINE); + + // Note if the caller contains NEWOBJ or NEWARR. + Compiler* rootCompiler = impInlineRoot(); + + if ((rootCompiler->optMethodFlags & OMF_HAS_NEWARRAY) != 0) + { + inlineResult->Note(InlineObservation::CALLER_HAS_NEWARRAY); } - else if (argInfo.argIsByRefToStructLocal && !argInfo.argHasStargOp) + + if ((rootCompiler->optMethodFlags & OMF_HAS_NEWOBJ) != 0) { - /* Argument is a by-ref address to a struct, a normed struct, or its field. - In these cases, don't spill the byref to a local, simply clone the tree and use it. - This way we will increase the chance for this byref to be optimized away by - a subsequent "dereference" operation. + inlineResult->Note(InlineObservation::CALLER_HAS_NEWOBJ); + } - From Dev11 bug #139955: Argument node can also be TYP_I_IMPL if we've bashed the tree - (in impInlineInitVars()), if the arg has argHasLdargaOp as well as argIsByRefToStructLocal. - For example, if the caller is: - ldloca.s V_1 // V_1 is a local struct - call void Test.ILPart::RunLdargaOnPointerArg(int32*) - and the callee being inlined has: - .method public static void RunLdargaOnPointerArg(int32* ptrToInts) cil managed - ldarga.s ptrToInts - call void Test.FourInts::NotInlined_SetExpectedValuesThroughPointerToPointer(int32**) - then we change the argument tree (of "ldloca.s V_1") to TYP_I_IMPL to match the callee signature. We'll - soon afterwards reject the inlining anyway, since the tree we return isn't a GT_LCL_VAR. - */ - assert(argNode->TypeGet() == TYP_BYREF || argNode->TypeGet() == TYP_I_IMPL); - op1 = gtCloneExpr(argNode); + bool calleeIsStatic = (info.compFlags & CORINFO_FLG_STATIC) != 0; + bool isSpecialMethod = (info.compFlags & CORINFO_FLG_CONSTRUCTOR) != 0; + + if (isSpecialMethod) + { + if (calleeIsStatic) + { + inlineResult->Note(InlineObservation::CALLEE_IS_CLASS_CTOR); + } + else + { + inlineResult->Note(InlineObservation::CALLEE_IS_INSTANCE_CTOR); + } } - else + else if (!calleeIsStatic) { - /* Argument is a complex expression - it must be evaluated into a temp */ - - if (argInfo.argHasTmp) + // Callee is an instance method. + // + // Check if the callee has the same 'this' as the root. + if (pInlineInfo != nullptr) { - assert(argInfo.argIsUsed); - assert(argInfo.argTmpNum < lvaCount); + GenTree* thisArg = pInlineInfo->iciCall->AsCall()->gtArgs.GetThisArg()->GetNode(); + assert(thisArg); + bool isSameThis = impIsThis(thisArg); + inlineResult->NoteBool(InlineObservation::CALLSITE_IS_SAME_THIS, isSameThis); + } + } - /* Create a new lcl var node - remember the argument lclNum */ - op1 = gtNewLclvNode(argInfo.argTmpNum, genActualType(lclTyp)); + bool callsiteIsGeneric = (rootCompiler->info.compMethodInfo->args.sigInst.methInstCount != 0) || + (rootCompiler->info.compMethodInfo->args.sigInst.classInstCount != 0); - /* This is the second or later use of the this argument, - so we have to use the temp (instead of the actual arg) */ - argInfo.argBashTmpNode = nullptr; - } - else - { - /* First time use */ - assert(!argInfo.argIsUsed); + bool calleeIsGeneric = (info.compMethodInfo->args.sigInst.methInstCount != 0) || + (info.compMethodInfo->args.sigInst.classInstCount != 0); - /* Reserve a temp for the expression. - * Use a large size node as we may change it later */ + if (!callsiteIsGeneric && calleeIsGeneric) + { + inlineResult->Note(InlineObservation::CALLSITE_NONGENERIC_CALLS_GENERIC); + } - const unsigned tmpNum = lvaGrabTemp(true DEBUGARG("Inlining Arg")); + // Inspect callee's arguments (and the actual values at the callsite for them) + CORINFO_SIG_INFO sig = info.compMethodInfo->args; + CORINFO_ARG_LIST_HANDLE sigArg = sig.args; - lvaTable[tmpNum].lvType = lclTyp; + CallArg* argUse = pInlineInfo == nullptr ? nullptr : pInlineInfo->iciCall->AsCall()->gtArgs.Args().begin().GetArg(); - // For ref types, determine the type of the temp. - if (lclTyp == TYP_REF) - { - if (!argCanBeModified) - { - // If the arg can't be modified in the method - // body, use the type of the value, if - // known. Otherwise, use the declared type. - assert(lvaTable[tmpNum].lvSingleDef == 0); - lvaTable[tmpNum].lvSingleDef = 1; - JITDUMP("Marked V%02u as a single def temp\n", tmpNum); - lvaSetClass(tmpNum, argNode, lclInfo.lclVerTypeInfo.GetClassHandleForObjRef()); - } - else - { - // Arg might be modified, use the declared type of - // the argument. - lvaSetClass(tmpNum, lclInfo.lclVerTypeInfo.GetClassHandleForObjRef()); - } - } + for (unsigned i = 0; i < info.compMethodInfo->args.numArgs; i++) + { + if ((argUse != nullptr) && (argUse->GetWellKnownArg() == WellKnownArg::ThisPointer)) + { + argUse = argUse->GetNext(); + } - assert(!lvaTable[tmpNum].IsAddressExposed()); - if (argInfo.argHasLdargaOp) - { - lvaTable[tmpNum].lvHasLdAddrOp = 1; - } + CORINFO_CLASS_HANDLE sigClass; + CorInfoType corType = strip(info.compCompHnd->getArgType(&sig, sigArg, &sigClass)); + GenTree* argNode = argUse == nullptr ? nullptr : argUse->GetEarlyNode(); - if (lclInfo.lclVerTypeInfo.IsStruct()) + if (corType == CORINFO_TYPE_CLASS) + { + sigClass = info.compCompHnd->getArgClass(&sig, sigArg); + } + else if (corType == CORINFO_TYPE_VALUECLASS) + { + inlineResult->Note(InlineObservation::CALLEE_ARG_STRUCT); + } + else if (corType == CORINFO_TYPE_BYREF) + { + sigClass = info.compCompHnd->getArgClass(&sig, sigArg); + corType = info.compCompHnd->getChildType(sigClass, &sigClass); + } + + if (argNode != nullptr) + { + bool isExact = false; + bool isNonNull = false; + CORINFO_CLASS_HANDLE argCls = gtGetClassHandle(argNode, &isExact, &isNonNull); + if (argCls != nullptr) { - if (varTypeIsStruct(lclTyp)) + const bool isArgValueType = eeIsValueClass(argCls); + // Exact class of the arg is known + if (isExact && !isArgValueType) { - lvaSetStruct(tmpNum, lclInfo.lclVerTypeInfo.GetClassHandle(), true /* unsafe value cls check */); - if (info.compIsVarArgs) + inlineResult->Note(InlineObservation::CALLSITE_ARG_EXACT_CLS); + if ((argCls != sigClass) && (sigClass != nullptr)) { - lvaSetStructUsedAsVarArg(tmpNum); + // .. but the signature accepts a less concrete type. + inlineResult->Note(InlineObservation::CALLSITE_ARG_EXACT_CLS_SIG_IS_NOT); } } - else + // Arg is a reference type in the signature and a boxed value type was passed. + else if (isArgValueType && (corType == CORINFO_TYPE_CLASS)) { - // This is a wrapped primitive. Make sure the verstate knows that - lvaTable[tmpNum].lvVerTypeInfo = lclInfo.lclVerTypeInfo; + inlineResult->Note(InlineObservation::CALLSITE_ARG_BOXED); } } - argInfo.argHasTmp = true; - argInfo.argTmpNum = tmpNum; - - // If we require strict exception order, then arguments must - // be evaluated in sequence before the body of the inlined method. - // So we need to evaluate them to a temp. - // Also, if arguments have global or local references, we need to - // evaluate them to a temp before the inlined body as the - // inlined body may be modifying the global ref. - // TODO-1stClassStructs: We currently do not reuse an existing lclVar - // if it is a struct, because it requires some additional handling. - - if ((!varTypeIsStruct(lclTyp) && !argInfo.argHasSideEff && !argInfo.argHasGlobRef && - !argInfo.argHasCallerLocalRef)) - { - /* Get a *LARGE* LCL_VAR node */ - op1 = gtNewLclLNode(tmpNum, genActualType(lclTyp) DEBUGARG(lclNum)); - - /* Record op1 as the very first use of this argument. - If there are no further uses of the arg, we may be - able to use the actual arg node instead of the temp. - If we do see any further uses, we will clear this. */ - argInfo.argBashTmpNode = op1; - } - else + if (argNode->OperIsConst()) { - /* Get a small LCL_VAR node */ - op1 = gtNewLclvNode(tmpNum, genActualType(lclTyp)); - /* No bashing of this argument */ - argInfo.argBashTmpNode = nullptr; + inlineResult->Note(InlineObservation::CALLSITE_ARG_CONST); } + argUse = argUse->GetNext(); } + sigArg = info.compCompHnd->getArgNext(sigArg); } - // Mark this argument as used. - argInfo.argIsUsed = true; - - return op1; -} + // Note if the callee's return type is a value type + if (info.compMethodInfo->args.retType == CORINFO_TYPE_VALUECLASS) + { + inlineResult->Note(InlineObservation::CALLEE_RETURNS_STRUCT); + } -/****************************************************************************** - Is this the original "this" argument to the call being inlined? + // Note if the callee's class is a promotable struct + if ((info.compClassAttr & CORINFO_FLG_VALUECLASS) != 0) + { + assert(structPromotionHelper != nullptr); + if (structPromotionHelper->CanPromoteStructType(info.compClassHnd)) + { + inlineResult->Note(InlineObservation::CALLEE_CLASS_PROMOTABLE); + } + inlineResult->Note(InlineObservation::CALLEE_CLASS_VALUETYPE); + } - Note that we do not inline methods with "starg 0", and so we do not need to - worry about it. -*/ +#ifdef FEATURE_SIMD -bool Compiler::impInlineIsThis(GenTree* tree, InlArgInfo* inlArgInfo) -{ - assert(compIsForInlining()); - return (tree->gtOper == GT_LCL_VAR && tree->AsLclVarCommon()->GetLclNum() == inlArgInfo[0].argTmpNum); -} + // Note if this method is has SIMD args or return value + if (pInlineInfo != nullptr && pInlineInfo->hasSIMDTypeArgLocalOrReturn) + { + inlineResult->Note(InlineObservation::CALLEE_HAS_SIMD); + } -//----------------------------------------------------------------------------- -// impInlineIsGuaranteedThisDerefBeforeAnySideEffects: Check if a dereference in -// the inlinee can guarantee that the "this" pointer is non-NULL. -// -// Arguments: -// additionalTree - a tree to check for side effects -// additionalCallArgs - a list of call args to check for side effects -// dereferencedAddress - address expression being dereferenced -// inlArgInfo - inlinee argument information -// -// Notes: -// If we haven't hit a branch or a side effect, and we are dereferencing -// from 'this' to access a field or make GTF_CALL_NULLCHECK call, -// then we can avoid a separate null pointer check. -// -// The importer stack and current statement list are searched for side effects. -// Trees that have been popped of the stack but haven't been appended to the -// statement list and have to be checked for side effects may be provided via -// additionalTree and additionalCallArgs. -// -bool Compiler::impInlineIsGuaranteedThisDerefBeforeAnySideEffects(GenTree* additionalTree, - CallArgs* additionalCallArgs, - GenTree* dereferencedAddress, - InlArgInfo* inlArgInfo) -{ - assert(compIsForInlining()); - assert(opts.OptEnabled(CLFLG_INLINING)); +#endif // FEATURE_SIMD - BasicBlock* block = compCurBB; + // Roughly classify callsite frequency. + InlineCallsiteFrequency frequency = InlineCallsiteFrequency::UNUSED; - if (block != fgFirstBB) + // If this is a prejit root, or a maximally hot block... + if ((pInlineInfo == nullptr) || (pInlineInfo->iciBlock->isMaxBBWeight())) { - return false; + frequency = InlineCallsiteFrequency::HOT; } - - if (!impInlineIsThis(dereferencedAddress, inlArgInfo)) + // No training data. Look for loop-like things. + // We consider a recursive call loop-like. Do not give the inlining boost to the method itself. + // However, give it to things nearby. + else if ((pInlineInfo->iciBlock->bbFlags & BBF_BACKWARD_JUMP) && + (pInlineInfo->fncHandle != pInlineInfo->inlineCandidateInfo->ilCallerHandle)) { - return false; + frequency = InlineCallsiteFrequency::LOOP; } - - if ((additionalTree != nullptr) && GTF_GLOBALLY_VISIBLE_SIDE_EFFECTS(additionalTree->gtFlags)) + else if (pInlineInfo->iciBlock->hasProfileWeight() && (pInlineInfo->iciBlock->bbWeight > BB_ZERO_WEIGHT)) { - return false; + frequency = InlineCallsiteFrequency::WARM; } - - if (additionalCallArgs != nullptr) + // Now modify the multiplier based on where we're called from. + else if (pInlineInfo->iciBlock->isRunRarely() || ((info.compFlags & FLG_CCTOR) == FLG_CCTOR)) { - for (CallArg& arg : additionalCallArgs->Args()) - { - if (GTF_GLOBALLY_VISIBLE_SIDE_EFFECTS(arg.GetEarlyNode()->gtFlags)) - { - return false; - } - } + frequency = InlineCallsiteFrequency::RARE; } - - for (Statement* stmt : StatementList(impStmtList)) + else { - GenTree* expr = stmt->GetRootNode(); - if (GTF_GLOBALLY_VISIBLE_SIDE_EFFECTS(expr->gtFlags)) - { - return false; - } + frequency = InlineCallsiteFrequency::BORING; } - for (unsigned level = 0; level < verCurrentState.esStackDepth; level++) + // Also capture the block weight of the call site. + // + // In the prejit root case, assume at runtime there might be a hot call site + // for this method, so we won't prematurely conclude this method should never + // be inlined. + // + weight_t weight = 0; + + if (pInlineInfo != nullptr) { - GenTreeFlags stackTreeFlags = verCurrentState.esStack[level].val->gtFlags; - if (GTF_GLOBALLY_VISIBLE_SIDE_EFFECTS(stackTreeFlags)) - { - return false; - } + weight = pInlineInfo->iciBlock->bbWeight; + } + else + { + const weight_t prejitHotCallerWeight = 1000000.0; + weight = prejitHotCallerWeight; } - return true; -} + inlineResult->NoteInt(InlineObservation::CALLSITE_FREQUENCY, static_cast(frequency)); + inlineResult->NoteInt(InlineObservation::CALLSITE_WEIGHT, (int)(weight)); -//------------------------------------------------------------------------ -// impMarkInlineCandidate: determine if this call can be subsequently inlined -// -// Arguments: -// callNode -- call under scrutiny -// exactContextHnd -- context handle for inlining -// exactContextNeedsRuntimeLookup -- true if context required runtime lookup -// callInfo -- call info from VM -// ilOffset -- the actual IL offset of the instruction that produced this inline candidate -// -// Notes: -// Mostly a wrapper for impMarkInlineCandidateHelper that also undoes -// guarded devirtualization for virtual calls where the method we'd -// devirtualize to cannot be inlined. - -void Compiler::impMarkInlineCandidate(GenTree* callNode, - CORINFO_CONTEXT_HANDLE exactContextHnd, - bool exactContextNeedsRuntimeLookup, - CORINFO_CALL_INFO* callInfo, - IL_OFFSET ilOffset) -{ - GenTreeCall* call = callNode->AsCall(); + bool hasProfile = false; + double profileFreq = 0.0; - // Do the actual evaluation - impMarkInlineCandidateHelper(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, ilOffset); + // If the call site has profile data, report the relative frequency of the site. + // + if ((pInlineInfo != nullptr) && rootCompiler->fgHaveSufficientProfileWeights()) + { + const weight_t callSiteWeight = pInlineInfo->iciBlock->bbWeight; + const weight_t entryWeight = rootCompiler->fgFirstBB->bbWeight; + profileFreq = fgProfileWeightsEqual(entryWeight, 0.0) ? 0.0 : callSiteWeight / entryWeight; + hasProfile = true; - // If this call is an inline candidate or is not a guarded devirtualization - // candidate, we're done. - if (call->IsInlineCandidate() || !call->IsGuardedDevirtualizationCandidate()) + assert(callSiteWeight >= 0); + assert(entryWeight >= 0); + } + else if (pInlineInfo == nullptr) { - return; + // Simulate a hot callsite for PrejitRoot mode. + hasProfile = true; + profileFreq = 1.0; } - // If we can't inline the call we'd guardedly devirtualize to, - // we undo the guarded devirtualization, as the benefit from - // just guarded devirtualization alone is likely not worth the - // extra jit time and code size. - // - // TODO: it is possibly interesting to allow this, but requires - // fixes elsewhere too... - JITDUMP("Revoking guarded devirtualization candidacy for call [%06u]: target method can't be inlined\n", - dspTreeID(call)); - - call->ClearGuardedDevirtualizationCandidate(); + inlineResult->NoteBool(InlineObservation::CALLSITE_HAS_PROFILE_WEIGHTS, hasProfile); + inlineResult->NoteDouble(InlineObservation::CALLSITE_PROFILE_FREQUENCY, profileFreq); } //------------------------------------------------------------------------ -// impMarkInlineCandidateHelper: determine if this call can be subsequently -// inlined +// impCanInlineIL: screen inline candate based on info from the method header // // Arguments: -// callNode -- call under scrutiny -// exactContextHnd -- context handle for inlining -// exactContextNeedsRuntimeLookup -- true if context required runtime lookup -// callInfo -- call info from VM -// ilOffset -- IL offset of instruction creating the inline candidate +// fncHandle -- inline candidate method +// methInfo -- method info from VN +// forceInline -- true if method is marked with AggressiveInlining +// inlineResult -- ongoing inline evaluation // -// Notes: -// If callNode is an inline candidate, this method sets the flag -// GTF_CALL_INLINE_CANDIDATE, and ensures that helper methods have -// filled in the associated InlineCandidateInfo. -// -// If callNode is not an inline candidate, and the reason is -// something that is inherent to the method being called, the -// method may be marked as "noinline" to short-circuit any -// future assessments of calls to this method. - -void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, - CORINFO_CONTEXT_HANDLE exactContextHnd, - bool exactContextNeedsRuntimeLookup, - CORINFO_CALL_INFO* callInfo, - IL_OFFSET ilOffset) +void Compiler::impCanInlineIL(CORINFO_METHOD_HANDLE fncHandle, + CORINFO_METHOD_INFO* methInfo, + bool forceInline, + InlineResult* inlineResult) { - // Let the strategy know there's another call - impInlineRoot()->m_inlineStrategy->NoteCall(); + unsigned codeSize = methInfo->ILCodeSize; + + // We shouldn't have made up our minds yet... + assert(!inlineResult->IsDecided()); - if (!opts.OptEnabled(CLFLG_INLINING)) + if (methInfo->EHcount) { - /* XXX Mon 8/18/2008 - * This assert is misleading. The caller does not ensure that we have CLFLG_INLINING set before - * calling impMarkInlineCandidate. However, if this assert trips it means that we're an inlinee and - * CLFLG_MINOPT is set. That doesn't make a lot of sense. If you hit this assert, work back and - * figure out why we did not set MAXOPT for this compile. - */ - assert(!compIsForInlining()); + inlineResult->NoteFatal(InlineObservation::CALLEE_HAS_EH); return; } - if (compIsForImportOnly()) + if ((methInfo->ILCode == nullptr) || (codeSize == 0)) { - // Don't bother creating the inline candidate during verification. - // Otherwise the call to info.compCompHnd->canInline will trigger a recursive verification - // that leads to the creation of multiple instances of Compiler. + inlineResult->NoteFatal(InlineObservation::CALLEE_HAS_NO_BODY); return; } - InlineResult inlineResult(this, call, nullptr, "impMarkInlineCandidate"); + // For now we don't inline varargs (import code can't handle it) - // Don't inline if not optimizing root method - if (opts.compDbgCode) + if (methInfo->args.isVarArg()) { - inlineResult.NoteFatal(InlineObservation::CALLER_DEBUG_CODEGEN); + inlineResult->NoteFatal(InlineObservation::CALLEE_HAS_MANAGED_VARARGS); return; } - // Don't inline if inlining into this method is disabled. - if (impInlineRoot()->m_inlineStrategy->IsInliningDisabled()) - { - inlineResult.NoteFatal(InlineObservation::CALLER_IS_JIT_NOINLINE); - return; - } + // Reject if it has too many locals. + // This is currently an implementation limit due to fixed-size arrays in the + // inline info, rather than a performance heuristic. - // Don't inline into callers that use the NextCallReturnAddress intrinsic. - if (info.compHasNextCallRetAddr) - { - inlineResult.NoteFatal(InlineObservation::CALLER_USES_NEXT_CALL_RET_ADDR); - return; - } + inlineResult->NoteInt(InlineObservation::CALLEE_NUMBER_OF_LOCALS, methInfo->locals.numArgs); - // Inlining candidate determination needs to honor only IL tail prefix. - // Inlining takes precedence over implicit tail call optimization (if the call is not directly recursive). - if (call->IsTailPrefixedCall()) + if (methInfo->locals.numArgs > MAX_INL_LCLS) { - inlineResult.NoteFatal(InlineObservation::CALLSITE_EXPLICIT_TAIL_PREFIX); + inlineResult->NoteFatal(InlineObservation::CALLEE_TOO_MANY_LOCALS); return; } - // Delegate Invoke method doesn't have a body and gets special cased instead. - // Don't even bother trying to inline it. - if (call->IsDelegateInvoke() && !call->IsGuardedDevirtualizationCandidate()) - { - inlineResult.NoteFatal(InlineObservation::CALLEE_HAS_NO_BODY); - return; - } + // Make sure there aren't too many arguments. + // This is currently an implementation limit due to fixed-size arrays in the + // inline info, rather than a performance heuristic. + + inlineResult->NoteInt(InlineObservation::CALLEE_NUMBER_OF_ARGUMENTS, methInfo->args.numArgs); - // Tail recursion elimination takes precedence over inlining. - // TODO: We may want to do some of the additional checks from fgMorphCall - // here to reduce the chance we don't inline a call that won't be optimized - // as a fast tail call or turned into a loop. - if (gtIsRecursiveCall(call) && call->IsImplicitTailCall()) + if (methInfo->args.numArgs > MAX_INL_ARGS) { - inlineResult.NoteFatal(InlineObservation::CALLSITE_IMPLICIT_REC_TAIL_CALL); + inlineResult->NoteFatal(InlineObservation::CALLEE_TOO_MANY_ARGUMENTS); return; } - if (call->IsVirtual()) - { - // Allow guarded devirt calls to be treated as inline candidates, - // but reject all other virtual calls. - if (!call->IsGuardedDevirtualizationCandidate()) - { - inlineResult.NoteFatal(InlineObservation::CALLSITE_IS_NOT_DIRECT); - return; - } - } + // Note force inline state + + inlineResult->NoteBool(InlineObservation::CALLEE_IS_FORCE_INLINE, forceInline); + + // Note IL code size - /* Ignore helper calls */ + inlineResult->NoteInt(InlineObservation::CALLEE_IL_CODE_SIZE, codeSize); - if (call->gtCallType == CT_HELPER) + if (inlineResult->IsFailure()) { - assert(!call->IsGuardedDevirtualizationCandidate()); - inlineResult.NoteFatal(InlineObservation::CALLSITE_IS_CALL_TO_HELPER); return; } - /* Ignore indirect calls */ - if (call->gtCallType == CT_INDIRECT) + // Make sure maxstack is not too big + + inlineResult->NoteInt(InlineObservation::CALLEE_MAXSTACK, methInfo->maxStack); + + if (inlineResult->IsFailure()) { - inlineResult.NoteFatal(InlineObservation::CALLSITE_IS_NOT_DIRECT_MANAGED); return; } +} - /* I removed the check for BBJ_THROW. BBJ_THROW is usually marked as rarely run. This more or less - * restricts the inliner to non-expanding inlines. I removed the check to allow for non-expanding - * inlining in throw blocks. I should consider the same thing for catch and filter regions. */ +//------------------------------------------------------------------------ +// impInlineRecordArgInfo: record information about an inline candidate argument +// +// Arguments: +// pInlineInfo - inline info for the inline candidate +// arg - the caller argument +// argNum - logical index of this argument +// inlineResult - result of ongoing inline evaluation +// +// Notes: +// +// Checks for various inline blocking conditions and makes notes in +// the inline info arg table about the properties of the actual. These +// properties are used later by impInlineFetchArg to determine how best to +// pass the argument into the inlinee. - CORINFO_METHOD_HANDLE fncHandle; - unsigned methAttr; +void Compiler::impInlineRecordArgInfo(InlineInfo* pInlineInfo, + CallArg* arg, + unsigned argNum, + InlineResult* inlineResult) +{ + InlArgInfo* inlCurArgInfo = &pInlineInfo->inlArgInfo[argNum]; - if (call->IsGuardedDevirtualizationCandidate()) - { - if (call->gtGuardedDevirtualizationCandidateInfo->guardedMethodUnboxedEntryHandle != nullptr) - { - fncHandle = call->gtGuardedDevirtualizationCandidateInfo->guardedMethodUnboxedEntryHandle; - } - else - { - fncHandle = call->gtGuardedDevirtualizationCandidateInfo->guardedMethodHandle; - } - methAttr = info.compCompHnd->getMethodAttribs(fncHandle); - } - else - { - fncHandle = call->gtCallMethHnd; + inlCurArgInfo->arg = arg; + GenTree* curArgVal = arg->GetNode(); - // Reuse method flags from the original callInfo if possible - if (fncHandle == callInfo->hMethod) - { - methAttr = callInfo->methodFlags; - } - else - { - methAttr = info.compCompHnd->getMethodAttribs(fncHandle); - } - } + assert(!curArgVal->OperIs(GT_RET_EXPR)); -#ifdef DEBUG - if (compStressCompile(STRESS_FORCE_INLINE, 0)) + if (curArgVal->gtOper == GT_MKREFANY) { - methAttr |= CORINFO_FLG_FORCEINLINE; + inlineResult->NoteFatal(InlineObservation::CALLSITE_ARG_IS_MKREFANY); + return; } -#endif - // Check for COMPlus_AggressiveInlining - if (compDoAggressiveInlining) + GenTree* lclVarTree; + const bool isAddressInLocal = impIsAddressInLocal(curArgVal, &lclVarTree); + if (isAddressInLocal) { - methAttr |= CORINFO_FLG_FORCEINLINE; - } + LclVarDsc* varDsc = lvaGetDesc(lclVarTree->AsLclVarCommon()); - if (!(methAttr & CORINFO_FLG_FORCEINLINE)) - { - /* Don't bother inline blocks that are in the filter region */ - if (bbInCatchHandlerILRange(compCurBB)) + if (varTypeIsStruct(varDsc)) { -#ifdef DEBUG - if (verbose) + inlCurArgInfo->argIsByRefToStructLocal = true; +#ifdef FEATURE_SIMD + if (varDsc->lvSIMDType) { - printf("\nWill not inline blocks that are in the catch handler region\n"); + pInlineInfo->hasSIMDTypeArgLocalOrReturn = true; } - -#endif - - inlineResult.NoteFatal(InlineObservation::CALLSITE_IS_WITHIN_CATCH); - return; +#endif // FEATURE_SIMD } - if (bbInFilterILRange(compCurBB)) - { -#ifdef DEBUG - if (verbose) - { - printf("\nWill not inline blocks that are in the filter region\n"); - } -#endif - - inlineResult.NoteFatal(InlineObservation::CALLSITE_IS_WITHIN_FILTER); - return; - } + // Spilling code relies on correct aliasability annotations. + assert(varDsc->lvHasLdAddrOp || varDsc->IsAddressExposed()); } - /* Check if we tried to inline this method before */ - - if (methAttr & CORINFO_FLG_DONT_INLINE) + if (curArgVal->gtFlags & GTF_ALL_EFFECT) { - inlineResult.NoteFatal(InlineObservation::CALLEE_IS_NOINLINE); - return; + inlCurArgInfo->argHasGlobRef = (curArgVal->gtFlags & GTF_GLOB_REF) != 0; + inlCurArgInfo->argHasSideEff = (curArgVal->gtFlags & (GTF_ALL_EFFECT & ~GTF_GLOB_REF)) != 0; } - /* Cannot inline synchronized methods */ - - if (methAttr & CORINFO_FLG_SYNCH) + if (curArgVal->gtOper == GT_LCL_VAR) { - inlineResult.NoteFatal(InlineObservation::CALLEE_IS_SYNCHRONIZED); - return; - } + inlCurArgInfo->argIsLclVar = true; - /* Check legality of PInvoke callsite (for inlining of marshalling code) */ + /* Remember the "original" argument number */ + INDEBUG(curArgVal->AsLclVar()->gtLclILoffs = argNum;) + } - if (methAttr & CORINFO_FLG_PINVOKE) + if (impIsInvariant(curArgVal)) { - // See comment in impCheckForPInvokeCall - BasicBlock* block = compIsForInlining() ? impInlineInfo->iciBlock : compCurBB; - if (!impCanPInvokeInlineCallSite(block)) + inlCurArgInfo->argIsInvariant = true; + if (inlCurArgInfo->argIsThis && (curArgVal->gtOper == GT_CNS_INT) && (curArgVal->AsIntCon()->gtIconVal == 0)) { - inlineResult.NoteFatal(InlineObservation::CALLSITE_PINVOKE_EH); + // Abort inlining at this call site + inlineResult->NoteFatal(InlineObservation::CALLSITE_ARG_HAS_NULL_THIS); return; } } - - InlineCandidateInfo* inlineCandidateInfo = nullptr; - impCheckCanInline(call, fncHandle, methAttr, exactContextHnd, &inlineCandidateInfo, &inlineResult); - - if (inlineResult.IsFailure()) + else if (gtIsTypeof(curArgVal)) { - return; + inlCurArgInfo->argIsInvariant = true; + inlCurArgInfo->argHasSideEff = false; } - // The old value should be null OR this call should be a guarded devirtualization candidate. - assert((call->gtInlineCandidateInfo == nullptr) || call->IsGuardedDevirtualizationCandidate()); - - // The new value should not be null. - assert(inlineCandidateInfo != nullptr); - inlineCandidateInfo->exactContextNeedsRuntimeLookup = exactContextNeedsRuntimeLookup; - inlineCandidateInfo->ilOffset = ilOffset; - call->gtInlineCandidateInfo = inlineCandidateInfo; + bool isExact = false; + bool isNonNull = false; + inlCurArgInfo->argIsExact = (gtGetClassHandle(curArgVal, &isExact, &isNonNull) != NO_CLASS_HANDLE) && isExact; - // If we're in an inlinee compiler, and have a return spill temp, and this inline candidate - // is also a tail call candidate, it can use the same return spill temp. + // If the arg is a local that is address-taken, we can't safely + // directly substitute it into the inlinee. // - if (compIsForInlining() && call->CanTailCall() && - (impInlineInfo->inlineCandidateInfo->preexistingSpillTemp != BAD_VAR_NUM)) - { - inlineCandidateInfo->preexistingSpillTemp = impInlineInfo->inlineCandidateInfo->preexistingSpillTemp; - JITDUMP("Inline candidate [%06u] can share spill temp V%02u\n", dspTreeID(call), - inlineCandidateInfo->preexistingSpillTemp); - } - - // Mark the call node as inline candidate. - call->gtFlags |= GTF_CALL_INLINE_CANDIDATE; - - // Let the strategy know there's another candidate. - impInlineRoot()->m_inlineStrategy->NoteCandidate(); - - // Since we're not actually inlining yet, and this call site is - // still just an inline candidate, there's nothing to report. - inlineResult.SetSuccessResult(INLINE_CHECK_CAN_INLINE_SUCCESS); -} - -/******************************************************************************/ -// Returns true if the given intrinsic will be implemented by target-specific -// instructions - -bool Compiler::IsTargetIntrinsic(NamedIntrinsic intrinsicName) -{ -#if defined(TARGET_XARCH) - switch (intrinsicName) + // Previously we'd accomplish this by setting "argHasLdargaOp" but + // that has a stronger meaning: that the arg value can change in + // the method body. Using that flag prevents type propagation, + // which is safe in this case. + // + // Instead mark the arg as having a caller local ref. + if (!inlCurArgInfo->argIsInvariant && gtHasLocalsWithAddrOp(curArgVal)) { - // AMD64/x86 has SSE2 instructions to directly compute sqrt/abs and SSE4.1 - // instructions to directly compute round/ceiling/floor/truncate. - - case NI_System_Math_Abs: - case NI_System_Math_Sqrt: - return true; - - case NI_System_Math_Ceiling: - case NI_System_Math_Floor: - case NI_System_Math_Truncate: - case NI_System_Math_Round: - return compOpportunisticallyDependsOn(InstructionSet_SSE41); - - case NI_System_Math_FusedMultiplyAdd: - return compOpportunisticallyDependsOn(InstructionSet_FMA); - - default: - return false; + inlCurArgInfo->argHasCallerLocalRef = true; } -#elif defined(TARGET_ARM64) - switch (intrinsicName) - { - case NI_System_Math_Abs: - case NI_System_Math_Ceiling: - case NI_System_Math_Floor: - case NI_System_Math_Truncate: - case NI_System_Math_Round: - case NI_System_Math_Sqrt: - case NI_System_Math_Max: - case NI_System_Math_Min: - return true; - - case NI_System_Math_FusedMultiplyAdd: - return compOpportunisticallyDependsOn(InstructionSet_AdvSimd); - default: - return false; - } -#elif defined(TARGET_ARM) - switch (intrinsicName) +#ifdef DEBUG + if (verbose) { - case NI_System_Math_Abs: - case NI_System_Math_Round: - case NI_System_Math_Sqrt: - return true; - - default: - return false; - } -#elif defined(TARGET_LOONGARCH64) - // TODO-LoongArch64: add some instrinsics. - return false; -#else - // TODO: This portion of logic is not implemented for other arch. - // The reason for returning true is that on all other arch the only intrinsic - // enabled are target intrinsics. - return true; -#endif -} - -/******************************************************************************/ -// Returns true if the given intrinsic will be implemented by calling System.Math -// methods. - -bool Compiler::IsIntrinsicImplementedByUserCall(NamedIntrinsic intrinsicName) -{ - // Currently, if a math intrinsic is not implemented by target-specific - // instructions, it will be implemented by a System.Math call. In the - // future, if we turn to implementing some of them with helper calls, - // this predicate needs to be revisited. - return !IsTargetIntrinsic(intrinsicName); -} - -bool Compiler::IsMathIntrinsic(NamedIntrinsic intrinsicName) -{ - switch (intrinsicName) - { - case NI_System_Math_Abs: - case NI_System_Math_Acos: - case NI_System_Math_Acosh: - case NI_System_Math_Asin: - case NI_System_Math_Asinh: - case NI_System_Math_Atan: - case NI_System_Math_Atanh: - case NI_System_Math_Atan2: - case NI_System_Math_Cbrt: - case NI_System_Math_Ceiling: - case NI_System_Math_Cos: - case NI_System_Math_Cosh: - case NI_System_Math_Exp: - case NI_System_Math_Floor: - case NI_System_Math_FMod: - case NI_System_Math_FusedMultiplyAdd: - case NI_System_Math_ILogB: - case NI_System_Math_Log: - case NI_System_Math_Log2: - case NI_System_Math_Log10: - case NI_System_Math_Max: - case NI_System_Math_Min: - case NI_System_Math_Pow: - case NI_System_Math_Round: - case NI_System_Math_Sin: - case NI_System_Math_Sinh: - case NI_System_Math_Sqrt: - case NI_System_Math_Tan: - case NI_System_Math_Tanh: - case NI_System_Math_Truncate: + if (inlCurArgInfo->argIsThis) + { + printf("thisArg:"); + } + else + { + printf("\nArgument #%u:", argNum); + } + if (inlCurArgInfo->argIsLclVar) + { + printf(" is a local var"); + } + if (inlCurArgInfo->argIsInvariant) + { + printf(" is a constant or invariant"); + } + if (inlCurArgInfo->argHasGlobRef) + { + printf(" has global refs"); + } + if (inlCurArgInfo->argHasCallerLocalRef) + { + printf(" has caller local ref"); + } + if (inlCurArgInfo->argHasSideEff) { - assert((intrinsicName > NI_SYSTEM_MATH_START) && (intrinsicName < NI_SYSTEM_MATH_END)); - return true; + printf(" has side effects"); } - - default: + if (inlCurArgInfo->argHasLdargaOp) { - assert((intrinsicName < NI_SYSTEM_MATH_START) || (intrinsicName > NI_SYSTEM_MATH_END)); - return false; + printf(" has ldarga effect"); + } + if (inlCurArgInfo->argHasStargOp) + { + printf(" has starg effect"); + } + if (inlCurArgInfo->argIsByRefToStructLocal) + { + printf(" is byref to a struct local"); } - } -} -bool Compiler::IsMathIntrinsic(GenTree* tree) -{ - return (tree->OperGet() == GT_INTRINSIC) && IsMathIntrinsic(tree->AsIntrinsic()->gtIntrinsicName); + printf("\n"); + gtDispTree(curArgVal); + printf("\n"); + } +#endif } //------------------------------------------------------------------------ -// impDevirtualizeCall: Attempt to change a virtual vtable call into a -// normal call +// impInlineInitVars: setup inline information for inlinee args and locals // // Arguments: -// call -- the call node to examine/modify -// pResolvedToken -- [IN] the resolved token used to create the call. Used for R2R. -// method -- [IN/OUT] the method handle for call. Updated iff call devirtualized. -// methodFlags -- [IN/OUT] flags for the method to call. Updated iff call devirtualized. -// pContextHandle -- [IN/OUT] context handle for the call. Updated iff call devirtualized. -// pExactContextHandle -- [OUT] updated context handle iff call devirtualized -// isLateDevirtualization -- if devirtualization is happening after importation -// isExplicitTailCalll -- [IN] true if we plan on using an explicit tail call -// ilOffset -- IL offset of the call +// pInlineInfo - inline info for the inline candidate // // Notes: -// Virtual calls in IL will always "invoke" the base class method. -// -// This transformation looks for evidence that the type of 'this' -// in the call is exactly known, is a final class or would invoke -// a final method, and if that and other safety checks pan out, -// modifies the call and the call info to create a direct call. -// -// This transformation is initially done in the importer and not -// in some subsequent optimization pass because we want it to be -// upstream of inline candidate identification. -// -// However, later phases may supply improved type information that -// can enable further devirtualization. We currently reinvoke this -// code after inlining, if the return value of the inlined call is -// the 'this obj' of a subsequent virtual call. +// This method primarily adds caller-supplied info to the inlArgInfo +// and sets up the lclVarInfo table. // -// If devirtualization succeeds and the call's this object is a -// (boxed) value type, the jit will ask the EE for the unboxed entry -// point. If this exists, the jit will invoke the unboxed entry -// on the box payload. In addition if the boxing operation is -// visible to the jit and the call is the only consmer of the box, -// the jit will try analyze the box to see if the call can be instead -// instead made on a local copy. If that is doable, the call is -// updated to invoke the unboxed entry on the local copy and the -// boxing operation is removed. +// For args, the inlArgInfo records properties of the actual argument +// including the tree node that produces the arg value. This node is +// usually the tree node present at the call, but may also differ in +// various ways: +// - when the call arg is a GT_RET_EXPR, we search back through the ret +// expr chain for the actual node. Note this will either be the original +// call (which will be a failed inline by this point), or the return +// expression from some set of inlines. +// - when argument type casting is needed the necessary casts are added +// around the argument node. +// - if an argument can be simplified by folding then the node here is the +// folded value. // -// When guarded devirtualization is enabled, this method will mark -// calls as guarded devirtualization candidates, if the type of `this` -// is not exactly known, and there is a plausible guess for the type. -void Compiler::impDevirtualizeCall(GenTreeCall* call, - CORINFO_RESOLVED_TOKEN* pResolvedToken, - CORINFO_METHOD_HANDLE* method, - unsigned* methodFlags, - CORINFO_CONTEXT_HANDLE* pContextHandle, - CORINFO_CONTEXT_HANDLE* pExactContextHandle, - bool isLateDevirtualization, - bool isExplicitTailCall, - IL_OFFSET ilOffset) -{ - assert(call != nullptr); - assert(method != nullptr); - assert(methodFlags != nullptr); - assert(pContextHandle != nullptr); - - // This should be a virtual vtable or virtual stub call. - // - assert(call->IsVirtual()); - assert(opts.OptimizationEnabled()); +// The method may make observations that lead to marking this candidate as +// a failed inline. If this happens the initialization is abandoned immediately +// to try and reduce the jit time cost for a failed inline. -#if defined(DEBUG) - // Bail if devirt is disabled. - if (JitConfig.JitEnableDevirtualization() == 0) - { - return; - } +void Compiler::impInlineInitVars(InlineInfo* pInlineInfo) +{ + assert(!compIsForInlining()); - // Optionally, print info on devirtualization - Compiler* const rootCompiler = impInlineRoot(); - const bool doPrint = JitConfig.JitPrintDevirtualizedMethods().contains(rootCompiler->info.compMethodName, - rootCompiler->info.compClassName, - &rootCompiler->info.compMethodInfo->args); -#endif // DEBUG + GenTreeCall* call = pInlineInfo->iciCall; + CORINFO_METHOD_INFO* methInfo = &pInlineInfo->inlineCandidateInfo->methInfo; + unsigned clsAttr = pInlineInfo->inlineCandidateInfo->clsAttr; + InlArgInfo* inlArgInfo = pInlineInfo->inlArgInfo; + InlLclVarInfo* lclVarInfo = pInlineInfo->lclVarInfo; + InlineResult* inlineResult = pInlineInfo->inlineResult; - // Fetch information about the virtual method we're calling. - CORINFO_METHOD_HANDLE baseMethod = *method; - unsigned baseMethodAttribs = *methodFlags; + /* init the argument struct */ + memset(inlArgInfo, 0, (MAX_INL_ARGS + 1) * sizeof(inlArgInfo[0])); - if (baseMethodAttribs == 0) - { - // For late devirt we may not have method attributes, so fetch them. - baseMethodAttribs = info.compCompHnd->getMethodAttribs(baseMethod); - } - else + unsigned ilArgCnt = 0; + for (CallArg& arg : call->gtArgs.Args()) { -#if defined(DEBUG) - // Validate that callInfo has up to date method flags - const DWORD freshBaseMethodAttribs = info.compCompHnd->getMethodAttribs(baseMethod); - - // All the base method attributes should agree, save that - // CORINFO_FLG_DONT_INLINE may have changed from 0 to 1 - // because of concurrent jitting activity. - // - // Note we don't look at this particular flag bit below, and - // later on (if we do try and inline) we will rediscover why - // the method can't be inlined, so there's no danger here in - // seeing this particular flag bit in different states between - // the cached and fresh values. - if ((freshBaseMethodAttribs & ~CORINFO_FLG_DONT_INLINE) != (baseMethodAttribs & ~CORINFO_FLG_DONT_INLINE)) + switch (arg.GetWellKnownArg()) { - assert(!"mismatched method attributes"); + case WellKnownArg::ThisPointer: + inlArgInfo[ilArgCnt].argIsThis = true; + break; + case WellKnownArg::RetBuffer: + case WellKnownArg::InstParam: + // These do not appear in the table of inline arg info; do not include them + continue; + default: + break; } -#endif // DEBUG - } - - // In R2R mode, we might see virtual stub calls to - // non-virtuals. For instance cases where the non-virtual method - // is in a different assembly but is called via CALLVIRT. For - // verison resilience we must allow for the fact that the method - // might become virtual in some update. - // - // In non-R2R modes CALLVIRT will be turned into a - // regular call+nullcheck upstream, so we won't reach this - // point. - if ((baseMethodAttribs & CORINFO_FLG_VIRTUAL) == 0) - { - assert(call->IsVirtualStub()); - assert(opts.IsReadyToRun()); - JITDUMP("\nimpDevirtualizeCall: [R2R] base method not virtual, sorry\n"); - return; - } - - // Fetch information about the class that introduced the virtual method. - CORINFO_CLASS_HANDLE baseClass = info.compCompHnd->getMethodClass(baseMethod); - const DWORD baseClassAttribs = info.compCompHnd->getClassAttribs(baseClass); - - // Is the call an interface call? - const bool isInterface = (baseClassAttribs & CORINFO_FLG_INTERFACE) != 0; - - // See what we know about the type of 'this' in the call. - assert(call->gtArgs.HasThisPointer()); - CallArg* thisArg = call->gtArgs.GetThisArg(); - GenTree* thisObj = thisArg->GetEarlyNode()->gtEffectiveVal(false); - bool isExact = false; - bool objIsNonNull = false; - CORINFO_CLASS_HANDLE objClass = gtGetClassHandle(thisObj, &isExact, &objIsNonNull); - // Bail if we know nothing. - if (objClass == NO_CLASS_HANDLE) - { - JITDUMP("\nimpDevirtualizeCall: no type available (op=%s)\n", GenTree::OpName(thisObj->OperGet())); + arg.SetEarlyNode(gtFoldExpr(arg.GetEarlyNode())); + impInlineRecordArgInfo(pInlineInfo, &arg, ilArgCnt, inlineResult); - // Don't try guarded devirtualiztion when we're doing late devirtualization. - // - if (isLateDevirtualization) + if (inlineResult->IsFailure()) { - JITDUMP("No guarded devirt during late devirtualization\n"); return; } - considerGuardedDevirtualization(call, ilOffset, isInterface, baseMethod, baseClass, pContextHandle); - - return; + ilArgCnt++; } - // If the objClass is sealed (final), then we may be able to devirtualize. - const DWORD objClassAttribs = info.compCompHnd->getClassAttribs(objClass); - const bool objClassIsFinal = (objClassAttribs & CORINFO_FLG_FINAL) != 0; + /* Make sure we got the arg number right */ + assert(ilArgCnt == methInfo->args.totalILArgs()); + +#ifdef FEATURE_SIMD + bool foundSIMDType = pInlineInfo->hasSIMDTypeArgLocalOrReturn; +#endif // FEATURE_SIMD -#if defined(DEBUG) - const char* callKind = isInterface ? "interface" : "virtual"; - const char* objClassNote = "[?]"; - const char* objClassName = "?objClass"; - const char* baseClassName = "?baseClass"; - const char* baseMethodName = "?baseMethod"; + /* We have typeless opcodes, get type information from the signature */ - if (verbose || doPrint) + CallArg* thisArg = call->gtArgs.GetThisArg(); + if (thisArg != nullptr) { - objClassNote = isExact ? " [exact]" : objClassIsFinal ? " [final]" : ""; - objClassName = eeGetClassName(objClass); - baseClassName = eeGetClassName(baseClass); - baseMethodName = eeGetMethodName(baseMethod, nullptr); + lclVarInfo[0].lclVerTypeInfo = verMakeTypeInfo(pInlineInfo->inlineCandidateInfo->clsHandle); + lclVarInfo[0].lclHasLdlocaOp = false; - if (verbose) +#ifdef FEATURE_SIMD + // We always want to check isSIMDClass, since we want to set foundSIMDType (to increase + // the inlining multiplier) for anything in that assembly. + // But we only need to normalize it if it is a TYP_STRUCT + // (which we need to do even if we have already set foundSIMDType). + if (!foundSIMDType && isSIMDorHWSIMDClass(&(lclVarInfo[0].lclVerTypeInfo))) { - printf("\nimpDevirtualizeCall: Trying to devirtualize %s call:\n" - " class for 'this' is %s%s (attrib %08x)\n" - " base method is %s::%s\n", - callKind, objClassName, objClassNote, objClassAttribs, baseClassName, baseMethodName); + foundSIMDType = true; } - } -#endif // defined(DEBUG) +#endif // FEATURE_SIMD - // See if the jit's best type for `obj` is an interface. - // See for instance System.ValueTuple`8::GetHashCode, where lcl 0 is System.IValueTupleInternal - // IL_021d: ldloc.0 - // IL_021e: callvirt instance int32 System.Object::GetHashCode() - // - // If so, we can't devirtualize, but we may be able to do guarded devirtualization. - // - if ((objClassAttribs & CORINFO_FLG_INTERFACE) != 0) - { - // Don't try guarded devirtualiztion when we're doing late devirtualization. - // - if (isLateDevirtualization) - { - JITDUMP("No guarded devirt during late devirtualization\n"); - return; - } + var_types sigType = ((clsAttr & CORINFO_FLG_VALUECLASS) != 0) ? TYP_BYREF : TYP_REF; + lclVarInfo[0].lclTypeInfo = sigType; - considerGuardedDevirtualization(call, ilOffset, isInterface, baseMethod, baseClass, pContextHandle); - return; - } + GenTree* thisArgNode = thisArg->GetEarlyNode(); - // If we get this far, the jit has a lower bound class type for the `this` object being used for dispatch. - // It may or may not know enough to devirtualize... - if (isInterface) - { - assert(call->IsVirtualStub()); - JITDUMP("--- base class is interface\n"); - } + assert(varTypeIsGC(thisArgNode->TypeGet()) || // "this" is managed + ((thisArgNode->TypeGet() == TYP_I_IMPL) && // "this" is unmgd but the method's class doesnt care + (clsAttr & CORINFO_FLG_VALUECLASS))); - // Fetch the method that would be called based on the declared type of 'this', - // and prepare to fetch the method attributes. - // - CORINFO_DEVIRTUALIZATION_INFO dvInfo; - dvInfo.virtualMethod = baseMethod; - dvInfo.objClass = objClass; - dvInfo.context = *pContextHandle; - dvInfo.detail = CORINFO_DEVIRTUALIZATION_UNKNOWN; - dvInfo.pResolvedTokenVirtualMethod = pResolvedToken; + if (genActualType(thisArgNode->TypeGet()) != genActualType(sigType)) + { + if (sigType == TYP_REF) + { + /* The argument cannot be bashed into a ref (see bug 750871) */ + inlineResult->NoteFatal(InlineObservation::CALLSITE_ARG_NO_BASH_TO_REF); + return; + } - info.compCompHnd->resolveVirtualMethod(&dvInfo); + /* This can only happen with byrefs <-> ints/shorts */ - CORINFO_METHOD_HANDLE derivedMethod = dvInfo.devirtualizedMethod; - CORINFO_CONTEXT_HANDLE exactContext = dvInfo.exactContext; - CORINFO_CLASS_HANDLE derivedClass = NO_CLASS_HANDLE; - CORINFO_RESOLVED_TOKEN* pDerivedResolvedToken = &dvInfo.resolvedTokenDevirtualizedMethod; + assert(sigType == TYP_BYREF); + assert((genActualType(thisArgNode->TypeGet()) == TYP_I_IMPL) || (thisArgNode->TypeGet() == TYP_BYREF)); - if (derivedMethod != nullptr) - { - assert(exactContext != nullptr); - assert(((size_t)exactContext & CORINFO_CONTEXTFLAGS_MASK) == CORINFO_CONTEXTFLAGS_CLASS); - derivedClass = (CORINFO_CLASS_HANDLE)((size_t)exactContext & ~CORINFO_CONTEXTFLAGS_MASK); + lclVarInfo[0].lclVerTypeInfo = typeInfo(varType2tiType(TYP_I_IMPL)); + } } - DWORD derivedMethodAttribs = 0; - bool derivedMethodIsFinal = false; - bool canDevirtualize = false; + /* Init the types of the arguments and make sure the types + * from the trees match the types in the signature */ -#if defined(DEBUG) - const char* derivedClassName = "?derivedClass"; - const char* derivedMethodName = "?derivedMethod"; - const char* note = "inexact or not final"; -#endif + CORINFO_ARG_LIST_HANDLE argLst; + argLst = methInfo->args.args; - // If we failed to get a method handle, we can't directly devirtualize. - // - // This can happen when prejitting, if the devirtualization crosses - // servicing bubble boundaries, or if objClass is a shared class. - // - if (derivedMethod == nullptr) - { - JITDUMP("--- no derived method: %s\n", devirtualizationDetailToString(dvInfo.detail)); - } - else + // TODO-ARGS: We can presumably just use type info stored in CallArgs + // instead of reiterating the signature. + unsigned i; + for (i = (thisArg ? 1 : 0); i < ilArgCnt; i++, argLst = info.compCompHnd->getArgNext(argLst)) { - // Fetch method attributes to see if method is marked final. - derivedMethodAttribs = info.compCompHnd->getMethodAttribs(derivedMethod); - derivedMethodIsFinal = ((derivedMethodAttribs & CORINFO_FLG_FINAL) != 0); + var_types sigType = (var_types)eeGetArgType(argLst, &methInfo->args); -#if defined(DEBUG) - if (isExact) - { - note = "exact"; - } - else if (objClassIsFinal) - { - note = "final class"; - } - else if (derivedMethodIsFinal) - { - note = "final method"; - } + lclVarInfo[i].lclVerTypeInfo = verParseArgSigToTypeInfo(&methInfo->args, argLst); - if (verbose || doPrint) +#ifdef FEATURE_SIMD + if ((!foundSIMDType || (sigType == TYP_STRUCT)) && isSIMDorHWSIMDClass(&(lclVarInfo[i].lclVerTypeInfo))) { - derivedMethodName = eeGetMethodName(derivedMethod, nullptr); - derivedClassName = eeGetClassName(derivedClass); - if (verbose) + // If this is a SIMD class (i.e. in the SIMD assembly), then we will consider that we've + // found a SIMD type, even if this may not be a type we recognize (the assumption is that + // it is likely to use a SIMD type, and therefore we want to increase the inlining multiplier). + foundSIMDType = true; + if (sigType == TYP_STRUCT) { - printf(" devirt to %s::%s -- %s\n", derivedClassName, derivedMethodName, note); - gtDispTree(call); + var_types structType = impNormStructType(lclVarInfo[i].lclVerTypeInfo.GetClassHandle()); + sigType = structType; } } -#endif // defined(DEBUG) +#endif // FEATURE_SIMD - canDevirtualize = isExact || objClassIsFinal || (!isInterface && derivedMethodIsFinal); - } + lclVarInfo[i].lclTypeInfo = sigType; + lclVarInfo[i].lclHasLdlocaOp = false; - // We still might be able to do a guarded devirtualization. - // Note the call might be an interface call or a virtual call. - // - if (!canDevirtualize) - { - JITDUMP(" Class not final or exact%s\n", isInterface ? "" : ", and method not final"); + // Does the tree type match the signature type? -#if defined(DEBUG) - // If we know the object type exactly, we generally expect we can devirtualize. - // (don't when doing late devirt as we won't have an owner type (yet)) - // - if (!isLateDevirtualization && (isExact || objClassIsFinal) && JitConfig.JitNoteFailedExactDevirtualization()) + GenTree* inlArgNode = inlArgInfo[i].arg->GetNode(); + + if (sigType == inlArgNode->gtType) { - printf("@@@ Exact/Final devirt failure in %s at [%06u] $ %s\n", info.compFullName, dspTreeID(call), - devirtualizationDetailToString(dvInfo.detail)); + continue; } -#endif - // Don't try guarded devirtualiztion if we're doing late devirtualization. - // - if (isLateDevirtualization) + assert(impCheckImplicitArgumentCoercion(sigType, inlArgNode->gtType)); + assert(!varTypeIsStruct(inlArgNode->gtType) && !varTypeIsStruct(sigType)); + + // In valid IL, this can only happen for short integer types or byrefs <-> [native] ints, + // but in bad IL cases with caller-callee signature mismatches we can see other types. + // Intentionally reject cases with mismatches so the jit is more flexible when + // encountering bad IL. + + bool isPlausibleTypeMatch = (genActualType(sigType) == genActualType(inlArgNode->gtType)) || + (genActualTypeIsIntOrI(sigType) && inlArgNode->gtType == TYP_BYREF) || + (sigType == TYP_BYREF && genActualTypeIsIntOrI(inlArgNode->gtType)); + + if (!isPlausibleTypeMatch) { - JITDUMP("No guarded devirt during late devirtualization\n"); + inlineResult->NoteFatal(InlineObservation::CALLSITE_ARG_TYPES_INCOMPATIBLE); return; } - considerGuardedDevirtualization(call, ilOffset, isInterface, baseMethod, baseClass, pContextHandle); - return; - } + // The same size but different type of the arguments. + GenTree** pInlArgNode = &inlArgInfo[i].arg->EarlyNodeRef(); - // All checks done. Time to transform the call. - // - // We should always have an exact class context. - // - // Note that wouldnt' be true if the runtime side supported array interface devirt, - // the resulting method would be a generic method of the non-generic SZArrayHelper class. - // - assert(canDevirtualize); + // Is it a narrowing or widening cast? + // Widening casts are ok since the value computed is already + // normalized to an int (on the IL stack) + if (genTypeSize(inlArgNode->gtType) >= genTypeSize(sigType)) + { + if (sigType == TYP_BYREF) + { + lclVarInfo[i].lclVerTypeInfo = typeInfo(varType2tiType(TYP_I_IMPL)); + } + else if (inlArgNode->gtType == TYP_BYREF) + { + assert(varTypeIsIntOrI(sigType)); - JITDUMP(" %s; can devirtualize\n", note); + /* If possible bash the BYREF to an int */ + if (inlArgNode->IsLocalAddrExpr() != nullptr) + { + inlArgNode->gtType = TYP_I_IMPL; + lclVarInfo[i].lclVerTypeInfo = typeInfo(varType2tiType(TYP_I_IMPL)); + } + else + { + // Arguments 'int <- byref' cannot be changed + inlineResult->NoteFatal(InlineObservation::CALLSITE_ARG_NO_BASH_TO_INT); + return; + } + } + else if (genTypeSize(sigType) < TARGET_POINTER_SIZE) + { + // Narrowing cast. + if (inlArgNode->OperIs(GT_LCL_VAR)) + { + const unsigned lclNum = inlArgNode->AsLclVarCommon()->GetLclNum(); + if (!lvaTable[lclNum].lvNormalizeOnLoad() && sigType == lvaGetRealType(lclNum)) + { + // We don't need to insert a cast here as the variable + // was assigned a normalized value of the right type. + continue; + } + } - // Make the updates. - call->gtFlags &= ~GTF_CALL_VIRT_VTABLE; - call->gtFlags &= ~GTF_CALL_VIRT_STUB; - call->gtCallMethHnd = derivedMethod; - call->gtCallType = CT_USER_FUNC; - call->gtControlExpr = nullptr; - call->gtCallMoreFlags |= GTF_CALL_M_DEVIRTUALIZED; + inlArgNode = gtNewCastNode(TYP_INT, inlArgNode, false, sigType); - // Virtual calls include an implicit null check, which we may - // now need to make explicit. - if (!objIsNonNull) - { - call->gtFlags |= GTF_CALL_NULLCHECK; - } + inlArgInfo[i].argIsLclVar = false; + // Try to fold the node in case we have constant arguments. + if (inlArgInfo[i].argIsInvariant) + { + inlArgNode = gtFoldExpr(inlArgNode); + } + *pInlArgNode = inlArgNode; + } +#ifdef TARGET_64BIT + else if (genTypeSize(genActualType(inlArgNode->gtType)) < genTypeSize(sigType)) + { + // This should only happen for int -> native int widening + inlArgNode = gtNewCastNode(genActualType(sigType), inlArgNode, false, sigType); - // Clear the inline candidate info (may be non-null since - // it's a union field used for other things by virtual - // stubs) - call->gtInlineCandidateInfo = nullptr; - call->gtCallMoreFlags &= ~GTF_CALL_M_HAS_LATE_DEVIRT_INFO; + inlArgInfo[i].argIsLclVar = false; -#if defined(DEBUG) - if (verbose) - { - printf("... after devirt...\n"); - gtDispTree(call); + // Try to fold the node in case we have constant arguments. + if (inlArgInfo[i].argIsInvariant) + { + inlArgNode = gtFoldExpr(inlArgNode); + } + *pInlArgNode = inlArgNode; + } +#endif // TARGET_64BIT + } } - if (doPrint) - { - printf("Devirtualized %s call to %s:%s; now direct call to %s:%s [%s]\n", callKind, baseClassName, - baseMethodName, derivedClassName, derivedMethodName, note); - } + /* Init the types of the local variables */ - // If we successfully devirtualized based on an exact or final class, - // and we have dynamic PGO data describing the likely class, make sure they agree. - // - // If pgo source is not dynamic we may see likely classes from other versions of this code - // where types had different properties. - // - // If method is an inlinee we may be specializing to a class that wasn't seen at runtime. - // - const bool canSensiblyCheck = - (isExact || objClassIsFinal) && (fgPgoSource == ICorJitInfo::PgoSource::Dynamic) && !compIsForInlining(); - if (JitConfig.JitCrossCheckDevirtualizationAndPGO() && canSensiblyCheck) - { - // We only can handle a single likely class for now - const int maxLikelyClasses = 1; - LikelyClassMethodRecord likelyClasses[maxLikelyClasses]; + CORINFO_ARG_LIST_HANDLE localsSig; + localsSig = methInfo->locals.args; - UINT32 numberOfClasses = - getLikelyClasses(likelyClasses, maxLikelyClasses, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset); - UINT32 likelihood = likelyClasses[0].likelihood; + for (i = 0; i < methInfo->locals.numArgs; i++) + { + bool isPinned; + var_types type = (var_types)eeGetArgType(localsSig, &methInfo->locals, &isPinned); - CORINFO_CLASS_HANDLE likelyClass = (CORINFO_CLASS_HANDLE)likelyClasses[0].handle; + lclVarInfo[i + ilArgCnt].lclHasLdlocaOp = false; + lclVarInfo[i + ilArgCnt].lclTypeInfo = type; - if (numberOfClasses > 0) + if (varTypeIsGC(type)) { - // PGO had better agree the class we devirtualized to is plausible. - // - if (likelyClass != derivedClass) + if (isPinned) { - // Managed type system may report different addresses for a class handle - // at different times....? - // - // Also, AOT may have a more nuanced notion of class equality. - // - if (!opts.jitFlags->IsSet(JitFlags::JIT_FLAG_PREJIT)) - { - bool mismatch = true; - - // derivedClass will be the introducer of derived method, so it's possible - // likelyClass is a non-overriding subclass. Check up the hierarchy. - // - CORINFO_CLASS_HANDLE parentClass = likelyClass; - while (parentClass != NO_CLASS_HANDLE) - { - if (parentClass == derivedClass) - { - mismatch = false; - break; - } - - parentClass = info.compCompHnd->getParentType(parentClass); - } - - if (mismatch || (numberOfClasses != 1) || (likelihood != 100)) - { - printf("@@@ Likely %p (%s) != Derived %p (%s) [n=%u, l=%u, il=%u] in %s \n", likelyClass, - eeGetClassName(likelyClass), derivedClass, eeGetClassName(derivedClass), numberOfClasses, - likelihood, ilOffset, info.compFullName); - } + JITDUMP("Inlinee local #%02u is pinned\n", i); + lclVarInfo[i + ilArgCnt].lclIsPinned = true; - assert(!(mismatch || (numberOfClasses != 1) || (likelihood != 100))); + // Pinned locals may cause inlines to fail. + inlineResult->Note(InlineObservation::CALLEE_HAS_PINNED_LOCALS); + if (inlineResult->IsFailure()) + { + return; } } - } - } -#endif // defined(DEBUG) - // If the 'this' object is a value class, see if we can rework the call to invoke the - // unboxed entry. This effectively inlines the normally un-inlineable wrapper stub - // and exposes the potentially inlinable unboxed entry method. - // - // We won't optimize explicit tail calls, as ensuring we get the right tail call info - // is tricky (we'd need to pass an updated sig and resolved token back to some callers). - // - // Note we may not have a derived class in some cases (eg interface call on an array) - // - if (info.compCompHnd->isValueClass(derivedClass)) - { - if (isExplicitTailCall) - { - JITDUMP("Have a direct explicit tail call to boxed entry point; can't optimize further\n"); + pInlineInfo->numberOfGcRefLocals++; } - else + else if (isPinned) { - JITDUMP("Have a direct call to boxed entry point. Trying to optimize to call an unboxed entry point\n"); + JITDUMP("Ignoring pin on inlinee local #%02u -- not a GC type\n", i); + } - // Note for some shared methods the unboxed entry point requires an extra parameter. - bool requiresInstMethodTableArg = false; - CORINFO_METHOD_HANDLE unboxedEntryMethod = - info.compCompHnd->getUnboxedEntry(derivedMethod, &requiresInstMethodTableArg); + lclVarInfo[i + ilArgCnt].lclVerTypeInfo = verParseArgSigToTypeInfo(&methInfo->locals, localsSig); - if (unboxedEntryMethod != nullptr) + // If this local is a struct type with GC fields, inform the inliner. It may choose to bail + // out on the inline. + if (type == TYP_STRUCT) + { + CORINFO_CLASS_HANDLE lclHandle = lclVarInfo[i + ilArgCnt].lclVerTypeInfo.GetClassHandle(); + DWORD typeFlags = info.compCompHnd->getClassAttribs(lclHandle); + if ((typeFlags & CORINFO_FLG_CONTAINS_GC_PTR) != 0) { - bool optimizedTheBox = false; - - // If the 'this' object is a local box, see if we can revise things - // to not require boxing. - // - if (thisObj->IsBoxedValue() && !isExplicitTailCall) + inlineResult->Note(InlineObservation::CALLEE_HAS_GC_STRUCT); + if (inlineResult->IsFailure()) { - // Since the call is the only consumer of the box, we know the box can't escape - // since it is being passed an interior pointer. - // - // So, revise the box to simply create a local copy, use the address of that copy - // as the this pointer, and update the entry point to the unboxed entry. - // - // Ideally, we then inline the boxed method and and if it turns out not to modify - // the copy, we can undo the copy too. - if (requiresInstMethodTableArg) - { - // Perform a trial box removal and ask for the type handle tree that fed the box. - // - JITDUMP("Unboxed entry needs method table arg...\n"); - GenTree* methodTableArg = - gtTryRemoveBoxUpstreamEffects(thisObj, BR_DONT_REMOVE_WANT_TYPE_HANDLE); - - if (methodTableArg != nullptr) - { - // If that worked, turn the box into a copy to a local var - // - JITDUMP("Found suitable method table arg tree [%06u]\n", dspTreeID(methodTableArg)); - GenTree* localCopyThis = gtTryRemoveBoxUpstreamEffects(thisObj, BR_MAKE_LOCAL_COPY); - - if (localCopyThis != nullptr) - { - // Pass the local var as this and the type handle as a new arg - // - JITDUMP("Success! invoking unboxed entry point on local copy, and passing method table " - "arg\n"); - // TODO-CallArgs-REVIEW: Might discard commas otherwise? - assert(thisObj == thisArg->GetEarlyNode()); - thisArg->SetEarlyNode(localCopyThis); - call->gtCallMoreFlags |= GTF_CALL_M_UNBOXED; - - call->gtArgs.InsertInstParam(this, methodTableArg); - - call->gtCallMethHnd = unboxedEntryMethod; - derivedMethod = unboxedEntryMethod; - pDerivedResolvedToken = &dvInfo.resolvedTokenDevirtualizedUnboxedMethod; - - // Method attributes will differ because unboxed entry point is shared - // - const DWORD unboxedMethodAttribs = - info.compCompHnd->getMethodAttribs(unboxedEntryMethod); - JITDUMP("Updating method attribs from 0x%08x to 0x%08x\n", derivedMethodAttribs, - unboxedMethodAttribs); - derivedMethodAttribs = unboxedMethodAttribs; - optimizedTheBox = true; - } - else - { - JITDUMP("Sorry, failed to undo the box -- can't convert to local copy\n"); - } - } - else - { - JITDUMP("Sorry, failed to undo the box -- can't find method table arg\n"); - } - } - else - { - JITDUMP("Found unboxed entry point, trying to simplify box to a local copy\n"); - GenTree* localCopyThis = gtTryRemoveBoxUpstreamEffects(thisObj, BR_MAKE_LOCAL_COPY); - - if (localCopyThis != nullptr) - { - JITDUMP("Success! invoking unboxed entry point on local copy\n"); - assert(thisObj == thisArg->GetEarlyNode()); - // TODO-CallArgs-REVIEW: Might discard commas otherwise? - thisArg->SetEarlyNode(localCopyThis); - call->gtCallMethHnd = unboxedEntryMethod; - call->gtCallMoreFlags |= GTF_CALL_M_UNBOXED; - derivedMethod = unboxedEntryMethod; - pDerivedResolvedToken = &dvInfo.resolvedTokenDevirtualizedUnboxedMethod; - - optimizedTheBox = true; - } - else - { - JITDUMP("Sorry, failed to undo the box\n"); - } - } - - if (optimizedTheBox) - { - -#if FEATURE_TAILCALL_OPT - if (call->IsImplicitTailCall()) - { - JITDUMP("Clearing the implicit tail call flag\n"); - - // If set, we clear the implicit tail call flag - // as we just introduced a new address taken local variable - // - call->gtCallMoreFlags &= ~GTF_CALL_M_IMPLICIT_TAILCALL; - } -#endif // FEATURE_TAILCALL_OPT - } + return; } - if (!optimizedTheBox) + // Do further notification in the case where the call site is rare; some policies do + // not track the relative hotness of call sites for "always" inline cases. + if (pInlineInfo->iciBlock->isRunRarely()) { - // If we get here, we have a boxed value class that either wasn't boxed - // locally, or was boxed locally but we were unable to remove the box for - // various reasons. - // - // We can still update the call to invoke the unboxed entry, if the - // boxed value is simple. - // - if (requiresInstMethodTableArg) - { - // Get the method table from the boxed object. - // - // TODO-CallArgs-REVIEW: Use thisObj here? Differs by gtEffectiveVal. - GenTree* const clonedThisArg = gtClone(thisArg->GetEarlyNode()); - - if (clonedThisArg == nullptr) - { - JITDUMP( - "unboxed entry needs MT arg, but `this` was too complex to clone. Deferring update.\n"); - } - else - { - JITDUMP("revising call to invoke unboxed entry with additional method table arg\n"); - - GenTree* const methodTableArg = gtNewMethodTableLookup(clonedThisArg); - - // Update the 'this' pointer to refer to the box payload - // - GenTree* const payloadOffset = gtNewIconNode(TARGET_POINTER_SIZE, TYP_I_IMPL); - GenTree* const boxPayload = - gtNewOperNode(GT_ADD, TYP_BYREF, thisArg->GetEarlyNode(), payloadOffset); - - assert(thisObj == thisArg->GetEarlyNode()); - thisArg->SetEarlyNode(boxPayload); - call->gtCallMethHnd = unboxedEntryMethod; - call->gtCallMoreFlags |= GTF_CALL_M_UNBOXED; - - // Method attributes will differ because unboxed entry point is shared - // - const DWORD unboxedMethodAttribs = info.compCompHnd->getMethodAttribs(unboxedEntryMethod); - JITDUMP("Updating method attribs from 0x%08x to 0x%08x\n", derivedMethodAttribs, - unboxedMethodAttribs); - derivedMethod = unboxedEntryMethod; - pDerivedResolvedToken = &dvInfo.resolvedTokenDevirtualizedUnboxedMethod; - derivedMethodAttribs = unboxedMethodAttribs; - - call->gtArgs.InsertInstParam(this, methodTableArg); - } - } - else + inlineResult->Note(InlineObservation::CALLSITE_RARE_GC_STRUCT); + if (inlineResult->IsFailure()) { - JITDUMP("revising call to invoke unboxed entry\n"); - - GenTree* const payloadOffset = gtNewIconNode(TARGET_POINTER_SIZE, TYP_I_IMPL); - GenTree* const boxPayload = - gtNewOperNode(GT_ADD, TYP_BYREF, thisArg->GetEarlyNode(), payloadOffset); - thisArg->SetEarlyNode(boxPayload); - call->gtCallMethHnd = unboxedEntryMethod; - call->gtCallMoreFlags |= GTF_CALL_M_UNBOXED; - derivedMethod = unboxedEntryMethod; - pDerivedResolvedToken = &dvInfo.resolvedTokenDevirtualizedUnboxedMethod; + return; } } } - else - { - // Many of the low-level methods on value classes won't have unboxed entries, - // as they need access to the type of the object. - // - // Note this may be a cue for us to stack allocate the boxed object, since - // we probably know that these objects don't escape. - JITDUMP("Sorry, failed to find unboxed entry point\n"); - } } - } - - // Need to update call info too. - // - *method = derivedMethod; - *methodFlags = derivedMethodAttribs; - // Update context handle - // - *pContextHandle = MAKE_METHODCONTEXT(derivedMethod); + localsSig = info.compCompHnd->getArgNext(localsSig); - // Update exact context handle. - // - if (pExactContextHandle != nullptr) - { - *pExactContextHandle = MAKE_CLASSCONTEXT(derivedClass); +#ifdef FEATURE_SIMD + if ((!foundSIMDType || (type == TYP_STRUCT)) && isSIMDorHWSIMDClass(&(lclVarInfo[i + ilArgCnt].lclVerTypeInfo))) + { + foundSIMDType = true; + if (type == TYP_STRUCT) + { + var_types structType = impNormStructType(lclVarInfo[i + ilArgCnt].lclVerTypeInfo.GetClassHandle()); + lclVarInfo[i + ilArgCnt].lclTypeInfo = structType; + } + } +#endif // FEATURE_SIMD } -#ifdef FEATURE_READYTORUN - if (opts.IsReadyToRun()) +#ifdef FEATURE_SIMD + if (!foundSIMDType && (call->AsCall()->gtRetClsHnd != nullptr) && isSIMDorHWSIMDClass(call->AsCall()->gtRetClsHnd)) { - // For R2R, getCallInfo triggers bookkeeping on the zap - // side and acquires the actual symbol to call so we need to call it here. - - // Look up the new call info. - CORINFO_CALL_INFO derivedCallInfo; - eeGetCallInfo(pDerivedResolvedToken, nullptr, CORINFO_CALLINFO_ALLOWINSTPARAM, &derivedCallInfo); - - // Update the call. - call->gtCallMoreFlags &= ~GTF_CALL_M_VIRTSTUB_REL_INDIRECT; - call->gtCallMoreFlags &= ~GTF_CALL_M_R2R_REL_INDIRECT; - call->setEntryPoint(derivedCallInfo.codePointerLookup.constLookup); + foundSIMDType = true; } -#endif // FEATURE_READYTORUN + pInlineInfo->hasSIMDTypeArgLocalOrReturn = foundSIMDType; +#endif // FEATURE_SIMD } //------------------------------------------------------------------------ -// impConsiderCallProbe: Consider whether a call should get a histogram probe -// and mark it if so. +// impInlineFetchLocal: get a local var that represents an inlinee local // // Arguments: -// call - The call -// ilOffset - The precise IL offset of the call +// lclNum -- number of the inlinee local +// reason -- debug string describing purpose of the local var // // Returns: -// True if the call was marked such that we will add a class or method probe for it. -// -bool Compiler::impConsiderCallProbe(GenTreeCall* call, IL_OFFSET ilOffset) -{ - // Possibly instrument. Note for OSR+PGO we will instrument when - // optimizing and (currently) won't devirtualize. We may want - // to revisit -- if we can devirtualize we should be able to - // suppress the probe. - // - // We strip BBINSTR from inlinees currently, so we'll only - // do this for the root method calls. - // - if (!opts.jitFlags->IsSet(JitFlags::JIT_FLAG_BBINSTR)) - { - return false; - } - - assert(opts.OptimizationDisabled() || opts.IsOSR()); - assert(!compIsForInlining()); - - // During importation, optionally flag this block as one that - // contains calls requiring class profiling. Ideally perhaps - // we'd just keep track of the calls themselves, so we don't - // have to search for them later. - // - if (compClassifyGDVProbeType(call) == GDVProbeType::None) - { - return false; - } - - JITDUMP("\n ... marking [%06u] in " FMT_BB " for method/class profile instrumentation\n", dspTreeID(call), - compCurBB->bbNum); - HandleHistogramProfileCandidateInfo* pInfo = new (this, CMK_Inlining) HandleHistogramProfileCandidateInfo; - - // Record some info needed for the class profiling probe. - // - pInfo->ilOffset = ilOffset; - pInfo->probeIndex = info.compHandleHistogramProbeCount++; - call->gtHandleHistogramProfileCandidateInfo = pInfo; - - // Flag block as needing scrutiny - // - compCurBB->bbFlags |= BBF_HAS_HISTOGRAM_PROFILE; - return true; -} - -//------------------------------------------------------------------------ -// compClassifyGDVProbeType: -// Classify the type of GDV probe to use for a call site. -// -// Arguments: -// call - The call +// Number of the local to use // -// Returns: -// The type of probe to use. +// Notes: +// This method is invoked only for locals actually used in the +// inlinee body. // -Compiler::GDVProbeType Compiler::compClassifyGDVProbeType(GenTreeCall* call) -{ - if (call->gtCallType == CT_INDIRECT) - { - return GDVProbeType::None; - } - - if (!opts.jitFlags->IsSet(JitFlags::JIT_FLAG_BBINSTR) || opts.jitFlags->IsSet(JitFlags::JIT_FLAG_PREJIT)) - { - return GDVProbeType::None; - } - - bool createTypeHistogram = false; - if (JitConfig.JitClassProfiling() > 0) - { - createTypeHistogram = call->IsVirtualStub() || call->IsVirtualVtable(); - - // Cast helpers may conditionally (depending on whether the class is - // exact or not) have probes. For those helpers we do not use this - // function to classify the probe type until after we have decided on - // whether we probe them or not. - createTypeHistogram = createTypeHistogram || (impIsCastHelperEligibleForClassProbe(call) && - (call->gtHandleHistogramProfileCandidateInfo != nullptr)); - } - - bool createMethodHistogram = ((JitConfig.JitDelegateProfiling() > 0) && call->IsDelegateInvoke()) || - ((JitConfig.JitVTableProfiling() > 0) && call->IsVirtualVtable()); - - if (createTypeHistogram && createMethodHistogram) - { - return GDVProbeType::MethodAndClassProfile; - } - - if (createTypeHistogram) - { - return GDVProbeType::ClassProfile; - } - - if (createMethodHistogram) - { - return GDVProbeType::MethodProfile; - } - - return GDVProbeType::None; -} +// Allocates a new temp if necessary, and copies key properties +// over from the inlinee local var info. -//------------------------------------------------------------------------ -// impGetSpecialIntrinsicExactReturnType: Look for special cases where a call -// to an intrinsic returns an exact type -// -// Arguments: -// methodHnd -- handle for the special intrinsic method -// -// Returns: -// Exact class handle returned by the intrinsic call, if known. -// Nullptr if not known, or not likely to lead to beneficial optimization. -CORINFO_CLASS_HANDLE Compiler::impGetSpecialIntrinsicExactReturnType(CORINFO_METHOD_HANDLE methodHnd) +unsigned Compiler::impInlineFetchLocal(unsigned lclNum DEBUGARG(const char* reason)) { - JITDUMP("Special intrinsic: looking for exact type returned by %s\n", eeGetMethodFullName(methodHnd)); + assert(compIsForInlining()); - CORINFO_CLASS_HANDLE result = nullptr; + unsigned tmpNum = impInlineInfo->lclTmpNum[lclNum]; - // See what intrinsic we have... - const NamedIntrinsic ni = lookupNamedIntrinsic(methodHnd); - switch (ni) + if (tmpNum == BAD_VAR_NUM) { - case NI_System_Collections_Generic_Comparer_get_Default: - case NI_System_Collections_Generic_EqualityComparer_get_Default: + const InlLclVarInfo& inlineeLocal = impInlineInfo->lclVarInfo[lclNum + impInlineInfo->argCnt]; + const var_types lclTyp = inlineeLocal.lclTypeInfo; + + // The lifetime of this local might span multiple BBs. + // So it is a long lifetime local. + impInlineInfo->lclTmpNum[lclNum] = tmpNum = lvaGrabTemp(false DEBUGARG(reason)); + + // Copy over key info + lvaTable[tmpNum].lvType = lclTyp; + lvaTable[tmpNum].lvHasLdAddrOp = inlineeLocal.lclHasLdlocaOp; + lvaTable[tmpNum].lvPinned = inlineeLocal.lclIsPinned; + lvaTable[tmpNum].lvHasILStoreOp = inlineeLocal.lclHasStlocOp; + lvaTable[tmpNum].lvHasMultipleILStoreOp = inlineeLocal.lclHasMultipleStlocOp; + + // Copy over class handle for ref types. Note this may be a + // shared type -- someday perhaps we can get the exact + // signature and pass in a more precise type. + if (lclTyp == TYP_REF) { - // Expect one class generic parameter; figure out which it is. - CORINFO_SIG_INFO sig; - info.compCompHnd->getMethodSig(methodHnd, &sig); - assert(sig.sigInst.classInstCount == 1); - CORINFO_CLASS_HANDLE typeHnd = sig.sigInst.classInst[0]; - assert(typeHnd != nullptr); - - // Lookup can incorrect when we have __Canon as it won't appear - // to implement any interface types. - // - // And if we do not have a final type, devirt & inlining is - // unlikely to result in much simplification. - // - // We can use CORINFO_FLG_FINAL to screen out both of these cases. - const DWORD typeAttribs = info.compCompHnd->getClassAttribs(typeHnd); - const bool isFinalType = ((typeAttribs & CORINFO_FLG_FINAL) != 0); + assert(lvaTable[tmpNum].lvSingleDef == 0); - if (isFinalType) + lvaTable[tmpNum].lvSingleDef = !inlineeLocal.lclHasMultipleStlocOp && !inlineeLocal.lclHasLdlocaOp; + if (lvaTable[tmpNum].lvSingleDef) { - if (ni == NI_System_Collections_Generic_EqualityComparer_get_Default) - { - result = info.compCompHnd->getDefaultEqualityComparerClass(typeHnd); - } - else - { - assert(ni == NI_System_Collections_Generic_Comparer_get_Default); - result = info.compCompHnd->getDefaultComparerClass(typeHnd); - } - JITDUMP("Special intrinsic for type %s: return type is %s\n", eeGetClassName(typeHnd), - result != nullptr ? eeGetClassName(result) : "unknown"); + JITDUMP("Marked V%02u as a single def temp\n", tmpNum); } - else + + lvaSetClass(tmpNum, inlineeLocal.lclVerTypeInfo.GetClassHandleForObjRef()); + } + + if (inlineeLocal.lclVerTypeInfo.IsStruct()) + { + if (varTypeIsStruct(lclTyp)) { - JITDUMP("Special intrinsic for type %s: type not final, so deferring opt\n", eeGetClassName(typeHnd)); + lvaSetStruct(tmpNum, inlineeLocal.lclVerTypeInfo.GetClassHandle(), true /* unsafe value cls check */); } - - break; } - default: +#ifdef DEBUG + // Sanity check that we're properly prepared for gc ref locals. + if (varTypeIsGC(lclTyp)) { - JITDUMP("This special intrinsic not handled, sorry...\n"); - break; + // Since there are gc locals we should have seen them earlier + // and if there was a return value, set up the spill temp. + assert(impInlineInfo->HasGcRefLocals()); + assert((info.compRetNativeType == TYP_VOID) || fgNeedReturnSpillTemp()); + } + else + { + // Make sure all pinned locals count as gc refs. + assert(!inlineeLocal.lclIsPinned); } +#endif // DEBUG } - return result; + return tmpNum; } //------------------------------------------------------------------------ -// impAllocateMethodPointerInfo: create methodPointerInfo into jit-allocated memory and init it. +// impInlineFetchArg: return tree node for argument value in an inlinee // // Arguments: -// token - init value for the allocated token. -// tokenConstrained - init value for the constraint associated with the token +// lclNum -- argument number in inlinee IL +// inlArgInfo -- argument info for inlinee +// lclVarInfo -- var info for inlinee // -// Return Value: -// pointer to token into jit-allocated memory. -methodPointerInfo* Compiler::impAllocateMethodPointerInfo(const CORINFO_RESOLVED_TOKEN& token, mdToken tokenConstrained) -{ - methodPointerInfo* memory = getAllocator(CMK_Unknown).allocate(1); - memory->m_token = token; - memory->m_tokenConstraint = tokenConstrained; - return memory; -} - -//------------------------------------------------------------------------ -// SpillRetExprHelper: iterate through arguments tree and spill ret_expr to local variables. +// Returns: +// Tree for the argument's value. Often an inlinee-scoped temp +// GT_LCL_VAR but can be other tree kinds, if the argument +// expression from the caller can be directly substituted into the +// inlinee body. +// +// Notes: +// Must be used only for arguments -- use impInlineFetchLocal for +// inlinee locals. +// +// Direct substitution is performed when the formal argument cannot +// change value in the inlinee body (no starg or ldarga), and the +// actual argument expression's value cannot be changed if it is +// substituted it into the inlinee body. +// +// Even if an inlinee-scoped temp is returned here, it may later be +// "bashed" to a caller-supplied tree when arguments are actually +// passed (see fgInlinePrependStatements). Bashing can happen if +// the argument ends up being single use and other conditions are +// met. So the contents of the tree returned here may not end up +// being the ones ultimately used for the argument. // -class SpillRetExprHelper +// This method will side effect inlArgInfo. It should only be called +// for actual uses of the argument in the inlinee. + +GenTree* Compiler::impInlineFetchArg(unsigned lclNum, InlArgInfo* inlArgInfo, InlLclVarInfo* lclVarInfo) { -public: - SpillRetExprHelper(Compiler* comp) : comp(comp) - { - } + // Cache the relevant arg and lcl info for this argument. + // We will modify argInfo but not lclVarInfo. + InlArgInfo& argInfo = inlArgInfo[lclNum]; + const InlLclVarInfo& lclInfo = lclVarInfo[lclNum]; + const bool argCanBeModified = argInfo.argHasLdargaOp || argInfo.argHasStargOp; + const var_types lclTyp = lclInfo.lclTypeInfo; + GenTree* op1 = nullptr; - void StoreRetExprResultsInArgs(GenTreeCall* call) - { - for (CallArg& arg : call->gtArgs.Args()) - { - comp->fgWalkTreePre(&arg.EarlyNodeRef(), SpillRetExprVisitor, this); - } - } + GenTree* argNode = argInfo.arg->GetNode(); + assert(!argNode->OperIs(GT_RET_EXPR)); -private: - static Compiler::fgWalkResult SpillRetExprVisitor(GenTree** pTree, Compiler::fgWalkData* fgWalkPre) + if (argInfo.argIsInvariant && !argCanBeModified) { - assert((pTree != nullptr) && (*pTree != nullptr)); - GenTree* tree = *pTree; - if ((tree->gtFlags & GTF_CALL) == 0) - { - // Trees with ret_expr are marked as GTF_CALL. - return Compiler::WALK_SKIP_SUBTREES; - } - if (tree->OperGet() == GT_RET_EXPR) + // Directly substitute constants or addresses of locals + // + // Clone the constant. Note that we cannot directly use + // argNode in the trees even if !argInfo.argIsUsed as this + // would introduce aliasing between inlArgInfo[].argNode and + // impInlineExpr. Then gtFoldExpr() could change it, causing + // further references to the argument working off of the + // bashed copy. + op1 = gtCloneExpr(argNode); + PREFIX_ASSUME(op1 != nullptr); + argInfo.argTmpNum = BAD_VAR_NUM; + + // We may need to retype to ensure we match the callee's view of the type. + // Otherwise callee-pass throughs of arguments can create return type + // mismatches that block inlining. + // + // Note argument type mismatches that prevent inlining should + // have been caught in impInlineInitVars. + if (op1->TypeGet() != lclTyp) { - SpillRetExprHelper* walker = static_cast(fgWalkPre->pCallbackData); - walker->StoreRetExprAsLocalVar(pTree); + op1->gtType = genActualType(lclTyp); } - return Compiler::WALK_CONTINUE; } - - void StoreRetExprAsLocalVar(GenTree** pRetExpr) + else if (argInfo.argIsLclVar && !argCanBeModified && !argInfo.argHasCallerLocalRef) { - GenTree* retExpr = *pRetExpr; - assert(retExpr->OperGet() == GT_RET_EXPR); - const unsigned tmp = comp->lvaGrabTemp(true DEBUGARG("spilling ret_expr")); - JITDUMP("Storing return expression [%06u] to a local var V%02u.\n", comp->dspTreeID(retExpr), tmp); - comp->impAssignTempGen(tmp, retExpr, (unsigned)Compiler::CHECK_SPILL_NONE); - *pRetExpr = comp->gtNewLclvNode(tmp, retExpr->TypeGet()); + // Directly substitute unaliased caller locals for args that cannot be modified + // + // Use the caller-supplied node if this is the first use. + op1 = argNode; + unsigned argLclNum = op1->AsLclVarCommon()->GetLclNum(); + argInfo.argTmpNum = argLclNum; - if (retExpr->TypeGet() == TYP_REF) + // Use an equivalent copy if this is the second or subsequent + // use. + // + // Note argument type mismatches that prevent inlining should + // have been caught in impInlineInitVars. If inlining is not prevented + // but a cast is necessary, we similarly expect it to have been inserted then. + // So here we may have argument type mismatches that are benign, for instance + // passing a TYP_SHORT local (eg. normalized-on-load) as a TYP_INT arg. + // The exception is when the inlining means we should start tracking the argument. + if (argInfo.argIsUsed || ((lclTyp == TYP_BYREF) && (op1->TypeGet() != TYP_BYREF))) { - assert(comp->lvaTable[tmp].lvSingleDef == 0); - comp->lvaTable[tmp].lvSingleDef = 1; - JITDUMP("Marked V%02u as a single def temp\n", tmp); + assert(op1->gtOper == GT_LCL_VAR); + assert(lclNum == op1->AsLclVar()->gtLclILoffs); - bool isExact = false; - bool isNonNull = false; - CORINFO_CLASS_HANDLE retClsHnd = comp->gtGetClassHandle(retExpr, &isExact, &isNonNull); - if (retClsHnd != nullptr) + // Create a new lcl var node - remember the argument lclNum + op1 = impCreateLocalNode(argLclNum DEBUGARG(op1->AsLclVar()->gtLclILoffs)); + // Start tracking things as a byref if the parameter is a byref. + if (lclTyp == TYP_BYREF) { - comp->lvaSetClass(tmp, retClsHnd, isExact); + op1->gtType = TYP_BYREF; } } } + else if (argInfo.argIsByRefToStructLocal && !argInfo.argHasStargOp) + { + /* Argument is a by-ref address to a struct, a normed struct, or its field. + In these cases, don't spill the byref to a local, simply clone the tree and use it. + This way we will increase the chance for this byref to be optimized away by + a subsequent "dereference" operation. -private: - Compiler* comp; -}; + From Dev11 bug #139955: Argument node can also be TYP_I_IMPL if we've bashed the tree + (in impInlineInitVars()), if the arg has argHasLdargaOp as well as argIsByRefToStructLocal. + For example, if the caller is: + ldloca.s V_1 // V_1 is a local struct + call void Test.ILPart::RunLdargaOnPointerArg(int32*) + and the callee being inlined has: + .method public static void RunLdargaOnPointerArg(int32* ptrToInts) cil managed + ldarga.s ptrToInts + call void Test.FourInts::NotInlined_SetExpectedValuesThroughPointerToPointer(int32**) + then we change the argument tree (of "ldloca.s V_1") to TYP_I_IMPL to match the callee signature. We'll + soon afterwards reject the inlining anyway, since the tree we return isn't a GT_LCL_VAR. + */ + assert(argNode->TypeGet() == TYP_BYREF || argNode->TypeGet() == TYP_I_IMPL); + op1 = gtCloneExpr(argNode); + } + else + { + /* Argument is a complex expression - it must be evaluated into a temp */ -//------------------------------------------------------------------------ -// addFatPointerCandidate: mark the call and the method, that they have a fat pointer candidate. -// Spill ret_expr in the call node, because they can't be cloned. -// -// Arguments: -// call - fat calli candidate -// -void Compiler::addFatPointerCandidate(GenTreeCall* call) -{ - JITDUMP("Marking call [%06u] as fat pointer candidate\n", dspTreeID(call)); - setMethodHasFatPointer(); - call->SetFatPointerCandidate(); - SpillRetExprHelper helper(this); - helper.StoreRetExprResultsInArgs(call); -} + if (argInfo.argHasTmp) + { + assert(argInfo.argIsUsed); + assert(argInfo.argTmpNum < lvaCount); -//------------------------------------------------------------------------ -// pickGDV: Use profile information to pick a GDV candidate for a call site. -// -// Arguments: -// call - the call -// ilOffset - exact IL offset of the call -// isInterface - whether or not the call target is defined on an interface -// classGuess - [out] the class to guess for (mutually exclusive with methodGuess) -// methodGuess - [out] the method to guess for (mutually exclusive with classGuess) -// likelihood - [out] an estimate of the likelihood that the guess will succeed -// -void Compiler::pickGDV(GenTreeCall* call, - IL_OFFSET ilOffset, - bool isInterface, - CORINFO_CLASS_HANDLE* classGuess, - CORINFO_METHOD_HANDLE* methodGuess, - unsigned* likelihood) -{ - *classGuess = NO_CLASS_HANDLE; - *methodGuess = NO_METHOD_HANDLE; - *likelihood = 0; + /* Create a new lcl var node - remember the argument lclNum */ + op1 = gtNewLclvNode(argInfo.argTmpNum, genActualType(lclTyp)); - const int maxLikelyClasses = 32; - LikelyClassMethodRecord likelyClasses[maxLikelyClasses]; - unsigned numberOfClasses = 0; - if (call->IsVirtualStub() || call->IsVirtualVtable()) - { - numberOfClasses = - getLikelyClasses(likelyClasses, maxLikelyClasses, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset); - } + /* This is the second or later use of the this argument, + so we have to use the temp (instead of the actual arg) */ + argInfo.argBashTmpNode = nullptr; + } + else + { + /* First time use */ + assert(!argInfo.argIsUsed); + + /* Reserve a temp for the expression. + * Use a large size node as we may change it later */ - const int maxLikelyMethods = 32; - LikelyClassMethodRecord likelyMethods[maxLikelyMethods]; - unsigned numberOfMethods = 0; + const unsigned tmpNum = lvaGrabTemp(true DEBUGARG("Inlining Arg")); - // TODO-GDV: R2R support requires additional work to reacquire the - // entrypoint, similar to what happens at the end of impDevirtualizeCall. - // As part of supporting this we should merge the tail of - // impDevirtualizeCall and what happens in - // GuardedDevirtualizationTransformer::CreateThen for method GDV. - // - if (!opts.IsReadyToRun() && (call->IsVirtualVtable() || call->IsDelegateInvoke())) - { - numberOfMethods = - getLikelyMethods(likelyMethods, maxLikelyMethods, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset); - } + lvaTable[tmpNum].lvType = lclTyp; - if ((numberOfClasses < 1) && (numberOfMethods < 1)) - { - JITDUMP("No likely class or method, sorry\n"); - return; - } + // For ref types, determine the type of the temp. + if (lclTyp == TYP_REF) + { + if (!argCanBeModified) + { + // If the arg can't be modified in the method + // body, use the type of the value, if + // known. Otherwise, use the declared type. + assert(lvaTable[tmpNum].lvSingleDef == 0); + lvaTable[tmpNum].lvSingleDef = 1; + JITDUMP("Marked V%02u as a single def temp\n", tmpNum); + lvaSetClass(tmpNum, argNode, lclInfo.lclVerTypeInfo.GetClassHandleForObjRef()); + } + else + { + // Arg might be modified, use the declared type of + // the argument. + lvaSetClass(tmpNum, lclInfo.lclVerTypeInfo.GetClassHandleForObjRef()); + } + } + + assert(!lvaTable[tmpNum].IsAddressExposed()); + if (argInfo.argHasLdargaOp) + { + lvaTable[tmpNum].lvHasLdAddrOp = 1; + } -#ifdef DEBUG - if ((verbose || JitConfig.EnableExtraSuperPmiQueries()) && (numberOfClasses > 0)) - { - bool isExact; - bool isNonNull; - CallArg* thisArg = call->gtArgs.GetThisArg(); - CORINFO_CLASS_HANDLE declaredThisClsHnd = gtGetClassHandle(thisArg->GetNode(), &isExact, &isNonNull); - JITDUMP("Likely classes for call [%06u]", dspTreeID(call)); - if (declaredThisClsHnd != NO_CLASS_HANDLE) - { - const char* baseClassName = eeGetClassName(declaredThisClsHnd); - JITDUMP(" on class %p (%s)", declaredThisClsHnd, baseClassName); - } - JITDUMP("\n"); + if (lclInfo.lclVerTypeInfo.IsStruct()) + { + if (varTypeIsStruct(lclTyp)) + { + lvaSetStruct(tmpNum, lclInfo.lclVerTypeInfo.GetClassHandle(), true /* unsafe value cls check */); + if (info.compIsVarArgs) + { + lvaSetStructUsedAsVarArg(tmpNum); + } + } + } - for (UINT32 i = 0; i < numberOfClasses; i++) - { - const char* className = eeGetClassName((CORINFO_CLASS_HANDLE)likelyClasses[i].handle); - JITDUMP(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, likelyClasses[i].handle, className, - likelyClasses[i].likelihood); - } - } + argInfo.argHasTmp = true; + argInfo.argTmpNum = tmpNum; - if ((verbose || JitConfig.EnableExtraSuperPmiQueries()) && (numberOfMethods > 0)) - { - assert(call->gtCallType == CT_USER_FUNC); - const char* baseMethName = eeGetMethodFullName(call->gtCallMethHnd); - JITDUMP("Likely methods for call [%06u] to method %s\n", dspTreeID(call), baseMethName); + // If we require strict exception order, then arguments must + // be evaluated in sequence before the body of the inlined method. + // So we need to evaluate them to a temp. + // Also, if arguments have global or local references, we need to + // evaluate them to a temp before the inlined body as the + // inlined body may be modifying the global ref. + // TODO-1stClassStructs: We currently do not reuse an existing lclVar + // if it is a struct, because it requires some additional handling. - for (UINT32 i = 0; i < numberOfMethods; i++) - { - CORINFO_CONST_LOOKUP lookup = {}; - info.compCompHnd->getFunctionFixedEntryPoint((CORINFO_METHOD_HANDLE)likelyMethods[i].handle, false, - &lookup); + if ((!varTypeIsStruct(lclTyp) && !argInfo.argHasSideEff && !argInfo.argHasGlobRef && + !argInfo.argHasCallerLocalRef)) + { + /* Get a *LARGE* LCL_VAR node */ + op1 = gtNewLclLNode(tmpNum, genActualType(lclTyp) DEBUGARG(lclNum)); - const char* methName = eeGetMethodFullName((CORINFO_METHOD_HANDLE)likelyMethods[i].handle); - switch (lookup.accessType) + /* Record op1 as the very first use of this argument. + If there are no further uses of the arg, we may be + able to use the actual arg node instead of the temp. + If we do see any further uses, we will clear this. */ + argInfo.argBashTmpNode = op1; + } + else { - case IAT_VALUE: - JITDUMP(" %u) %p (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, - likelyMethods[i].likelihood); - break; - case IAT_PVALUE: - JITDUMP(" %u) [%p] (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, - likelyMethods[i].likelihood); - break; - case IAT_PPVALUE: - JITDUMP(" %u) [[%p]] (%s) [likelihood:%u%%]\n", i + 1, lookup.addr, methName, - likelyMethods[i].likelihood); - break; - default: - JITDUMP(" %u) %s [likelihood:%u%%]\n", i + 1, methName, likelyMethods[i].likelihood); - break; + /* Get a small LCL_VAR node */ + op1 = gtNewLclvNode(tmpNum, genActualType(lclTyp)); + /* No bashing of this argument */ + argInfo.argBashTmpNode = nullptr; } } } - // Optional stress mode to pick a random known class, rather than - // the most likely known class. - // - if (JitConfig.JitRandomGuardedDevirtualization() != 0) - { - // Reuse the random inliner's random state. - // - CLRRandom* const random = - impInlineRoot()->m_inlineStrategy->GetRandom(JitConfig.JitRandomGuardedDevirtualization()); - unsigned index = static_cast(random->Next(static_cast(numberOfClasses + numberOfMethods))); - if (index < numberOfClasses) - { - *classGuess = (CORINFO_CLASS_HANDLE)likelyClasses[index].handle; - *likelihood = 100; - JITDUMP("Picked random class for GDV: %p (%s)\n", *classGuess, eeGetClassName(*classGuess)); - return; - } - else - { - *methodGuess = (CORINFO_METHOD_HANDLE)likelyMethods[index - numberOfClasses].handle; - *likelihood = 100; - JITDUMP("Picked random method for GDV: %p (%s)\n", *methodGuess, eeGetMethodFullName(*methodGuess)); - return; - } - } -#endif + // Mark this argument as used. + argInfo.argIsUsed = true; - // Prefer class guess as it is cheaper - if (numberOfClasses > 0) - { - unsigned likelihoodThreshold = isInterface ? 25 : 30; - if (likelyClasses[0].likelihood >= likelihoodThreshold) - { - *classGuess = (CORINFO_CLASS_HANDLE)likelyClasses[0].handle; - *likelihood = likelyClasses[0].likelihood; - return; - } + return op1; +} - JITDUMP("Not guessing for class; likelihood is below %s call threshold %u\n", - isInterface ? "interface" : "virtual", likelihoodThreshold); - } +/****************************************************************************** + Is this the original "this" argument to the call being inlined? - if (numberOfMethods > 0) - { - unsigned likelihoodThreshold = 30; - if (likelyMethods[0].likelihood >= likelihoodThreshold) - { - *methodGuess = (CORINFO_METHOD_HANDLE)likelyMethods[0].handle; - *likelihood = likelyMethods[0].likelihood; - return; - } + Note that we do not inline methods with "starg 0", and so we do not need to + worry about it. +*/ - JITDUMP("Not guessing for method; likelihood is below %s call threshold %u\n", - call->IsDelegateInvoke() ? "delegate" : "virtual", likelihoodThreshold); - } +bool Compiler::impInlineIsThis(GenTree* tree, InlArgInfo* inlArgInfo) +{ + assert(compIsForInlining()); + return (tree->gtOper == GT_LCL_VAR && tree->AsLclVarCommon()->GetLclNum() == inlArgInfo[0].argTmpNum); } -//------------------------------------------------------------------------ -// isCompatibleMethodGDV: -// Check if devirtualizing a call node as a specified target method call is -// reasonable. +//----------------------------------------------------------------------------- +// impInlineIsGuaranteedThisDerefBeforeAnySideEffects: Check if a dereference in +// the inlinee can guarantee that the "this" pointer is non-NULL. // // Arguments: -// call - the call -// gdvTarget - the target method that we want to guess for and devirtualize to -// -// Returns: -// true if we can proceed with GDV. +// additionalTree - a tree to check for side effects +// additionalCallArgs - a list of call args to check for side effects +// dereferencedAddress - address expression being dereferenced +// inlArgInfo - inlinee argument information // // Notes: -// This implements a small simplified signature-compatibility check to -// verify that a guess is reasonable. The main goal here is to avoid blowing -// up the JIT on PGO data with stale GDV candidates; if they are not -// compatible in the ECMA sense then we do not expect the guard to ever pass -// at runtime, so we can get by with simplified rules here. +// If we haven't hit a branch or a side effect, and we are dereferencing +// from 'this' to access a field or make GTF_CALL_NULLCHECK call, +// then we can avoid a separate null pointer check. +// +// The importer stack and current statement list are searched for side effects. +// Trees that have been popped of the stack but haven't been appended to the +// statement list and have to be checked for side effects may be provided via +// additionalTree and additionalCallArgs. // -bool Compiler::isCompatibleMethodGDV(GenTreeCall* call, CORINFO_METHOD_HANDLE gdvTarget) +bool Compiler::impInlineIsGuaranteedThisDerefBeforeAnySideEffects(GenTree* additionalTree, + CallArgs* additionalCallArgs, + GenTree* dereferencedAddress, + InlArgInfo* inlArgInfo) { - CORINFO_SIG_INFO sig; - info.compCompHnd->getMethodSig(gdvTarget, &sig); - - CORINFO_ARG_LIST_HANDLE sigParam = sig.args; - unsigned numParams = sig.numArgs; - unsigned numArgs = 0; - for (CallArg& arg : call->gtArgs.Args()) - { - switch (arg.GetWellKnownArg()) - { - case WellKnownArg::RetBuffer: - case WellKnownArg::ThisPointer: - // Not part of signature but we still expect to see it here - continue; - case WellKnownArg::None: - break; - default: - assert(!"Unexpected well known arg to method GDV candidate"); - continue; - } - - numArgs++; - if (numArgs > numParams) - { - JITDUMP("Incompatible method GDV: call [%06u] has more arguments than signature (sig has %d parameters)\n", - dspTreeID(call), numParams); - return false; - } - - CORINFO_CLASS_HANDLE classHnd = NO_CLASS_HANDLE; - CorInfoType corType = strip(info.compCompHnd->getArgType(&sig, sigParam, &classHnd)); - var_types sigType = JITtype2varType(corType); - - if (!impCheckImplicitArgumentCoercion(sigType, arg.GetNode()->TypeGet())) - { - JITDUMP("Incompatible method GDV: arg [%06u] is type-incompatible with signature of target\n", - dspTreeID(arg.GetNode())); - return false; - } - - // Best-effort check for struct compatibility here. - if (varTypeIsStruct(sigType) && (arg.GetSignatureClassHandle() != classHnd)) - { - ClassLayout* callLayout = typGetObjLayout(arg.GetSignatureClassHandle()); - ClassLayout* tarLayout = typGetObjLayout(classHnd); - - if (!ClassLayout::AreCompatible(callLayout, tarLayout)) - { - JITDUMP("Incompatible method GDV: struct arg [%06u] is layout-incompatible with signature of target\n", - dspTreeID(arg.GetNode())); - return false; - } - } + assert(compIsForInlining()); + assert(opts.OptEnabled(CLFLG_INLINING)); - sigParam = info.compCompHnd->getArgNext(sigParam); - } + BasicBlock* block = compCurBB; - if (numArgs < numParams) + if (block != fgFirstBB) { - JITDUMP("Incompatible method GDV: call [%06u] has fewer arguments (%d) than signature (%d)\n", dspTreeID(call), - numArgs, numParams); return false; } - return true; -} - -//------------------------------------------------------------------------ -// considerGuardedDevirtualization: see if we can profitably guess at the -// class involved in an interface or virtual call. -// -// Arguments: -// -// call - potential guarded devirtualization candidate -// ilOffset - IL ofset of the call instruction -// baseMethod - target method of the call -// baseClass - class that introduced the target method -// pContextHandle - context handle for the call -// -// Notes: -// Consults with VM to see if there's a likely class at runtime, -// if so, adds a candidate for guarded devirtualization. -// -void Compiler::considerGuardedDevirtualization(GenTreeCall* call, - IL_OFFSET ilOffset, - bool isInterface, - CORINFO_METHOD_HANDLE baseMethod, - CORINFO_CLASS_HANDLE baseClass, - CORINFO_CONTEXT_HANDLE* pContextHandle) -{ - JITDUMP("Considering guarded devirtualization at IL offset %u (0x%x)\n", ilOffset, ilOffset); - - // We currently only get likely class guesses when there is PGO data - // with class profiles. - // - if ((fgPgoClassProfiles == 0) && (fgPgoMethodProfiles == 0)) + if (!impInlineIsThis(dereferencedAddress, inlArgInfo)) { - JITDUMP("Not guessing for class or method: no GDV profile pgo data, or pgo disabled\n"); - return; + return false; } - CORINFO_CLASS_HANDLE likelyClass; - CORINFO_METHOD_HANDLE likelyMethod; - unsigned likelihood; - pickGDV(call, ilOffset, isInterface, &likelyClass, &likelyMethod, &likelihood); - - if ((likelyClass == NO_CLASS_HANDLE) && (likelyMethod == NO_METHOD_HANDLE)) + if ((additionalTree != nullptr) && GTF_GLOBALLY_VISIBLE_SIDE_EFFECTS(additionalTree->gtFlags)) { - return; + return false; } - uint32_t likelyClassAttribs = 0; - if (likelyClass != NO_CLASS_HANDLE) + if (additionalCallArgs != nullptr) { - likelyClassAttribs = info.compCompHnd->getClassAttribs(likelyClass); - - if ((likelyClassAttribs & CORINFO_FLG_ABSTRACT) != 0) - { - // We may see an abstract likely class, if we have a stale profile. - // No point guessing for this. - // - JITDUMP("Not guessing for class; abstract (stale profile)\n"); - return; - } - - // Figure out which method will be called. - // - CORINFO_DEVIRTUALIZATION_INFO dvInfo; - dvInfo.virtualMethod = baseMethod; - dvInfo.objClass = likelyClass; - dvInfo.context = *pContextHandle; - dvInfo.exactContext = *pContextHandle; - dvInfo.pResolvedTokenVirtualMethod = nullptr; - - const bool canResolve = info.compCompHnd->resolveVirtualMethod(&dvInfo); - - if (!canResolve) + for (CallArg& arg : additionalCallArgs->Args()) { - JITDUMP("Can't figure out which method would be invoked, sorry\n"); - return; + if (GTF_GLOBALLY_VISIBLE_SIDE_EFFECTS(arg.GetEarlyNode()->gtFlags)) + { + return false; + } } - - likelyMethod = dvInfo.devirtualizedMethod; } - uint32_t likelyMethodAttribs = info.compCompHnd->getMethodAttribs(likelyMethod); - - if (likelyClass == NO_CLASS_HANDLE) + for (Statement* stmt : StatementList(impStmtList)) { - // For method GDV do a few more checks that we get for free in the - // resolve call above for class-based GDV. - if ((likelyMethodAttribs & CORINFO_FLG_STATIC) != 0) - { - assert(call->IsDelegateInvoke()); - JITDUMP("Cannot currently handle devirtualizing static delegate calls, sorry\n"); - return; - } - - CORINFO_CLASS_HANDLE definingClass = info.compCompHnd->getMethodClass(likelyMethod); - likelyClassAttribs = info.compCompHnd->getClassAttribs(definingClass); - - // For instance methods on value classes we need an extended check to - // check for the unboxing stub. This is NYI. - // Note: For dynamic PGO likelyMethod above will be the unboxing stub - // which would fail GDV for other reasons. - // However, with static profiles or textual PGO input it is still - // possible that likelyMethod is not the unboxing stub. So we do need - // this explicit check. - if ((likelyClassAttribs & CORINFO_FLG_VALUECLASS) != 0) + GenTree* expr = stmt->GetRootNode(); + if (GTF_GLOBALLY_VISIBLE_SIDE_EFFECTS(expr->gtFlags)) { - JITDUMP("Cannot currently handle devirtualizing delegate calls on value types, sorry\n"); - return; + return false; } + } - // Verify that the call target and args look reasonable so that the JIT - // does not blow up during inlining/call morphing. - // - // NOTE: Once we want to support devirtualization of delegate calls to - // static methods and remove the check above we will start failing here - // for delegates pointing to static methods that have the first arg - // bound. For example: - // - // public static void E(this C c) ... - // Action a = new C().E; - // - // The delegate instance looks exactly like one pointing to an instance - // method in this case and the call will have zero args while the - // signature has 1 arg. - // - if (!isCompatibleMethodGDV(call, likelyMethod)) + for (unsigned level = 0; level < verCurrentState.esStackDepth; level++) + { + GenTreeFlags stackTreeFlags = verCurrentState.esStack[level].val->gtFlags; + if (GTF_GLOBALLY_VISIBLE_SIDE_EFFECTS(stackTreeFlags)) { - JITDUMP("Target for method-based GDV is incompatible (stale profile?)\n"); - assert((fgPgoSource != ICorJitInfo::PgoSource::Dynamic) && "Unexpected stale profile in dynamic PGO data"); - return; + return false; } } - JITDUMP("%s call would invoke method %s\n", - isInterface ? "interface" : call->IsDelegateInvoke() ? "delegate" : "virtual", - eeGetMethodName(likelyMethod, nullptr)); - - // Add this as a potential candidate. - // - addGuardedDevirtualizationCandidate(call, likelyMethod, likelyClass, likelyMethodAttribs, likelyClassAttribs, - likelihood); + return true; } //------------------------------------------------------------------------ -// addGuardedDevirtualizationCandidate: potentially mark the call as a guarded -// devirtualization candidate -// -// Notes: -// -// Call sites in rare or unoptimized code, and calls that require cookies are -// not marked as candidates. -// -// As part of marking the candidate, the code spills GT_RET_EXPRs anywhere in any -// child tree, because and we need to clone all these trees when we clone the call -// as part of guarded devirtualization, and these IR nodes can't be cloned. +// impAllocateMethodPointerInfo: create methodPointerInfo into jit-allocated memory and init it. // // Arguments: -// call - potential guarded devirtualization candidate -// methodHandle - method that will be invoked if the class test succeeds -// classHandle - class that will be tested for at runtime -// methodAttr - attributes of the method -// classAttr - attributes of the class -// likelihood - odds that this class is the class seen at runtime +// token - init value for the allocated token. +// tokenConstrained - init value for the constraint associated with the token // -void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call, - CORINFO_METHOD_HANDLE methodHandle, - CORINFO_CLASS_HANDLE classHandle, - unsigned methodAttr, - unsigned classAttr, - unsigned likelihood) +// Return Value: +// pointer to token into jit-allocated memory. +methodPointerInfo* Compiler::impAllocateMethodPointerInfo(const CORINFO_RESOLVED_TOKEN& token, mdToken tokenConstrained) { - // This transformation only makes sense for delegate and virtual calls - assert(call->IsDelegateInvoke() || call->IsVirtual()); - - // Only mark calls if the feature is enabled. - const bool isEnabled = JitConfig.JitEnableGuardedDevirtualization() > 0; - - if (!isEnabled) - { - JITDUMP("NOT Marking call [%06u] as guarded devirtualization candidate -- disabled by jit config\n", - dspTreeID(call)); - return; - } - - // Bail if not optimizing or the call site is very likely cold - if (compCurBB->isRunRarely() || opts.OptimizationDisabled()) - { - JITDUMP("NOT Marking call [%06u] as guarded devirtualization candidate -- rare / dbg / minopts\n", - dspTreeID(call)); - return; - } - - // CT_INDIRECT calls may use the cookie, bail if so... - // - // If transforming these provides a benefit, we could save this off in the same way - // we save the stub address below. - if ((call->gtCallType == CT_INDIRECT) && (call->AsCall()->gtCallCookie != nullptr)) - { - JITDUMP("NOT Marking call [%06u] as guarded devirtualization candidate -- CT_INDIRECT with cookie\n", - dspTreeID(call)); - return; - } - -#ifdef DEBUG - - // See if disabled by range - // - static ConfigMethodRange JitGuardedDevirtualizationRange; - JitGuardedDevirtualizationRange.EnsureInit(JitConfig.JitGuardedDevirtualizationRange()); - assert(!JitGuardedDevirtualizationRange.Error()); - if (!JitGuardedDevirtualizationRange.Contains(impInlineRoot()->info.compMethodHash())) - { - JITDUMP("NOT Marking call [%06u] as guarded devirtualization candidate -- excluded by " - "JitGuardedDevirtualizationRange", - dspTreeID(call)); - return; - } - -#endif - - // We're all set, proceed with candidate creation. - // - JITDUMP("Marking call [%06u] as guarded devirtualization candidate; will guess for %s %s\n", dspTreeID(call), - classHandle != NO_CLASS_HANDLE ? "class" : "method", - classHandle != NO_CLASS_HANDLE ? eeGetClassName(classHandle) : eeGetMethodFullName(methodHandle)); - setMethodHasGuardedDevirtualization(); - call->SetGuardedDevirtualizationCandidate(); - - // Spill off any GT_RET_EXPR subtrees so we can clone the call. - // - SpillRetExprHelper helper(this); - helper.StoreRetExprResultsInArgs(call); - - // Gather some information for later. Note we actually allocate InlineCandidateInfo - // here, as the devirtualized half of this call will likely become an inline candidate. - // - GuardedDevirtualizationCandidateInfo* pInfo = new (this, CMK_Inlining) InlineCandidateInfo; - - pInfo->guardedMethodHandle = methodHandle; - pInfo->guardedMethodUnboxedEntryHandle = nullptr; - pInfo->guardedClassHandle = classHandle; - pInfo->likelihood = likelihood; - pInfo->requiresInstMethodTableArg = false; - - // If the guarded class is a value class, look for an unboxed entry point. - // - if ((classAttr & CORINFO_FLG_VALUECLASS) != 0) - { - JITDUMP(" ... class is a value class, looking for unboxed entry\n"); - bool requiresInstMethodTableArg = false; - CORINFO_METHOD_HANDLE unboxedEntryMethodHandle = - info.compCompHnd->getUnboxedEntry(methodHandle, &requiresInstMethodTableArg); - - if (unboxedEntryMethodHandle != nullptr) - { - JITDUMP(" ... updating GDV candidate with unboxed entry info\n"); - pInfo->guardedMethodUnboxedEntryHandle = unboxedEntryMethodHandle; - pInfo->requiresInstMethodTableArg = requiresInstMethodTableArg; - } - } - - call->gtGuardedDevirtualizationCandidateInfo = pInfo; + methodPointerInfo* memory = getAllocator(CMK_Unknown).allocate(1); + memory->m_token = token; + memory->m_tokenConstraint = tokenConstrained; + return memory; } void Compiler::addExpRuntimeLookupCandidate(GenTreeCall* call) @@ -23144,9 +14005,9 @@ bool Compiler::impCanSkipCovariantStoreCheck(GenTree* value, GenTree* array) return true; } // Non-0 const refs can only occur with frozen objects - assert(value->IsIconHandle(GTF_ICON_STR_HDL)); - assert(doesMethodHaveFrozenString() || - (compIsForInlining() && impInlineInfo->InlinerCompiler->doesMethodHaveFrozenString())); + assert(value->IsIconHandle(GTF_ICON_OBJ_HDL)); + assert(doesMethodHaveFrozenObjects() || + (compIsForInlining() && impInlineInfo->InlinerCompiler->doesMethodHaveFrozenObjects())); } // Try and get a class handle for the array From 7adb3633207edc3b74ca35eb80f0b199e9af4fb4 Mon Sep 17 00:00:00 2001 From: petris Date: Fri, 16 Dec 2022 18:13:21 +0100 Subject: [PATCH 22/31] Reorganize tests --- .../MemoryMarshalGetArrayDataReference.cs | 48 +++++++++++-------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs index 34db736c283b5b..23632606631c9e 100644 --- a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs +++ b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs @@ -172,11 +172,37 @@ static int Problem4(byte[] a) Equals(Problem4(new byte[] { 1, 1 }), 0); - try + [MethodImpl(MethodImplOptions.NoInlining)] + static int Problem5() { int[] inputArray = CreateArray(17); Create(out Vector v, inputArray, inputArray.Length); + + static void Create(out Vector result, int[] values, int index) + { + // We explicitly don't check for `null` because historically this has thrown `NullReferenceException` for perf reasons + + if ((index < 0) || ((values.Length - index) < Vector.Count)) + { + ThrowArgumentOutOfRangeException(); + } + + result = Unsafe.ReadUnaligned>(ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(values), index))); + } + + static void ThrowArgumentOutOfRangeException() + { + throw new ArgumentOutOfRangeException(); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static int[] CreateArray(int size) => new int[size]; + } + + try + { + Problem5(); } catch { @@ -186,26 +212,6 @@ static int Problem4(byte[] a) return 100 + _errors; } - public static void Create(out Vector result, int[] values, int index) - { - // We explicitly don't check for `null` because historically this has thrown `NullReferenceException` for perf reasons - - if ((index < 0) || ((values.Length - index) < Vector.Count)) - { - ThrowArgumentOutOfRangeException(); - } - - result = Unsafe.ReadUnaligned>(ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(values), index))); - } - - public static void ThrowArgumentOutOfRangeException() - { - throw new ArgumentOutOfRangeException(); - } - - [MethodImpl(MethodImplOptions.NoInlining)] - public static int[] CreateArray(int size) => new int[size]; - [MethodImpl(MethodImplOptions.NoInlining)] static void Equals(T left, T right, [CallerLineNumber] int line = 0, [CallerFilePath] string file = "") { From 5d177129d54a7f422ae9a3551c9b5d362fe9ca51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Petryka?= <35800402+MichalPetryka@users.noreply.github.com> Date: Fri, 16 Dec 2022 19:27:25 +0100 Subject: [PATCH 23/31] Update MemoryMarshalGetArrayDataReference.cs --- src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs index 23632606631c9e..00ed190a8ef988 100644 --- a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs +++ b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; +using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; From aade930bae0514e1cb03f9b48e4fb89343d15a98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Petryka?= <35800402+MichalPetryka@users.noreply.github.com> Date: Fri, 16 Dec 2022 20:06:53 +0100 Subject: [PATCH 24/31] Update MemoryMarshalGetArrayDataReference.cs --- src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs index 00ed190a8ef988..66ceec81df4906 100644 --- a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs +++ b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs @@ -174,7 +174,7 @@ static int Problem4(byte[] a) Equals(Problem4(new byte[] { 1, 1 }), 0); [MethodImpl(MethodImplOptions.NoInlining)] - static int Problem5() + static void Problem5() { int[] inputArray = CreateArray(17); From 47641528fcd5c441fc4c388a3a88a8a9f19d6a01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Petryka?= <35800402+MichalPetryka@users.noreply.github.com> Date: Fri, 16 Dec 2022 22:45:32 +0100 Subject: [PATCH 25/31] Test --- src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs index 66ceec81df4906..58d2f127b6a56f 100644 --- a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs +++ b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs @@ -201,9 +201,10 @@ static void ThrowArgumentOutOfRangeException() static int[] CreateArray(int size) => new int[size]; } + Problem5(); try { - Problem5(); + } catch { From bfbf746a3ab33e44412dd4f5bff61e8738469333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Petryka?= <35800402+MichalPetryka@users.noreply.github.com> Date: Sat, 17 Dec 2022 11:42:41 +0100 Subject: [PATCH 26/31] Update MemoryMarshalGetArrayDataReference.cs --- .../JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs index 58d2f127b6a56f..c59af9750a057d 100644 --- a/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs +++ b/src/tests/JIT/Intrinsics/MemoryMarshalGetArrayDataReference.cs @@ -201,10 +201,14 @@ static void ThrowArgumentOutOfRangeException() static int[] CreateArray(int size) => new int[size]; } - Problem5(); try { - + Problem5(); + _errors++; + } + catch (ArgumentOutOfRangeException) + { + // expected } catch { From cf597a89b1215ca3cda455a88a9c35b3ec75de96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Petryka?= <35800402+MichalPetryka@users.noreply.github.com> Date: Sat, 17 Dec 2022 15:29:09 +0100 Subject: [PATCH 27/31] Update importercalls.cpp --- src/coreclr/jit/importercalls.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index d46457470d6b9d..a723a952a8291c 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -2610,14 +2610,18 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, CorInfoType jitType = info.compCompHnd->asCorInfoType(elemHnd); var_types elemType = JITtype2varType(jitType); - GenTree* arrayClone; - array = impCloneExpr(array, &arrayClone, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, - nullptr DEBUGARG("MemoryMarshal.GetArrayDataReference array")); + if (fgAddrCanBeNull(array)) + { + GenTree* arrayClone; + array = impCloneExpr(array, &arrayClone, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, + nullptr DEBUGARG("MemoryMarshal.GetArrayDataReference array")); - impAppendTree(gtNewNullCheck(array, compCurBB), (unsigned)CHECK_SPILL_ALL, impCurStmtDI); + impAppendTree(gtNewNullCheck(array, compCurBB), (unsigned)CHECK_SPILL_ALL, impCurStmtDI); + array = arrayClone; + } GenTree* index = gtNewIconNode(0, TYP_I_IMPL); - GenTreeIndexAddr* indexAddr = gtNewArrayIndexAddr(arrayClone, index, elemType, elemHnd); + GenTreeIndexAddr* indexAddr = gtNewArrayIndexAddr(array, index, elemType, elemHnd); indexAddr->gtFlags &= ~GTF_INX_RNGCHK; indexAddr->gtFlags |= GTF_INX_ADDR_NONNULL; retNode = indexAddr; From 110479d4d2c7eea445e8c49de4c6a84fd33f91d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Petryka?= <35800402+MichalPetryka@users.noreply.github.com> Date: Sat, 17 Dec 2022 15:29:44 +0100 Subject: [PATCH 28/31] Update compiler.hpp --- src/coreclr/jit/compiler.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index c153944aa38f68..6adb9e80327808 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -918,7 +918,7 @@ inline GenTreeIntCon* Compiler::gtNewIconHandleNode(size_t value, GenTreeFlags f #if defined(LATE_DISASM) node = new (this, LargeOpOpcode()) GenTreeIntCon(TYP_I_IMPL, value, fields DEBUGARG(/*largeNode*/ true)); #else - node = new (this, GT_CNS_INT) GenTreeIntCon(TYP_I_IMPL, value, fields); + node = new (this, GT_CNS_INT) GenTreeIntCon(TYP_I_IMPL, value, fields); #endif node->gtFlags |= flags; return node; From 3ac696987d7d2c29f971f6d684fcfd5aabf181f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Petryka?= <35800402+MichalPetryka@users.noreply.github.com> Date: Sat, 17 Dec 2022 15:49:14 +0100 Subject: [PATCH 29/31] Update compiler.hpp --- src/coreclr/jit/compiler.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 6adb9e80327808..c153944aa38f68 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -918,7 +918,7 @@ inline GenTreeIntCon* Compiler::gtNewIconHandleNode(size_t value, GenTreeFlags f #if defined(LATE_DISASM) node = new (this, LargeOpOpcode()) GenTreeIntCon(TYP_I_IMPL, value, fields DEBUGARG(/*largeNode*/ true)); #else - node = new (this, GT_CNS_INT) GenTreeIntCon(TYP_I_IMPL, value, fields); + node = new (this, GT_CNS_INT) GenTreeIntCon(TYP_I_IMPL, value, fields); #endif node->gtFlags |= flags; return node; From 41576752b20142cac7ec62bcc8020903e3d4a459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Petryka?= <35800402+MichalPetryka@users.noreply.github.com> Date: Sat, 17 Dec 2022 15:49:44 +0100 Subject: [PATCH 30/31] Update importercalls.cpp --- src/coreclr/jit/importercalls.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index a723a952a8291c..0693697b77ab72 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -2610,7 +2610,7 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, CorInfoType jitType = info.compCompHnd->asCorInfoType(elemHnd); var_types elemType = JITtype2varType(jitType); - if (fgAddrCanBeNull(array)) + if (fgAddrCouldBeNull(array)) { GenTree* arrayClone; array = impCloneExpr(array, &arrayClone, NO_CLASS_HANDLE, (unsigned)CHECK_SPILL_ALL, From 359e41a3c1860406ba0c87d4a649137fbe989b38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Petryka?= <35800402+MichalPetryka@users.noreply.github.com> Date: Thu, 5 Jan 2023 21:41:30 +0100 Subject: [PATCH 31/31] Add an assert --- src/coreclr/jit/importercalls.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 0693697b77ab72..c381b3a74a1dd1 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -2604,6 +2604,7 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, case NI_System_Runtime_InteropService_MemoryMarshal_GetArrayDataReference: { assert(sig->numArgs == 1); + assert(sig->sigInst.methInstCount == 1); GenTree* array = impPopStack().val; CORINFO_CLASS_HANDLE elemHnd = sig->sigInst.methInst[0];