diff --git a/src/coreclr/jit/CMakeLists.txt b/src/coreclr/jit/CMakeLists.txt index 6b6ce92bbfea8f..b1410853c73359 100644 --- a/src/coreclr/jit/CMakeLists.txt +++ b/src/coreclr/jit/CMakeLists.txt @@ -134,6 +134,7 @@ set( JIT_SOURCES inline.cpp inlinepolicy.cpp instr.cpp + integralrange.cpp jitconfig.cpp jiteh.cpp jithashtable.cpp @@ -178,6 +179,7 @@ set( JIT_SOURCES treelifeupdater.cpp utils.cpp valuenum.cpp + vnbasedmorph.cpp ) set ( JIT_NATIVE_TARGET_SOURCES diff --git a/src/coreclr/jit/assertionprop.cpp b/src/coreclr/jit/assertionprop.cpp index 6290f6be85fee6..ab2319079058d7 100644 --- a/src/coreclr/jit/assertionprop.cpp +++ b/src/coreclr/jit/assertionprop.cpp @@ -16,1616 +16,446 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #pragma hdrstop #endif -//------------------------------------------------------------------------ -// Contains: Whether the range contains a given integral value, inclusive. -// -// Arguments: -// value - the integral value in question -// -// Return Value: -// "true" if the value is within the range's bounds, "false" otherwise. -// -bool IntegralRange::Contains(int64_t value) const -{ - int64_t lowerBound = SymbolicToRealValue(m_lowerBound); - int64_t upperBound = SymbolicToRealValue(m_upperBound); - - return (lowerBound <= value) && (value <= upperBound); -} - -//------------------------------------------------------------------------ -// SymbolicToRealValue: Convert a symbolic value to a 64-bit signed integer. +//------------------------------------------------------------------------------ +// GetAssertionDep: Retrieve the assertions on this local variable // // Arguments: -// value - the symbolic value in question +// lclNum - The local var id. +// mustExist - If true, assert that the dependent assertions must exist. // // Return Value: -// Integer corresponding to the symbolic value. +// The dependent assertions (assertions using the value of the local var) +// of the local var. // -/* static */ int64_t IntegralRange::SymbolicToRealValue(SymbolicIntegerValue value) -{ - static const int64_t SymbolicToRealMap[]{ - INT64_MIN, // SymbolicIntegerValue::LongMin - INT32_MIN, // SymbolicIntegerValue::IntMin - INT16_MIN, // SymbolicIntegerValue::ShortMin - INT8_MIN, // SymbolicIntegerValue::ByteMin - 0, // SymbolicIntegerValue::Zero - 1, // SymbolicIntegerValue::One - INT8_MAX, // SymbolicIntegerValue::ByteMax - UINT8_MAX, // SymbolicIntegerValue::UByteMax - INT16_MAX, // SymbolicIntegerValue::ShortMax - UINT16_MAX, // SymbolicIntegerValue::UShortMax - CORINFO_Array_MaxLength, // SymbolicIntegerValue::ArrayLenMax - INT32_MAX, // SymbolicIntegerValue::IntMax - UINT32_MAX, // SymbolicIntegerValue::UIntMax - INT64_MAX // SymbolicIntegerValue::LongMax - }; - - assert(sizeof(SymbolicIntegerValue) == sizeof(int32_t)); - assert(SymbolicToRealMap[static_cast(SymbolicIntegerValue::LongMin)] == INT64_MIN); - assert(SymbolicToRealMap[static_cast(SymbolicIntegerValue::Zero)] == 0); - assert(SymbolicToRealMap[static_cast(SymbolicIntegerValue::LongMax)] == INT64_MAX); - return SymbolicToRealMap[static_cast(value)]; -} - -//------------------------------------------------------------------------ -// LowerBoundForType: Get the symbolic lower bound for a type. -// -// Arguments: -// type - the integral type in question -// -// Return Value: -// Symbolic value representing the smallest possible value "type" can represent. -// -/* static */ SymbolicIntegerValue IntegralRange::LowerBoundForType(var_types type) +ASSERT_TP& Compiler::GetAssertionDep(unsigned lclNum, bool mustExist) { - switch (type) + JitExpandArray& dep = *optAssertionDep; + if (dep[lclNum] == nullptr) { - case TYP_UBYTE: - case TYP_USHORT: - return SymbolicIntegerValue::Zero; - case TYP_BYTE: - return SymbolicIntegerValue::ByteMin; - case TYP_SHORT: - return SymbolicIntegerValue::ShortMin; - case TYP_INT: - return SymbolicIntegerValue::IntMin; - case TYP_LONG: - return SymbolicIntegerValue::LongMin; - default: - unreached(); + if (mustExist) + { + assert(!"No dependent assertions for local var"); + } + dep[lclNum] = BitVecOps::MakeEmpty(apTraits); } + return dep[lclNum]; } -//------------------------------------------------------------------------ -// UpperBoundForType: Get the symbolic upper bound for a type. -// -// Arguments: -// type - the integral type in question -// -// Return Value: -// Symbolic value representing the largest possible value "type" can represent. -// -/* static */ SymbolicIntegerValue IntegralRange::UpperBoundForType(var_types type) +/***************************************************************************** + * + * Initialize the assertion prop bitset traits and the default bitsets. + */ + +void Compiler::optAssertionTraitsInit(AssertionIndex assertionCount) { - switch (type) - { - case TYP_BYTE: - return SymbolicIntegerValue::ByteMax; - case TYP_UBYTE: - return SymbolicIntegerValue::UByteMax; - case TYP_SHORT: - return SymbolicIntegerValue::ShortMax; - case TYP_USHORT: - return SymbolicIntegerValue::UShortMax; - case TYP_INT: - return SymbolicIntegerValue::IntMax; - case TYP_UINT: - return SymbolicIntegerValue::UIntMax; - case TYP_LONG: - return SymbolicIntegerValue::LongMax; - default: - unreached(); - } + apTraits = new (this, CMK_AssertionProp) BitVecTraits(assertionCount, this); + apFull = BitVecOps::MakeFull(apTraits); } -//------------------------------------------------------------------------ -// ForNode: Compute the integral range for a node. -// -// Arguments: -// node - the node, of an integral type, in question -// compiler - the Compiler, used to retrieve additional info -// -// Return Value: -// The integral range this node produces. -// -/* static */ IntegralRange IntegralRange::ForNode(GenTree* node, Compiler* compiler) -{ - assert(varTypeIsIntegral(node)); +/***************************************************************************** + * + * Initialize the assertion prop tracking logic. + */ - var_types rangeType = node->TypeGet(); +void Compiler::optAssertionInit(bool isLocalProp) +{ + assert(NO_ASSERTION_INDEX == 0); + const unsigned maxTrackedLocals = (unsigned)JitConfig.JitMaxLocalsToTrack(); - switch (node->OperGet()) + // We initialize differently for local prop / global prop + // + if (isLocalProp) { - case GT_EQ: - case GT_NE: - case GT_LT: - case GT_LE: - case GT_GE: - case GT_GT: - return {SymbolicIntegerValue::Zero, SymbolicIntegerValue::One}; + optLocalAssertionProp = true; + optCrossBlockLocalAssertionProp = true; - case GT_AND: + // Disable via config + // + if (JitConfig.JitEnableCrossBlockLocalAssertionProp() == 0) { - IntegralRange leftRange = IntegralRange::ForNode(node->gtGetOp1(), compiler); - IntegralRange rightRange = IntegralRange::ForNode(node->gtGetOp2(), compiler); - if (leftRange.IsNonNegative() && rightRange.IsNonNegative()) - { - // If both sides are known to be non-negative, the result is non-negative. - // Further, the top end of the range cannot exceed the min of the two upper bounds. - return {SymbolicIntegerValue::Zero, min(leftRange.GetUpperBound(), rightRange.GetUpperBound())}; - } - - if (leftRange.IsNonNegative() || rightRange.IsNonNegative()) - { - // If only one side is known to be non-negative, however it is harder to - // reason about the upper bound. - return {SymbolicIntegerValue::Zero, UpperBoundForType(rangeType)}; - } - - break; + JITDUMP("Disabling cross-block assertion prop by config setting\n"); + optCrossBlockLocalAssertionProp = false; } - case GT_ARR_LENGTH: - case GT_MDARR_LENGTH: - return {SymbolicIntegerValue::Zero, SymbolicIntegerValue::ArrayLenMax}; - - case GT_CALL: - if (node->AsCall()->NormalizesSmallTypesOnReturn()) - { - rangeType = static_cast(node->AsCall()->gtReturnType); - } - break; - - case GT_IND: +#ifdef DEBUG + // Disable per method via range + // + static ConfigMethodRange s_range; + s_range.EnsureInit(JitConfig.JitEnableCrossBlockLocalAssertionPropRange()); + if (!s_range.Contains(info.compMethodHash())) { - GenTree* const addr = node->AsIndir()->Addr(); - - if (node->TypeIs(TYP_INT) && addr->OperIs(GT_ADD) && addr->gtGetOp1()->OperIs(GT_LCL_VAR) && - addr->gtGetOp2()->IsIntegralConst(OFFSETOF__CORINFO_Span__length)) - { - GenTreeLclVar* const lclVar = addr->gtGetOp1()->AsLclVar(); - - if (compiler->lvaGetDesc(lclVar->GetLclNum())->IsSpan()) - { - assert(compiler->lvaIsImplicitByRefLocal(lclVar->GetLclNum())); - return {SymbolicIntegerValue::Zero, UpperBoundForType(rangeType)}; - } - } - break; + JITDUMP("Disabling cross-block assertion prop by config range\n"); + optCrossBlockLocalAssertionProp = false; } +#endif - case GT_LCL_FLD: + // Disable if too many locals + // + // The typical number of local assertions is roughly proportional + // to the number of locals. So when we have huge numbers of locals, + // just do within-block local assertion prop. + // + if (lvaCount > maxTrackedLocals) { - GenTreeLclFld* const lclFld = node->AsLclFld(); - LclVarDsc* const varDsc = compiler->lvaGetDesc(lclFld); - - if (node->TypeIs(TYP_INT) && varDsc->IsSpan() && lclFld->GetLclOffs() == OFFSETOF__CORINFO_Span__length) - { - return {SymbolicIntegerValue::Zero, UpperBoundForType(rangeType)}; - } - - break; + JITDUMP("Disabling cross-block assertion prop: too many locals\n"); + optCrossBlockLocalAssertionProp = false; } - case GT_LCL_VAR: + if (optCrossBlockLocalAssertionProp) { - LclVarDsc* const varDsc = compiler->lvaGetDesc(node->AsLclVar()); - - if (varDsc->lvNormalizeOnStore()) - { - rangeType = compiler->lvaGetDesc(node->AsLclVar())->TypeGet(); - } - - if (varDsc->IsNeverNegative()) + // We may need a fairly large table. Keep size a multiple of 64. + // Empirical studies show about 1.16 asserions/ tracked local. + // + if (lvaTrackedCount < 24) { - return {SymbolicIntegerValue::Zero, UpperBoundForType(rangeType)}; + optMaxAssertionCount = 64; } - break; - } - - case GT_CNS_INT: - { - if (node->IsIntegralConst(0) || node->IsIntegralConst(1)) + else if (lvaTrackedCount < 64) { - return {SymbolicIntegerValue::Zero, SymbolicIntegerValue::One}; + optMaxAssertionCount = 128; } - - int64_t constValue = node->AsIntCon()->IntegralValue(); - if (constValue >= 0) + else { - return {SymbolicIntegerValue::Zero, UpperBoundForType(rangeType)}; + optMaxAssertionCount = (AssertionIndex)min(maxTrackedLocals, ((3 * lvaTrackedCount / 128) + 1) * 64); } - break; + JITDUMP("Cross-block table size %u (for %u tracked locals)\n", optMaxAssertionCount, lvaTrackedCount); + } + else + { + // The assertion table will be reset for each block, so it can be smaller. + // + optMaxAssertionCount = 64; } - case GT_QMARK: - return Union(ForNode(node->AsQmark()->ThenNode(), compiler), - ForNode(node->AsQmark()->ElseNode(), compiler)); - - case GT_CAST: - return ForCastOutput(node->AsCast(), compiler); - -#if defined(FEATURE_HW_INTRINSICS) - case GT_HWINTRINSIC: - switch (node->AsHWIntrinsic()->GetHWIntrinsicId()) - { -#if defined(TARGET_XARCH) - case NI_Vector128_op_Equality: - case NI_Vector128_op_Inequality: - case NI_Vector256_op_Equality: - case NI_Vector256_op_Inequality: - case NI_Vector512_op_Equality: - case NI_Vector512_op_Inequality: - case NI_X86Base_CompareScalarOrderedEqual: - case NI_X86Base_CompareScalarOrderedNotEqual: - case NI_X86Base_CompareScalarOrderedLessThan: - case NI_X86Base_CompareScalarOrderedLessThanOrEqual: - case NI_X86Base_CompareScalarOrderedGreaterThan: - case NI_X86Base_CompareScalarOrderedGreaterThanOrEqual: - case NI_X86Base_CompareScalarUnorderedEqual: - case NI_X86Base_CompareScalarUnorderedNotEqual: - case NI_X86Base_CompareScalarUnorderedLessThanOrEqual: - case NI_X86Base_CompareScalarUnorderedLessThan: - case NI_X86Base_CompareScalarUnorderedGreaterThanOrEqual: - case NI_X86Base_CompareScalarUnorderedGreaterThan: - case NI_X86Base_TestC: - case NI_X86Base_TestZ: - case NI_X86Base_TestNotZAndNotC: - case NI_AVX_TestC: - case NI_AVX_TestZ: - case NI_AVX_TestNotZAndNotC: - return {SymbolicIntegerValue::Zero, SymbolicIntegerValue::One}; - - case NI_X86Base_Extract: - case NI_X86Base_X64_Extract: - case NI_Vector128_ToScalar: - case NI_Vector256_ToScalar: - case NI_Vector512_ToScalar: - case NI_Vector128_GetElement: - case NI_Vector256_GetElement: - case NI_Vector512_GetElement: - if (varTypeIsSmall(node->AsHWIntrinsic()->GetSimdBaseType())) - { - return ForType(node->AsHWIntrinsic()->GetSimdBaseType()); - } - break; + // Local assertion prop keeps mappings from each local var to the assertions about that var. + // + optAssertionDep = + new (this, CMK_AssertionProp) JitExpandArray(getAllocator(CMK_AssertionProp), max(1u, lvaCount)); - case NI_AVX2_LeadingZeroCount: - case NI_AVX2_TrailingZeroCount: - case NI_AVX2_X64_LeadingZeroCount: - case NI_AVX2_X64_TrailingZeroCount: - case NI_X86Base_PopCount: - case NI_X86Base_X64_PopCount: - // Note: No advantage in using a precise range for IntegralRange. - // Example: IntCns = 42 gives [0..127] with a non -precise range, [42,42] with a precise range. - return {SymbolicIntegerValue::Zero, SymbolicIntegerValue::ByteMax}; -#elif defined(TARGET_ARM64) - case NI_Vector64_op_Equality: - case NI_Vector64_op_Inequality: - case NI_Vector128_op_Equality: - case NI_Vector128_op_Inequality: - return {SymbolicIntegerValue::Zero, SymbolicIntegerValue::One}; - - case NI_AdvSimd_Extract: - case NI_Vector64_ToScalar: - case NI_Vector128_ToScalar: - case NI_Vector64_GetElement: - case NI_Vector128_GetElement: - if (varTypeIsSmall(node->AsHWIntrinsic()->GetSimdBaseType())) - { - return ForType(node->AsHWIntrinsic()->GetSimdBaseType()); - } - break; + if (optCrossBlockLocalAssertionProp) + { + optComplementaryAssertionMap = new (this, CMK_AssertionProp) + AssertionIndex[optMaxAssertionCount + 1](); // zero-inited (NO_ASSERTION_INDEX) + } + } + else + { + // General assertion prop. + // + optLocalAssertionProp = false; + optCrossBlockLocalAssertionProp = false; - case NI_AdvSimd_PopCount: - case NI_AdvSimd_LeadingZeroCount: - case NI_AdvSimd_LeadingSignCount: - case NI_ArmBase_LeadingZeroCount: - case NI_ArmBase_Arm64_LeadingZeroCount: - case NI_ArmBase_Arm64_LeadingSignCount: - // Note: No advantage in using a precise range for IntegralRange. - // Example: IntCns = 42 gives [0..127] with a non -precise range, [42,42] with a precise range. - return {SymbolicIntegerValue::Zero, SymbolicIntegerValue::ByteMax}; -#else -#error Unsupported platform -#endif - default: - break; - } - break; -#endif // defined(FEATURE_HW_INTRINSICS) + // Heuristic for sizing the assertion table. + // + // The weighting of basicBlocks vs locals reflects their relative contribution observed empirically. + // Validated against 1,115,046 compiled methods: + // - 94.6% of methods stay at the floor of 64 (only 1.9% actually need more). + // - Underpredicts for 481 methods (0.043%), with a worst-case deficit of 127. + // - Only 0.4% of methods hit the 256 cap. + optMaxAssertionCount = (AssertionIndex)max(64, min(256, (int)(lvaTrackedCount + 3 * fgBBcount + 48) >> 2)); - default: - break; + optComplementaryAssertionMap = new (this, CMK_AssertionProp) + AssertionIndex[optMaxAssertionCount + 1](); // zero-inited (NO_ASSERTION_INDEX) } - return ForType(rangeType); + optAssertionTabPrivate = new (this, CMK_AssertionProp) AssertionDsc[optMaxAssertionCount]; + optAssertionTraitsInit(optMaxAssertionCount); + + optAssertionCount = 0; + optAssertionOverflow = 0; + optAssertionPropagated = false; + bbJtrueAssertionOut = nullptr; + optCanPropLclVar = false; + optCanPropEqual = false; + optCanPropNonNull = false; + optCanPropBndsChk = false; + optCanPropSubRange = false; } -//------------------------------------------------------------------------ -// ForCastInput: Get the non-overflowing input range for a cast. -// -// This routine computes the input range for a cast from -// an integer to an integer for which it will not overflow. -// See also the specification comment for IntegralRange. -// -// Arguments: -// cast - the cast node for which the range will be computed -// -// Return Value: -// The range this cast consumes without overflowing - see description. -// -/* static */ IntegralRange IntegralRange::ForCastInput(GenTreeCast* cast) +#ifdef DEBUG +void Compiler::optPrintAssertion(const AssertionDsc& curAssertion, AssertionIndex assertionIndex /* = 0 */) { - var_types fromType = genActualType(cast->CastOp()); - var_types toType = cast->CastToType(); - bool fromUnsigned = cast->IsUnsigned(); - - assert((fromType == TYP_INT) || (fromType == TYP_LONG) || varTypeIsGC(fromType)); - assert(varTypeIsIntegral(toType)); - - // Cast from a GC type is the same as a cast from TYP_I_IMPL for our purposes. - if (varTypeIsGC(fromType)) + // Print assertion index if provided + if (assertionIndex > 0) { - fromType = TYP_I_IMPL; + optPrintAssertionIndex(assertionIndex); + printf(" "); } - if (!cast->gtOverflow()) + switch (curAssertion.GetOp1().GetKind()) { - // CAST(small type <- uint/int/ulong/long) - [TO_TYPE_MIN..TO_TYPE_MAX] - if (varTypeIsSmall(toType)) - { - return {LowerBoundForType(toType), UpperBoundForType(toType)}; - } - - // We choose to say here that representation-changing casts never overflow. - // It does not really matter what we do here because representation-changing - // non-overflowing casts cannot be deleted from the IR in any case. - // CAST(uint/int <- uint/int) - [INT_MIN..INT_MAX] - // CAST(uint/int <- ulong/long) - [LONG_MIN..LONG_MAX] - // CAST(ulong/long <- uint/int) - [INT_MIN..INT_MAX] - // CAST(ulong/long <- ulong/long) - [LONG_MIN..LONG_MAX] - return ForType(fromType); - } - - SymbolicIntegerValue lowerBound; - SymbolicIntegerValue upperBound; + case O1K_LCLVAR: + if (optLocalAssertionProp) + { + printf("lclvar V%02u", curAssertion.GetOp1().GetLclNum()); + } + else + { + printf("lclvar " FMT_VN "", curAssertion.GetOp1().GetVN()); + } + break; - // CAST_OVF(small type <- int/long) - [TO_TYPE_MIN..TO_TYPE_MAX] - // CAST_OVF(small type <- uint/ulong) - [0..TO_TYPE_MAX] - if (varTypeIsSmall(toType)) - { - lowerBound = fromUnsigned ? SymbolicIntegerValue::Zero : LowerBoundForType(toType); - upperBound = UpperBoundForType(toType); + case O1K_VN: + printf("VN " FMT_VN "", curAssertion.GetOp1().GetVN()); + break; + + case O1K_EXACT_TYPE: + printf("ExactType " FMT_VN "", curAssertion.GetOp1().GetVN()); + break; + + case O1K_SUBTYPE: + printf("SubType " FMT_VN "", curAssertion.GetOp1().GetVN()); + break; + + default: + unreached(); + break; } - else - { - switch (toType) - { - // CAST_OVF(uint <- uint) - [INT_MIN..INT_MAX] - // CAST_OVF(uint <- int) - [0..INT_MAX] - // CAST_OVF(uint <- ulong/long) - [0..UINT_MAX] - case TYP_UINT: - if (fromType == TYP_LONG) - { - lowerBound = SymbolicIntegerValue::Zero; - upperBound = SymbolicIntegerValue::UIntMax; - } - else - { - lowerBound = fromUnsigned ? SymbolicIntegerValue::IntMin : SymbolicIntegerValue::Zero; - upperBound = SymbolicIntegerValue::IntMax; - } - break; - // CAST_OVF(int <- uint/ulong) - [0..INT_MAX] - // CAST_OVF(int <- int/long) - [INT_MIN..INT_MAX] - case TYP_INT: - lowerBound = fromUnsigned ? SymbolicIntegerValue::Zero : SymbolicIntegerValue::IntMin; - upperBound = SymbolicIntegerValue::IntMax; - break; + switch (curAssertion.GetKind()) + { + case OAK_EQUAL: + printf(" == "); + break; - // CAST_OVF(ulong <- uint) - [INT_MIN..INT_MAX] - // CAST_OVF(ulong <- int) - [0..INT_MAX] - // CAST_OVF(ulong <- ulong) - [LONG_MIN..LONG_MAX] - // CAST_OVF(ulong <- long) - [0..LONG_MAX] - case TYP_ULONG: - lowerBound = fromUnsigned ? LowerBoundForType(fromType) : SymbolicIntegerValue::Zero; - upperBound = UpperBoundForType(fromType); - break; + case OAK_NOT_EQUAL: + printf(" != "); + break; - // CAST_OVF(long <- uint/int) - [INT_MIN..INT_MAX] - // CAST_OVF(long <- ulong) - [0..LONG_MAX] - // CAST_OVF(long <- long) - [LONG_MIN..LONG_MAX] - case TYP_LONG: - if (fromUnsigned && (fromType == TYP_LONG)) - { - lowerBound = SymbolicIntegerValue::Zero; - } - else - { - lowerBound = LowerBoundForType(fromType); - } - upperBound = UpperBoundForType(fromType); - break; + case OAK_LT: + printf(" < "); + break; - default: - unreached(); - } - } + case OAK_LT_UN: + printf(" u< "); + break; - return {lowerBound, upperBound}; -} + case OAK_LE: + printf(" <= "); + break; -//------------------------------------------------------------------------ -// ForCastOutput: Get the output range for a cast. -// -// This method is the "output" counterpart to ForCastInput, it returns -// a range produced by a cast (by definition, non-overflowing one). -// The output range is the same for representation-preserving casts, but -// can be different for others. One example is CAST_OVF(uint <- long). -// The input range is [0..UINT_MAX], while the output is [INT_MIN..INT_MAX]. -// Unlike ForCastInput, this method supports casts from floating point types. -// -// Arguments: -// cast - the cast node for which the range will be computed -// compiler - Compiler object -// -// Return Value: -// The range this cast produces - see description. -// -/* static */ IntegralRange IntegralRange::ForCastOutput(GenTreeCast* cast, Compiler* compiler) -{ - var_types fromType = genActualType(cast->CastOp()); - var_types toType = cast->CastToType(); - bool fromUnsigned = cast->IsUnsigned(); + case OAK_LE_UN: + printf(" u<= "); + break; - assert((fromType == TYP_INT) || (fromType == TYP_LONG) || varTypeIsFloating(fromType) || varTypeIsGC(fromType)); - assert(varTypeIsIntegral(toType)); + case OAK_GT: + printf(" > "); + break; - // CAST/CAST_OVF(small type <- float/double) - [TO_TYPE_MIN..TO_TYPE_MAX] - // CAST/CAST_OVF(uint/int <- float/double) - [INT_MIN..INT_MAX] - // CAST/CAST_OVF(ulong/long <- float/double) - [LONG_MIN..LONG_MAX] - if (varTypeIsFloating(fromType)) - { - if (!varTypeIsSmall(toType)) - { - toType = genActualType(toType); - } + case OAK_GT_UN: + printf(" u> "); + break; - return IntegralRange::ForType(toType); - } + case OAK_GE: + printf(" >= "); + break; - // Cast from a GC type is the same as a cast from TYP_I_IMPL for our purposes. - if (varTypeIsGC(fromType)) - { - fromType = TYP_I_IMPL; - } + case OAK_GE_UN: + printf(" u>= "); + break; - if (varTypeIsSmall(toType) || (genActualType(toType) == fromType)) - { - return ForCastInput(cast); - } + case OAK_SUBRANGE: + printf(" in "); + break; - // if we're upcasting and the cast op is a known non-negative - consider - // this cast unsigned - if (!fromUnsigned && (genTypeSize(toType) >= genTypeSize(fromType))) - { - fromUnsigned = cast->CastOp()->IsNeverNegative(compiler); + default: + unreached(); + break; } - // CAST(uint/int <- ulong/long) - [INT_MIN..INT_MAX] - // CAST(ulong/long <- uint) - [0..UINT_MAX] - // CAST(ulong/long <- int) - [INT_MIN..INT_MAX] - if (!cast->gtOverflow()) + switch (curAssertion.GetOp2().GetKind()) { - if ((fromType == TYP_INT) && fromUnsigned) - { - return {SymbolicIntegerValue::Zero, SymbolicIntegerValue::UIntMax}; - } + case O2K_LCLVAR_COPY: + printf("lclvar V%02u", curAssertion.GetOp2().GetLclNum()); + break; - return {SymbolicIntegerValue::IntMin, SymbolicIntegerValue::IntMax}; - } + case O2K_CONST_INT: + if (curAssertion.GetOp1().KindIs(O1K_EXACT_TYPE, O1K_SUBTYPE)) + { + ssize_t iconVal = curAssertion.GetOp2().GetIntConstant(); + if (IsAot()) + { + printf("MT(%p)", dspPtr(iconVal)); + } + else + { + printf("MT(%s)", eeGetClassName(reinterpret_cast(iconVal))); + } + } + else if (curAssertion.GetOp2().IsNullConstant()) + { + printf("null"); + } + else if (curAssertion.GetOp2().HasIconFlag()) + { + printf("[%p]", dspPtr(curAssertion.GetOp2().GetIntConstant())); + } + else + { + printf("%lld", (int64_t)curAssertion.GetOp2().GetIntConstant()); + } + break; - SymbolicIntegerValue lowerBound; - SymbolicIntegerValue upperBound; - switch (toType) - { - // CAST_OVF(uint <- ulong) - [INT_MIN..INT_MAX] - // CAST_OVF(uint <- long) - [INT_MIN..INT_MAX] - case TYP_UINT: - lowerBound = SymbolicIntegerValue::IntMin; - upperBound = SymbolicIntegerValue::IntMax; + case O2K_CONST_DOUBLE: + if (FloatingPointUtils::isNegativeZero(curAssertion.GetOp2().GetDoubleConstant())) + { + printf("-0.0"); + } + else + { + printf("%#lg", curAssertion.GetOp2().GetDoubleConstant()); + } break; - // CAST_OVF(int <- ulong) - [0..INT_MAX] - // CAST_OVF(int <- long) - [INT_MIN..INT_MAX] - case TYP_INT: - lowerBound = fromUnsigned ? SymbolicIntegerValue::Zero : SymbolicIntegerValue::IntMin; - upperBound = SymbolicIntegerValue::IntMax; + case O2K_ZEROOBJ: + printf("ZeroObj"); break; - // CAST_OVF(ulong <- uint) - [0..UINT_MAX] - // CAST_OVF(ulong <- int) - [0..INT_MAX] - case TYP_ULONG: - lowerBound = SymbolicIntegerValue::Zero; - upperBound = fromUnsigned ? SymbolicIntegerValue::UIntMax : SymbolicIntegerValue::IntMax; + case O2K_SUBRANGE: + IntegralRange::Print(curAssertion.GetOp2().GetIntegralRange()); break; - // CAST_OVF(long <- uint) - [0..UINT_MAX] - // CAST_OVF(long <- int) - [INT_MIN..INT_MAX] - case TYP_LONG: - lowerBound = fromUnsigned ? SymbolicIntegerValue::Zero : SymbolicIntegerValue::IntMin; - upperBound = fromUnsigned ? SymbolicIntegerValue::UIntMax : SymbolicIntegerValue::IntMax; + case O2K_CHECKED_BOUND_ADD_CNS: + printf("(Checked_Bnd_BinOp " FMT_VN " + %d)", curAssertion.GetOp2().GetCheckedBound(), + curAssertion.GetOp2().GetCheckedBoundConstant()); break; default: unreached(); + break; } - return {lowerBound, upperBound}; + printf("\n"); } -/* static */ IntegralRange IntegralRange::Union(IntegralRange range1, IntegralRange range2) +void Compiler::optPrintAssertionIndex(AssertionIndex index) { - return IntegralRange(min(range1.GetLowerBound(), range2.GetLowerBound()), - max(range1.GetUpperBound(), range2.GetUpperBound())); + if (index == NO_ASSERTION_INDEX) + { + printf("#NA"); + return; + } + + printf("#%02u", index); } -#ifdef DEBUG -/* static */ void IntegralRange::Print(IntegralRange range) +void Compiler::optPrintAssertionIndices(ASSERT_TP assertions) { - printf("[%lld", SymbolicToRealValue(range.m_lowerBound)); - printf(".."); - printf("%lld]", SymbolicToRealValue(range.m_upperBound)); + if (BitVecOps::IsEmpty(apTraits, assertions)) + { + optPrintAssertionIndex(NO_ASSERTION_INDEX); + return; + } + + BitVecOps::Iter iter(apTraits, assertions); + unsigned bitIndex = 0; + if (iter.NextElem(&bitIndex)) + { + optPrintAssertionIndex(static_cast(bitIndex + 1)); + while (iter.NextElem(&bitIndex)) + { + printf(" "); + optPrintAssertionIndex(static_cast(bitIndex + 1)); + } + } } #endif // DEBUG -//------------------------------------------------------------------------------ -// GetAssertionDep: Retrieve the assertions on this local variable -// -// Arguments: -// lclNum - The local var id. -// mustExist - If true, assert that the dependent assertions must exist. -// -// Return Value: -// The dependent assertions (assertions using the value of the local var) -// of the local var. -// - -ASSERT_TP& Compiler::GetAssertionDep(unsigned lclNum, bool mustExist) +/* static */ +void Compiler::optDumpAssertionIndices(const char* header, ASSERT_TP assertions, const char* footer /* = nullptr */) { - JitExpandArray& dep = *optAssertionDep; - if (dep[lclNum] == nullptr) +#ifdef DEBUG + Compiler* compiler = JitTls::GetCompiler(); + if (compiler->verbose) { - if (mustExist) + printf(header); + compiler->optPrintAssertionIndices(assertions); + if (footer != nullptr) { - assert(!"No dependent assertions for local var"); + printf(footer); } - dep[lclNum] = BitVecOps::MakeEmpty(apTraits); } - return dep[lclNum]; +#endif // DEBUG } -/***************************************************************************** - * - * Initialize the assertion prop bitset traits and the default bitsets. - */ - -void Compiler::optAssertionTraitsInit(AssertionIndex assertionCount) +/* static */ +void Compiler::optDumpAssertionIndices(ASSERT_TP assertions, const char* footer /* = nullptr */) { - apTraits = new (this, CMK_AssertionProp) BitVecTraits(assertionCount, this); - apFull = BitVecOps::MakeFull(apTraits); + optDumpAssertionIndices("", assertions, footer); } -/***************************************************************************** +/****************************************************************************** + * + * Helper to retrieve the "assertIndex" assertion. Note that assertIndex 0 + * is NO_ASSERTION_INDEX and "optAssertionCount" is the last valid index. * - * Initialize the assertion prop tracking logic. */ - -void Compiler::optAssertionInit(bool isLocalProp) +const Compiler::AssertionDsc& Compiler::optGetAssertion(AssertionIndex assertIndex) const { assert(NO_ASSERTION_INDEX == 0); - const unsigned maxTrackedLocals = (unsigned)JitConfig.JitMaxLocalsToTrack(); + assert(assertIndex != NO_ASSERTION_INDEX); + assert(assertIndex <= optAssertionCount); + const AssertionDsc& assertion = optAssertionTabPrivate[assertIndex - 1]; +#ifdef DEBUG + optDebugCheckAssertion(assertion); +#endif - // We initialize differently for local prop / global prop - // - if (isLocalProp) + return assertion; +} + +ValueNum Compiler::optConservativeNormalVN(GenTree* tree) +{ + if (optLocalAssertionProp) { - optLocalAssertionProp = true; - optCrossBlockLocalAssertionProp = true; + return ValueNumStore::NoVN; + } - // Disable via config - // - if (JitConfig.JitEnableCrossBlockLocalAssertionProp() == 0) - { - JITDUMP("Disabling cross-block assertion prop by config setting\n"); - optCrossBlockLocalAssertionProp = false; - } - -#ifdef DEBUG - // Disable per method via range - // - static ConfigMethodRange s_range; - s_range.EnsureInit(JitConfig.JitEnableCrossBlockLocalAssertionPropRange()); - if (!s_range.Contains(info.compMethodHash())) - { - JITDUMP("Disabling cross-block assertion prop by config range\n"); - optCrossBlockLocalAssertionProp = false; - } -#endif - - // Disable if too many locals - // - // The typical number of local assertions is roughly proportional - // to the number of locals. So when we have huge numbers of locals, - // just do within-block local assertion prop. - // - if (lvaCount > maxTrackedLocals) - { - JITDUMP("Disabling cross-block assertion prop: too many locals\n"); - optCrossBlockLocalAssertionProp = false; - } - - if (optCrossBlockLocalAssertionProp) - { - // We may need a fairly large table. Keep size a multiple of 64. - // Empirical studies show about 1.16 asserions/ tracked local. - // - if (lvaTrackedCount < 24) - { - optMaxAssertionCount = 64; - } - else if (lvaTrackedCount < 64) - { - optMaxAssertionCount = 128; - } - else - { - optMaxAssertionCount = (AssertionIndex)min(maxTrackedLocals, ((3 * lvaTrackedCount / 128) + 1) * 64); - } - - JITDUMP("Cross-block table size %u (for %u tracked locals)\n", optMaxAssertionCount, lvaTrackedCount); - } - else - { - // The assertion table will be reset for each block, so it can be smaller. - // - optMaxAssertionCount = 64; - } - - // Local assertion prop keeps mappings from each local var to the assertions about that var. - // - optAssertionDep = - new (this, CMK_AssertionProp) JitExpandArray(getAllocator(CMK_AssertionProp), max(1u, lvaCount)); - - if (optCrossBlockLocalAssertionProp) - { - optComplementaryAssertionMap = new (this, CMK_AssertionProp) - AssertionIndex[optMaxAssertionCount + 1](); // zero-inited (NO_ASSERTION_INDEX) - } - } - else - { - // General assertion prop. - // - optLocalAssertionProp = false; - optCrossBlockLocalAssertionProp = false; - - // Heuristic for sizing the assertion table. - // - // The weighting of basicBlocks vs locals reflects their relative contribution observed empirically. - // Validated against 1,115,046 compiled methods: - // - 94.6% of methods stay at the floor of 64 (only 1.9% actually need more). - // - Underpredicts for 481 methods (0.043%), with a worst-case deficit of 127. - // - Only 0.4% of methods hit the 256 cap. - optMaxAssertionCount = (AssertionIndex)max(64, min(256, (int)(lvaTrackedCount + 3 * fgBBcount + 48) >> 2)); - - optComplementaryAssertionMap = new (this, CMK_AssertionProp) - AssertionIndex[optMaxAssertionCount + 1](); // zero-inited (NO_ASSERTION_INDEX) - } - - optAssertionTabPrivate = new (this, CMK_AssertionProp) AssertionDsc[optMaxAssertionCount]; - optAssertionTraitsInit(optMaxAssertionCount); - - optAssertionCount = 0; - optAssertionOverflow = 0; - optAssertionPropagated = false; - bbJtrueAssertionOut = nullptr; - optCanPropLclVar = false; - optCanPropEqual = false; - optCanPropNonNull = false; - optCanPropBndsChk = false; - optCanPropSubRange = false; -} - -#ifdef DEBUG -void Compiler::optPrintAssertion(const AssertionDsc& curAssertion, AssertionIndex assertionIndex /* = 0 */) -{ - // Print assertion index if provided - if (assertionIndex > 0) - { - optPrintAssertionIndex(assertionIndex); - printf(" "); - } - - switch (curAssertion.GetOp1().GetKind()) - { - case O1K_LCLVAR: - if (optLocalAssertionProp) - { - printf("lclvar V%02u", curAssertion.GetOp1().GetLclNum()); - } - else - { - printf("lclvar " FMT_VN "", curAssertion.GetOp1().GetVN()); - } - break; - - case O1K_VN: - printf("VN " FMT_VN "", curAssertion.GetOp1().GetVN()); - break; - - case O1K_EXACT_TYPE: - printf("ExactType " FMT_VN "", curAssertion.GetOp1().GetVN()); - break; - - case O1K_SUBTYPE: - printf("SubType " FMT_VN "", curAssertion.GetOp1().GetVN()); - break; - - default: - unreached(); - break; - } - - switch (curAssertion.GetKind()) - { - case OAK_EQUAL: - printf(" == "); - break; - - case OAK_NOT_EQUAL: - printf(" != "); - break; - - case OAK_LT: - printf(" < "); - break; - - case OAK_LT_UN: - printf(" u< "); - break; - - case OAK_LE: - printf(" <= "); - break; - - case OAK_LE_UN: - printf(" u<= "); - break; - - case OAK_GT: - printf(" > "); - break; - - case OAK_GT_UN: - printf(" u> "); - break; - - case OAK_GE: - printf(" >= "); - break; - - case OAK_GE_UN: - printf(" u>= "); - break; - - case OAK_SUBRANGE: - printf(" in "); - break; - - default: - unreached(); - break; - } - - switch (curAssertion.GetOp2().GetKind()) - { - case O2K_LCLVAR_COPY: - printf("lclvar V%02u", curAssertion.GetOp2().GetLclNum()); - break; - - case O2K_CONST_INT: - if (curAssertion.GetOp1().KindIs(O1K_EXACT_TYPE, O1K_SUBTYPE)) - { - ssize_t iconVal = curAssertion.GetOp2().GetIntConstant(); - if (IsAot()) - { - printf("MT(%p)", dspPtr(iconVal)); - } - else - { - printf("MT(%s)", eeGetClassName(reinterpret_cast(iconVal))); - } - } - else if (curAssertion.GetOp2().IsNullConstant()) - { - printf("null"); - } - else if (curAssertion.GetOp2().HasIconFlag()) - { - printf("[%p]", dspPtr(curAssertion.GetOp2().GetIntConstant())); - } - else - { - printf("%lld", (int64_t)curAssertion.GetOp2().GetIntConstant()); - } - break; - - case O2K_CONST_DOUBLE: - if (FloatingPointUtils::isNegativeZero(curAssertion.GetOp2().GetDoubleConstant())) - { - printf("-0.0"); - } - else - { - printf("%#lg", curAssertion.GetOp2().GetDoubleConstant()); - } - break; - - case O2K_ZEROOBJ: - printf("ZeroObj"); - break; - - case O2K_SUBRANGE: - IntegralRange::Print(curAssertion.GetOp2().GetIntegralRange()); - break; - - case O2K_CHECKED_BOUND_ADD_CNS: - printf("(Checked_Bnd_BinOp " FMT_VN " + %d)", curAssertion.GetOp2().GetCheckedBound(), - curAssertion.GetOp2().GetCheckedBoundConstant()); - break; - - default: - unreached(); - break; - } - - printf("\n"); -} - -void Compiler::optPrintAssertionIndex(AssertionIndex index) -{ - if (index == NO_ASSERTION_INDEX) - { - printf("#NA"); - return; - } - - printf("#%02u", index); -} - -void Compiler::optPrintAssertionIndices(ASSERT_TP assertions) -{ - if (BitVecOps::IsEmpty(apTraits, assertions)) - { - optPrintAssertionIndex(NO_ASSERTION_INDEX); - return; - } - - BitVecOps::Iter iter(apTraits, assertions); - unsigned bitIndex = 0; - if (iter.NextElem(&bitIndex)) - { - optPrintAssertionIndex(static_cast(bitIndex + 1)); - while (iter.NextElem(&bitIndex)) - { - printf(" "); - optPrintAssertionIndex(static_cast(bitIndex + 1)); - } - } -} -#endif // DEBUG - -/* static */ -void Compiler::optDumpAssertionIndices(const char* header, ASSERT_TP assertions, const char* footer /* = nullptr */) -{ -#ifdef DEBUG - Compiler* compiler = JitTls::GetCompiler(); - if (compiler->verbose) - { - printf(header); - compiler->optPrintAssertionIndices(assertions); - if (footer != nullptr) - { - printf(footer); - } - } -#endif // DEBUG -} - -/* static */ -void Compiler::optDumpAssertionIndices(ASSERT_TP assertions, const char* footer /* = nullptr */) -{ - optDumpAssertionIndices("", assertions, footer); -} - -/****************************************************************************** - * - * Helper to retrieve the "assertIndex" assertion. Note that assertIndex 0 - * is NO_ASSERTION_INDEX and "optAssertionCount" is the last valid index. - * - */ -const Compiler::AssertionDsc& Compiler::optGetAssertion(AssertionIndex assertIndex) const -{ - assert(NO_ASSERTION_INDEX == 0); - assert(assertIndex != NO_ASSERTION_INDEX); - assert(assertIndex <= optAssertionCount); - const AssertionDsc& assertion = optAssertionTabPrivate[assertIndex - 1]; -#ifdef DEBUG - optDebugCheckAssertion(assertion); -#endif - - return assertion; -} - -ValueNum Compiler::optConservativeNormalVN(GenTree* tree) -{ - if (optLocalAssertionProp) - { - return ValueNumStore::NoVN; - } - - assert(vnStore != nullptr); - return vnStore->VNConservativeNormalValue(tree->gtVNPair); -} - -//------------------------------------------------------------------------ -// optCastConstantSmall: Cast a constant to a small type. -// -// Parameters: -// iconVal - the integer constant -// smallType - the small type to cast to -// -// Returns: -// The cast constant after sign/zero extension. -// -ssize_t Compiler::optCastConstantSmall(ssize_t iconVal, var_types smallType) -{ - switch (smallType) - { - case TYP_BYTE: - return int8_t(iconVal); - - case TYP_SHORT: - return int16_t(iconVal); - - case TYP_USHORT: - return uint16_t(iconVal); - - case TYP_UBYTE: - return uint8_t(iconVal); - - default: - assert(!"Unexpected type to truncate to"); - return iconVal; - } -} - -//------------------------------------------------------------------------ -// optCreateAssertion: Create an (op1 assertionKind op2) assertion. -// -// Arguments: -// op1 - the first assertion operand -// op2 - the second assertion operand -// equals - the assertion kind (equals / not equals) - -// Return Value: -// The new assertion index or NO_ASSERTION_INDEX if a new assertion -// was not created. -// -// Notes: -// Assertion creation may fail either because the provided assertion -// operands aren't supported or because the assertion table is full. -// -AssertionIndex Compiler::optCreateAssertion(GenTree* op1, GenTree* op2, bool equals) -{ - assert(op1 != nullptr); - - if (op2 == nullptr) - { - // Must be an OAK_NOT_EQUAL assertion - assert(!equals); - - // Set op1 to the instance pointer of the indirection - op1 = op1->gtEffectiveVal(); - - // TODO-Cleanup: Replace with gtPeelOffset with proper fgBigOffset check - // It will produce a few regressions. - ssize_t offset = 0; - while (op1->OperIs(GT_ADD) && op1->TypeIs(TYP_BYREF)) - { - if (op1->gtGetOp2()->IsCnsIntOrI()) - { - offset += op1->gtGetOp2()->AsIntCon()->gtIconVal; - op1 = op1->gtGetOp1()->gtEffectiveVal(); - } - else if (op1->gtGetOp1()->IsCnsIntOrI()) - { - offset += op1->gtGetOp1()->AsIntCon()->gtIconVal; - op1 = op1->gtGetOp2()->gtEffectiveVal(); - } - else - { - break; - } - } - - if (!fgIsBigOffset(offset) && op1->OperIs(GT_LCL_VAR) && !lvaVarAddrExposed(op1->AsLclVar()->GetLclNum())) - { - if (optLocalAssertionProp) - { - AssertionDsc assertion = AssertionDsc::CreateLclNonNullAssertion(this, op1->AsLclVar()->GetLclNum()); - return optAddAssertion(assertion); - } - - ValueNum op1VN = optConservativeNormalVN(op1); - if (op1VN == ValueNumStore::NoVN) - { - return NO_ASSERTION_INDEX; - } - AssertionDsc assertion = AssertionDsc::CreateVNNonNullAssertion(this, op1VN); - return optAddAssertion(assertion); - } - } - // - // Are we making an assertion about a local variable? - // - else if (op1->OperIsScalarLocal()) - { - unsigned const lclNum = op1->AsLclVarCommon()->GetLclNum(); - LclVarDsc* const lclVar = lvaGetDesc(lclNum); - - // If the local variable has its address exposed then bail - // - if (lclVar->IsAddressExposed()) - { - return NO_ASSERTION_INDEX; - } - - /* Skip over a GT_COMMA node(s), if necessary */ - while (op2->OperIs(GT_COMMA)) - { - op2 = op2->AsOp()->gtOp2; - } - - switch (op2->OperGet()) - { - // - // Constant Assertions - // - case GT_CNS_DBL: - { - double dblCns = op2->AsDblCon()->DconValue(); - if (FloatingPointUtils::isNaN(dblCns)) - { - return NO_ASSERTION_INDEX; - } - - ValueNum op1VN = optConservativeNormalVN(op1); - ValueNum op2VN = optConservativeNormalVN(op2); - if (!optLocalAssertionProp && (op1VN == ValueNumStore::NoVN || op2VN == ValueNumStore::NoVN)) - { - // GlobalAP requires valid VNs. - return NO_ASSERTION_INDEX; - } - - AssertionDsc dsc = AssertionDsc::CreateConstLclVarAssertion(this, lclNum, op1VN, dblCns, op2VN, equals); - return optAddAssertion(dsc); - } - - case GT_CNS_INT: - { - ValueNum op1VN = optConservativeNormalVN(op1); - ValueNum op2VN = optConservativeNormalVN(op2); - if (!optLocalAssertionProp && (op1VN == ValueNumStore::NoVN || op2VN == ValueNumStore::NoVN)) - { - return NO_ASSERTION_INDEX; - } - - ssize_t iconVal = op2->AsIntCon()->IconValue(); - if (op1->TypeIs(TYP_STRUCT)) - { - assert(iconVal == 0); - AssertionDsc dsc = - AssertionDsc::CreateConstLclVarAssertion(this, lclNum, op1VN, O2K_ZEROOBJ, op2VN, equals); - return optAddAssertion(dsc); - } - - if (varTypeIsSmall(lclVar)) - { - ssize_t truncatedIconVal = optCastConstantSmall(iconVal, lclVar->TypeGet()); - if (!op1->OperIs(GT_STORE_LCL_VAR) && (truncatedIconVal != iconVal)) - { - // This assertion would be saying that a small local is equal to a value - // outside its range. It means this block is unreachable. Avoid creating - // such impossible assertions which can hit assertions in other places. - return NO_ASSERTION_INDEX; - } - - iconVal = truncatedIconVal; - if (!optLocalAssertionProp) - { - op2VN = vnStore->VNForIntCon(static_cast(iconVal)); - } - } - - AssertionDsc dsc = - AssertionDsc::CreateConstLclVarAssertion(this, lclNum, op1VN, iconVal, op2VN, equals, - op2->GetIconHandleFlag(), op2->AsIntCon()->gtFieldSeq); - return optAddAssertion(dsc); - } - - case GT_LCL_VAR: - { - if (!optLocalAssertionProp) - { - // O2K_LCLVAR_COPY is local assertion prop only - return NO_ASSERTION_INDEX; - } - - unsigned lclNum2 = op2->AsLclVarCommon()->GetLclNum(); - LclVarDsc* lclVar2 = lvaGetDesc(lclNum2); - - // If the two locals are the same then bail - if (lclNum == lclNum2) - { - return NO_ASSERTION_INDEX; - } - - // If the types are different then bail */ - if (lclVar->lvType != lclVar2->lvType) - { - return NO_ASSERTION_INDEX; - } - - // If we're making a copy of a "normalize on load" lclvar then the destination - // has to be "normalize on load" as well, otherwise we risk skipping normalization. - if (lclVar2->lvNormalizeOnLoad() && !lclVar->lvNormalizeOnLoad()) - { - return NO_ASSERTION_INDEX; - } - - // If the local variable has its address exposed then bail - if (lclVar2->IsAddressExposed()) - { - return NO_ASSERTION_INDEX; - } - - // We process locals when we see the LCL_VAR node instead - // of at its actual use point (its parent). That opens us - // up to problems in a case like the following, assuming we - // allowed creating an assertion like V10 = V35: - // - // └──▌ ADD int - // ├──▌ LCL_VAR int V10 tmp6 -> copy propagated to [V35 tmp31] - // └──▌ COMMA int - // ├──▌ STORE_LCL_VAR int V35 tmp31 - // │ └──▌ LCL_FLD int V03 loc1 [+4] - if (lclVar2->lvRedefinedInEmbeddedStatement) - { - return NO_ASSERTION_INDEX; - } - - // Ok everything has been set and the assertion looks good - AssertionDsc assertion = AssertionDsc::CreateLclvarCopy(this, lclNum, lclNum2, equals); - return optAddAssertion(assertion); - } - - case GT_CALL: - { - if (optLocalAssertionProp) - { - GenTreeCall* const call = op2->AsCall(); - if (call->IsHelperCall() && s_helperCallProperties.NonNullReturn(call->GetHelperNum())) - { - AssertionDsc assertion = AssertionDsc::CreateLclNonNullAssertion(this, lclNum); - return optAddAssertion(assertion); - } - } - break; - } - - default: - break; - } - - // Try and see if we can make a subrange assertion. - if (optLocalAssertionProp && equals && varTypeIsIntegral(op2)) - { - IntegralRange nodeRange = IntegralRange::ForNode(op2, this); - IntegralRange typeRange = IntegralRange::ForType(genActualType(op2)); - assert(typeRange.Contains(nodeRange)); - - if (!typeRange.Equals(nodeRange)) - { - AssertionDsc assertion = AssertionDsc::CreateSubrange(this, lclNum, nodeRange); - return optAddAssertion(assertion); - } - } - } - else - { - // Currently, O1K_VN serves as a backup for O1K_LCLVAR (where it's not a local), - // but long term we should keep O1K_LCLVAR for local assertions only. - if (!optLocalAssertionProp) - { - ValueNum op1VN = optConservativeNormalVN(op1); - ValueNum op2VN = optConservativeNormalVN(op2); - - // For TP reasons, limited to 32-bit constants on the op2 side. - if (op1VN != ValueNumStore::NoVN && op2VN != ValueNumStore::NoVN && vnStore->IsVNInt32Constant(op2VN) && - !vnStore->IsVNHandle(op2VN)) - { - AssertionDsc assertion = AssertionDsc::CreateInt32ConstantVNAssertion(this, op1VN, op2VN, equals); - return optAddAssertion(assertion); - } - } - } - return NO_ASSERTION_INDEX; -} - -/***************************************************************************** - * - * If tree is a constant node holding an integral value, retrieve the value in - * pConstant. If the method returns true, pConstant holds the appropriate - * constant. Set "vnBased" to true to indicate local or global assertion prop. - * "pFlags" indicates if the constant is a handle marked by GTF_ICON_HDL_MASK. - */ -bool Compiler::optIsTreeKnownIntValue(bool vnBased, GenTree* tree, ssize_t* pConstant, GenTreeFlags* pFlags) -{ - // Is Local assertion prop? - if (!vnBased) - { - if (tree->OperIs(GT_CNS_INT)) - { - *pConstant = tree->AsIntCon()->IconValue(); - *pFlags = tree->GetIconHandleFlag(); - return true; - } - return false; - } - - // Global assertion prop - ValueNum vn = vnStore->VNConservativeNormalValue(tree->gtVNPair); - if (!vnStore->IsVNConstant(vn)) - { - return false; - } - - // ValueNumber 'vn' indicates that this node evaluates to a constant - - var_types vnType = vnStore->TypeOfVN(vn); - if (vnType == TYP_INT) - { - *pConstant = vnStore->ConstantValue(vn); - *pFlags = vnStore->IsVNHandle(vn) ? vnStore->GetHandleFlags(vn) : GTF_EMPTY; - return true; - } -#ifdef TARGET_64BIT - else if (vnType == TYP_LONG) - { - *pConstant = vnStore->ConstantValue(vn); - *pFlags = vnStore->IsVNHandle(vn) ? vnStore->GetHandleFlags(vn) : GTF_EMPTY; - return true; - } -#endif - - return false; -} - -/***************************************************************************** - * - * Given an assertion add it to the assertion table - * - * If it is already in the assertion table return the assertionIndex that - * we use to refer to this element. - * Otherwise add it to the assertion table and return the assertionIndex that - * we use to refer to this element. - * If we need to add to the table and the table is full return the value zero - */ -AssertionIndex Compiler::optAddAssertion(const AssertionDsc& newAssertion) -{ - bool canAddNewAssertions = optAssertionCount < optMaxAssertionCount; - - // See if we already have this assertion in the table. - // - // For local assertion prop we can speed things up by checking the dep vector. - // Note we only need check the op1 vector; copies get indexed on both op1 - // and op2, so searching the first will find any existing match. - // - if (optLocalAssertionProp) - { - assert(newAssertion.GetOp1().KindIs(O1K_LCLVAR)); - - unsigned lclNum = newAssertion.GetOp1().GetLclNum(); - BitVecOps::Iter iter(apTraits, GetAssertionDep(lclNum)); - unsigned bvIndex = 0; - while (iter.NextElem(&bvIndex)) - { - AssertionIndex const index = GetAssertionIndex(bvIndex); - const AssertionDsc& curAssertion = optGetAssertion(index); - - if (curAssertion.Equals(newAssertion, /* vnBased */ false)) - { - return index; - } - } - } - else - { - bool mayHaveDuplicates = - optAssertionHasAssertionsForVN(newAssertion.GetOp1().GetVN(), /* addIfNotFound */ canAddNewAssertions); - // We need to register op2.vn too, even if we know for sure there are no duplicates - if (newAssertion.GetOp2().KindIs(O2K_CHECKED_BOUND_ADD_CNS)) - { - mayHaveDuplicates |= optAssertionHasAssertionsForVN(newAssertion.GetOp2().GetCheckedBound(), - /* addIfNotFound */ canAddNewAssertions); - - // Additionally, check for the pattern of "VN + const == checkedBndVN" and register "VN" as well. - ValueNum addOpVN; - if (vnStore->IsVNBinFuncWithConst(newAssertion.GetOp1().GetVN(), VNF_ADD, &addOpVN, nullptr)) - { - mayHaveDuplicates |= optAssertionHasAssertionsForVN(addOpVN, /* addIfNotFound */ canAddNewAssertions); - } - } - - if (mayHaveDuplicates) - { - // For global prop we search the entire table. - // - // Check if exists already, so we can skip adding new one. Search backwards. - for (AssertionIndex index = optAssertionCount; index >= 1; index--) - { - const AssertionDsc& curAssertion = optGetAssertion(index); - if (curAssertion.Equals(newAssertion, /* vnBased */ true)) - { - return index; - } - } - } - } - - // Check if we are within max count. - if (!canAddNewAssertions) - { - optAssertionOverflow++; - return NO_ASSERTION_INDEX; - } - - optAssertionTabPrivate[optAssertionCount] = newAssertion; - optAssertionCount++; - -#ifdef DEBUG - if (verbose) - { - printf("GenTreeNode creates assertion:\n"); - gtDispTree(optAssertionPropCurrentTree, nullptr, nullptr, true); - printf(optLocalAssertionProp ? "In " FMT_BB " New Local " : "In " FMT_BB " New Global ", compCurBB->bbNum); - optPrintAssertion(newAssertion, optAssertionCount); - } -#endif // DEBUG - - // Track the short-circuit criteria - optCanPropLclVar |= newAssertion.CanPropLclVar(); - optCanPropEqual |= newAssertion.CanPropEqualOrNotEqual(); - optCanPropNonNull |= newAssertion.CanPropNonNull(); - optCanPropSubRange |= newAssertion.CanPropSubRange(); - optCanPropBndsChk |= newAssertion.CanPropBndsCheck(); - - // Assertion mask bits are [index + 1]. - if (optLocalAssertionProp) - { - assert(newAssertion.GetOp1().KindIs(O1K_LCLVAR)); - - // Mark the variables this index depends on - unsigned lclNum = newAssertion.GetOp1().GetLclNum(); - BitVecOps::AddElemD(apTraits, GetAssertionDep(lclNum), optAssertionCount - 1); - if (newAssertion.GetOp2().KindIs(O2K_LCLVAR_COPY)) - { - lclNum = newAssertion.GetOp2().GetLclNum(); - BitVecOps::AddElemD(apTraits, GetAssertionDep(lclNum), optAssertionCount - 1); - } - } - -#ifdef DEBUG - optDebugCheckAssertions(optAssertionCount); -#endif - return optAssertionCount; -} - -//------------------------------------------------------------------------ -// optAssertionHasAssertionsForVN: Check if we already have assertions for the given VN. -// If "addIfNotFound" is true, add the VN to the map if it's not already there. -// -// Arguments: -// vn - the VN to check for -// addIfNotFound - whether to add the VN to the map if it's not found -// -// Return Value: -// true if we already have assertions for the given VN, false otherwise. -// -bool Compiler::optAssertionHasAssertionsForVN(ValueNum vn, bool addIfNotFound) -{ - assert(!optLocalAssertionProp); - if (vn == ValueNumStore::NoVN) - { - assert(!addIfNotFound); - return false; - } - - if (addIfNotFound) - { - // Lazy initialize the map when we first need to add to it - if (optAssertionVNsMap == nullptr) - { - optAssertionVNsMap = new (this, CMK_AssertionProp) VNSet(getAllocator(CMK_AssertionProp)); - } - - // Avoid double lookup by using the return value of LookupPointerOrAdd to - // determine whether the VN was already in the map. - bool* pValue = optAssertionVNsMap->LookupPointerOrAdd(vn, false); - if (!*pValue) - { - *pValue = true; - return false; - } - return true; - } - - // Otherwise just do a normal lookup - return (optAssertionVNsMap != nullptr) && optAssertionVNsMap->Lookup(vn); -} - -#ifdef DEBUG -void Compiler::optDebugCheckAssertion(const AssertionDsc& assertion) const -{ - switch (assertion.GetOp1().GetKind()) - { - case O1K_EXACT_TYPE: - case O1K_SUBTYPE: - case O1K_VN: - assert(!optLocalAssertionProp); - break; - default: - break; - } - - switch (assertion.GetOp2().GetKind()) - { - case O2K_SUBRANGE: - case O2K_LCLVAR_COPY: - assert(optLocalAssertionProp); - break; - - case O2K_ZEROOBJ: - // We only make these assertion for stores (not control flow). - assert(assertion.KindIs(OAK_EQUAL)); - // We use "optLocalAssertionIsEqualOrNotEqual" to find these. - break; - - case O2K_CONST_DOUBLE: - assert(!FloatingPointUtils::isNaN(assertion.GetOp2().GetDoubleConstant())); - break; - - default: - // for all other 'assertion.GetOp2().GetKind()' values we don't check anything - break; - } -} - -/***************************************************************************** - * - * Verify that assertion prop related assumptions are valid. If "index" - * is 0 (i.e., NO_ASSERTION_INDEX) then verify all assertions in the table. - * If "index" is between 1 and optAssertionCount, then verify the assertion - * desc corresponding to "index." - */ -void Compiler::optDebugCheckAssertions(AssertionIndex index) -{ - AssertionIndex start = (index == NO_ASSERTION_INDEX) ? 1 : index; - AssertionIndex end = (index == NO_ASSERTION_INDEX) ? optAssertionCount : index; - for (AssertionIndex ind = start; ind <= end; ++ind) - { - const AssertionDsc& assertion = optGetAssertion(ind); - optDebugCheckAssertion(assertion); - } -} -#endif + assert(vnStore != nullptr); + return vnStore->VNConservativeNormalValue(tree->gtVNPair); +} //------------------------------------------------------------------------ -// optCreateComplementaryAssertion: Create an assertion that is the complementary -// of the specified assertion. -// -// Arguments: -// assertionIndex - the index of the assertion -// -// Notes: -// The created complementary assertion is associated with the original -// assertion such that it can be found by optFindComplementary. +// optCastConstantSmall: Cast a constant to a small type. // -void Compiler::optCreateComplementaryAssertion(AssertionIndex assertionIndex) -{ - if (assertionIndex == NO_ASSERTION_INDEX) - { - return; - } - - const AssertionDsc& candidateAssertion = optGetAssertion(assertionIndex); - if (candidateAssertion.KindIs(OAK_EQUAL)) - { - // Don't create useless OAK_NOT_EQUAL assertions - - if (candidateAssertion.GetOp1().KindIs(O1K_LCLVAR, O1K_VN)) - { - // "LCLVAR != CNS" is not a useful assertion (unless CNS is 0/1) - if (candidateAssertion.GetOp2().KindIs(O2K_CONST_INT) && - (candidateAssertion.GetOp2().GetIntConstant() != 0) && - (candidateAssertion.GetOp2().GetIntConstant() != 1)) - { - return; - } - - // "LCLVAR != LCLVAR_COPY" - if (candidateAssertion.GetOp2().KindIs(O2K_LCLVAR_COPY)) - { - return; - } - } - - // "Object is not Class" is also not a useful assertion (at least for now) - if (candidateAssertion.GetOp1().KindIs(O1K_EXACT_TYPE, O1K_SUBTYPE)) - { - return; - } - AssertionDsc reversed = candidateAssertion.Reverse(); - optMapComplementary(optAddAssertion(reversed), assertionIndex); - } - else if (candidateAssertion.KindIs(OAK_LT_UN, OAK_LE_UN) && - candidateAssertion.GetOp2().KindIs(O2K_CHECKED_BOUND_ADD_CNS)) - { - // Assertions such as "X > checkedBndVN" aren't very useful. - return; - } - else if (AssertionDsc::IsReversible(candidateAssertion.GetKind())) +// Parameters: +// iconVal - the integer constant +// smallType - the small type to cast to +// +// Returns: +// The cast constant after sign/zero extension. +// +ssize_t Compiler::optCastConstantSmall(ssize_t iconVal, var_types smallType) +{ + switch (smallType) { - AssertionDsc reversed = candidateAssertion.Reverse(); - optMapComplementary(optAddAssertion(reversed), assertionIndex); + case TYP_BYTE: + return int8_t(iconVal); + + case TYP_SHORT: + return int16_t(iconVal); + + case TYP_USHORT: + return uint16_t(iconVal); + + case TYP_UBYTE: + return uint8_t(iconVal); + + default: + assert(!"Unexpected type to truncate to"); + return iconVal; } } //------------------------------------------------------------------------ -// optCreateJtrueAssertions: Create assertions about a JTRUE's relop operands. +// optCreateAssertion: Create an (op1 assertionKind op2) assertion. // // Arguments: // op1 - the first assertion operand // op2 - the second assertion operand // equals - the assertion kind (equals / not equals) -// + // Return Value: // The new assertion index or NO_ASSERTION_INDEX if a new assertion // was not created. @@ -1633,1341 +463,1208 @@ void Compiler::optCreateComplementaryAssertion(AssertionIndex assertionIndex) // Notes: // Assertion creation may fail either because the provided assertion // operands aren't supported or because the assertion table is full. -// If an assertion is created successfully then an attempt is made to also -// create a second, complementary assertion. This may too fail, for the -// same reasons as the first one. // -AssertionIndex Compiler::optCreateJtrueAssertions(GenTree* op1, GenTree* op2, bool equals) +AssertionIndex Compiler::optCreateAssertion(GenTree* op1, GenTree* op2, bool equals) { - AssertionIndex assertionIndex = optCreateAssertion(op1, op2, equals); - // Don't bother if we don't have an assertion on the JTrue False path. Current implementation - // allows for a complementary only if there is an assertion on the False path (tree->HasAssertion()). - if (assertionIndex != NO_ASSERTION_INDEX) - { - optCreateComplementaryAssertion(assertionIndex); - } - return assertionIndex; -} + assert(op1 != nullptr); -AssertionInfo Compiler::optCreateJTrueBoundsAssertion(GenTree* tree) -{ - // These assertions are VN based, so not relevant for local prop - // - if (optLocalAssertionProp) + if (op2 == nullptr) { - return NO_ASSERTION_INDEX; - } + // Must be an OAK_NOT_EQUAL assertion + assert(!equals); - GenTree* relop = tree->gtGetOp1(); - if (!relop->OperIsCompare()) - { - return NO_ASSERTION_INDEX; - } + // Set op1 to the instance pointer of the indirection + op1 = op1->gtEffectiveVal(); - ValueNum relopVN = optConservativeNormalVN(relop); - VNFuncApp relopFuncApp; - if (!vnStore->GetVNFunc(relopVN, &relopFuncApp)) - { - // We're expecting a relop here - return NO_ASSERTION_INDEX; - } + // TODO-Cleanup: Replace with gtPeelOffset with proper fgBigOffset check + // It will produce a few regressions. + ssize_t offset = 0; + while (op1->OperIs(GT_ADD) && op1->TypeIs(TYP_BYREF)) + { + if (op1->gtGetOp2()->IsCnsIntOrI()) + { + offset += op1->gtGetOp2()->AsIntCon()->gtIconVal; + op1 = op1->gtGetOp1()->gtEffectiveVal(); + } + else if (op1->gtGetOp1()->IsCnsIntOrI()) + { + offset += op1->gtGetOp1()->AsIntCon()->gtIconVal; + op1 = op1->gtGetOp2()->gtEffectiveVal(); + } + else + { + break; + } + } - bool isUnsignedRelop; - if (relopFuncApp.FuncIs(VNF_LE, VNF_LT, VNF_GE, VNF_GT)) - { - isUnsignedRelop = false; - } - else if (relopFuncApp.FuncIs(VNF_LE_UN, VNF_LT_UN, VNF_GE_UN, VNF_GT_UN)) - { - isUnsignedRelop = true; + if (!fgIsBigOffset(offset) && op1->OperIs(GT_LCL_VAR) && !lvaVarAddrExposed(op1->AsLclVar()->GetLclNum())) + { + if (optLocalAssertionProp) + { + AssertionDsc assertion = AssertionDsc::CreateLclNonNullAssertion(this, op1->AsLclVar()->GetLclNum()); + return optAddAssertion(assertion); + } + + ValueNum op1VN = optConservativeNormalVN(op1); + if (op1VN == ValueNumStore::NoVN) + { + return NO_ASSERTION_INDEX; + } + AssertionDsc assertion = AssertionDsc::CreateVNNonNullAssertion(this, op1VN); + return optAddAssertion(assertion); + } } - else + // + // Are we making an assertion about a local variable? + // + else if (op1->OperIsScalarLocal()) { - // Not a relop we're interested in. - // Assertions for NE/EQ are handled elsewhere. - return NO_ASSERTION_INDEX; - } + unsigned const lclNum = op1->AsLclVarCommon()->GetLclNum(); + LclVarDsc* const lclVar = lvaGetDesc(lclNum); - VNFunc relopFunc = relopFuncApp.m_func; - ValueNum op1VN = relopFuncApp.m_args[0]; - ValueNum op2VN = relopFuncApp.m_args[1]; + // If the local variable has its address exposed then bail + // + if (lclVar->IsAddressExposed()) + { + return NO_ASSERTION_INDEX; + } - if ((genActualType(vnStore->TypeOfVN(op1VN)) != TYP_INT) || (genActualType(vnStore->TypeOfVN(op2VN)) != TYP_INT)) - { - // For now, we don't have consumers for assertions derived from non-int32 comparisons - return NO_ASSERTION_INDEX; - } + /* Skip over a GT_COMMA node(s), if necessary */ + while (op2->OperIs(GT_COMMA)) + { + op2 = op2->AsOp()->gtOp2; + } - // "CheckedBnd X" - if (!isUnsignedRelop && vnStore->IsVNCheckedBound(op1VN)) - { - // Move the checked bound to the right side for simplicity - relopFunc = ValueNumStore::SwapRelop(relopFunc); - AssertionDsc dsc = AssertionDsc::CreateCompareCheckedBound(relopFunc, op2VN, op1VN, 0); - AssertionIndex idx = optAddAssertion(dsc); - optCreateComplementaryAssertion(idx); - return idx; - } + switch (op2->OperGet()) + { + // + // Constant Assertions + // + case GT_CNS_DBL: + { + double dblCns = op2->AsDblCon()->DconValue(); + if (FloatingPointUtils::isNaN(dblCns)) + { + return NO_ASSERTION_INDEX; + } - // "X CheckedBnd" - if (!isUnsignedRelop && vnStore->IsVNCheckedBound(op2VN)) - { - AssertionDsc dsc = AssertionDsc::CreateCompareCheckedBound(relopFunc, op1VN, op2VN, 0); - AssertionIndex idx = optAddAssertion(dsc); - optCreateComplementaryAssertion(idx); - return idx; - } + ValueNum op1VN = optConservativeNormalVN(op1); + ValueNum op2VN = optConservativeNormalVN(op2); + if (!optLocalAssertionProp && (op1VN == ValueNumStore::NoVN || op2VN == ValueNumStore::NoVN)) + { + // GlobalAP requires valid VNs. + return NO_ASSERTION_INDEX; + } + + AssertionDsc dsc = AssertionDsc::CreateConstLclVarAssertion(this, lclNum, op1VN, dblCns, op2VN, equals); + return optAddAssertion(dsc); + } + + case GT_CNS_INT: + { + ValueNum op1VN = optConservativeNormalVN(op1); + ValueNum op2VN = optConservativeNormalVN(op2); + if (!optLocalAssertionProp && (op1VN == ValueNumStore::NoVN || op2VN == ValueNumStore::NoVN)) + { + return NO_ASSERTION_INDEX; + } + + ssize_t iconVal = op2->AsIntCon()->IconValue(); + if (op1->TypeIs(TYP_STRUCT)) + { + assert(iconVal == 0); + AssertionDsc dsc = + AssertionDsc::CreateConstLclVarAssertion(this, lclNum, op1VN, O2K_ZEROOBJ, op2VN, equals); + return optAddAssertion(dsc); + } + + if (varTypeIsSmall(lclVar)) + { + ssize_t truncatedIconVal = optCastConstantSmall(iconVal, lclVar->TypeGet()); + if (!op1->OperIs(GT_STORE_LCL_VAR) && (truncatedIconVal != iconVal)) + { + // This assertion would be saying that a small local is equal to a value + // outside its range. It means this block is unreachable. Avoid creating + // such impossible assertions which can hit assertions in other places. + return NO_ASSERTION_INDEX; + } + + iconVal = truncatedIconVal; + if (!optLocalAssertionProp) + { + op2VN = vnStore->VNForIntCon(static_cast(iconVal)); + } + } + + AssertionDsc dsc = + AssertionDsc::CreateConstLclVarAssertion(this, lclNum, op1VN, iconVal, op2VN, equals, + op2->GetIconHandleFlag(), op2->AsIntCon()->gtFieldSeq); + return optAddAssertion(dsc); + } + + case GT_LCL_VAR: + { + if (!optLocalAssertionProp) + { + // O2K_LCLVAR_COPY is local assertion prop only + return NO_ASSERTION_INDEX; + } + + unsigned lclNum2 = op2->AsLclVarCommon()->GetLclNum(); + LclVarDsc* lclVar2 = lvaGetDesc(lclNum2); + + // If the two locals are the same then bail + if (lclNum == lclNum2) + { + return NO_ASSERTION_INDEX; + } + + // If the types are different then bail */ + if (lclVar->lvType != lclVar2->lvType) + { + return NO_ASSERTION_INDEX; + } + + // If we're making a copy of a "normalize on load" lclvar then the destination + // has to be "normalize on load" as well, otherwise we risk skipping normalization. + if (lclVar2->lvNormalizeOnLoad() && !lclVar->lvNormalizeOnLoad()) + { + return NO_ASSERTION_INDEX; + } - // "(CheckedBnd + CNS) X" - ValueNum checkedBnd; - int checkedBndCns; - if (!isUnsignedRelop && vnStore->IsVNCheckedBoundAddConst(op1VN, &checkedBnd, &checkedBndCns)) - { - // Move the (CheckedBnd + CNS) part to the right side for simplicity - relopFunc = ValueNumStore::SwapRelop(relopFunc); - AssertionDsc dsc = AssertionDsc::CreateCompareCheckedBound(relopFunc, op2VN, checkedBnd, checkedBndCns); - AssertionIndex idx = optAddAssertion(dsc); - optCreateComplementaryAssertion(idx); - return idx; - } + // If the local variable has its address exposed then bail + if (lclVar2->IsAddressExposed()) + { + return NO_ASSERTION_INDEX; + } - // "X (CheckedBnd + CNS)" - if (!isUnsignedRelop && vnStore->IsVNCheckedBoundAddConst(op2VN, &checkedBnd, &checkedBndCns)) - { - AssertionDsc dsc = AssertionDsc::CreateCompareCheckedBound(relopFunc, op1VN, checkedBnd, checkedBndCns); - AssertionIndex idx = optAddAssertion(dsc); - optCreateComplementaryAssertion(idx); - return idx; - } + // We process locals when we see the LCL_VAR node instead + // of at its actual use point (its parent). That opens us + // up to problems in a case like the following, assuming we + // allowed creating an assertion like V10 = V35: + // + // └──▌ ADD int + // ├──▌ LCL_VAR int V10 tmp6 -> copy propagated to [V35 tmp31] + // └──▌ COMMA int + // ├──▌ STORE_LCL_VAR int V35 tmp31 + // │ └──▌ LCL_FLD int V03 loc1 [+4] + if (lclVar2->lvRedefinedInEmbeddedStatement) + { + return NO_ASSERTION_INDEX; + } - // Loop condition like "(uint)i < (uint)bnd" or equivalent - // Assertion: "no throw" since this condition guarantees that i is both >= 0 and < bnd (on the appropriate edge) - ValueNumStore::UnsignedCompareCheckedBoundInfo unsignedCompareBnd; - if (vnStore->IsVNUnsignedCompareCheckedBound(relopVN, &unsignedCompareBnd)) - { - ValueNum idxVN = vnStore->VNNormalValue(unsignedCompareBnd.vnIdx); - ValueNum lenVN = vnStore->VNNormalValue(unsignedCompareBnd.vnBound); + // Ok everything has been set and the assertion looks good + AssertionDsc assertion = AssertionDsc::CreateLclvarCopy(this, lclNum, lclNum2, equals); + return optAddAssertion(assertion); + } - AssertionDsc dsc = AssertionDsc::CreateNoThrowArrBnd(idxVN, lenVN); - AssertionIndex index = optAddAssertion(dsc); - if (unsignedCompareBnd.cmpOper == VNF_GE_UN) + case GT_CALL: + { + if (optLocalAssertionProp) + { + GenTreeCall* const call = op2->AsCall(); + if (call->IsHelperCall() && s_helperCallProperties.NonNullReturn(call->GetHelperNum())) + { + AssertionDsc assertion = AssertionDsc::CreateLclNonNullAssertion(this, lclNum); + return optAddAssertion(assertion); + } + } + break; + } + + default: + break; + } + + // Try and see if we can make a subrange assertion. + if (optLocalAssertionProp && equals && varTypeIsIntegral(op2)) { - // By default JTRUE generated assertions hold on the "jump" edge. We have i >= bnd but we're really - // after i < bnd so we need to change the assertion edge to "next". - return AssertionInfo::ForNextEdge(index); + IntegralRange nodeRange = IntegralRange::ForNode(op2, this); + IntegralRange typeRange = IntegralRange::ForType(genActualType(op2)); + assert(typeRange.Contains(nodeRange)); + + if (!typeRange.Equals(nodeRange)) + { + AssertionDsc assertion = AssertionDsc::CreateSubrange(this, lclNum, nodeRange); + return optAddAssertion(assertion); + } } - return index; } - - // Create "X relop CNS" assertion (both signed and unsigned relops) - // Ignore non-positive constants for unsigned relops as they don't add any useful information. - ssize_t cns; - if (vnStore->IsVNIntegralConstant(op1VN, &cns) && (!isUnsignedRelop || (cns > 0))) + else { - relopFunc = ValueNumStore::SwapRelop(relopFunc); - AssertionDsc dsc = AssertionDsc::CreateConstantBound(this, relopFunc, op2VN, op1VN); - AssertionIndex idx = optAddAssertion(dsc); - optCreateComplementaryAssertion(idx); - return idx; - } + // Currently, O1K_VN serves as a backup for O1K_LCLVAR (where it's not a local), + // but long term we should keep O1K_LCLVAR for local assertions only. + if (!optLocalAssertionProp) + { + ValueNum op1VN = optConservativeNormalVN(op1); + ValueNum op2VN = optConservativeNormalVN(op2); - if (vnStore->IsVNIntegralConstant(op2VN, &cns) && (!isUnsignedRelop || (cns > 0))) - { - AssertionDsc dsc = AssertionDsc::CreateConstantBound(this, relopFunc, op1VN, op2VN); - AssertionIndex idx = optAddAssertion(dsc); - optCreateComplementaryAssertion(idx); - return idx; + // For TP reasons, limited to 32-bit constants on the op2 side. + if (op1VN != ValueNumStore::NoVN && op2VN != ValueNumStore::NoVN && vnStore->IsVNInt32Constant(op2VN) && + !vnStore->IsVNHandle(op2VN)) + { + AssertionDsc assertion = AssertionDsc::CreateInt32ConstantVNAssertion(this, op1VN, op2VN, equals); + return optAddAssertion(assertion); + } + } } - return NO_ASSERTION_INDEX; } /***************************************************************************** * - * Compute assertions for the JTrue node. + * If tree is a constant node holding an integral value, retrieve the value in + * pConstant. If the method returns true, pConstant holds the appropriate + * constant. Set "vnBased" to true to indicate local or global assertion prop. + * "pFlags" indicates if the constant is a handle marked by GTF_ICON_HDL_MASK. */ -AssertionInfo Compiler::optAssertionGenJtrue(GenTree* tree) +bool Compiler::optIsTreeKnownIntValue(bool vnBased, GenTree* tree, ssize_t* pConstant, GenTreeFlags* pFlags) { - GenTree* const relop = tree->AsOp()->gtOp1; - if (!relop->OperIsCompare()) + // Is Local assertion prop? + if (!vnBased) { - return NO_ASSERTION_INDEX; + if (tree->OperIs(GT_CNS_INT)) + { + *pConstant = tree->AsIntCon()->IconValue(); + *pFlags = tree->GetIconHandleFlag(); + return true; + } + return false; } - AssertionInfo info = optCreateJTrueBoundsAssertion(tree); - if (info.HasAssertion()) + // Global assertion prop + ValueNum vn = vnStore->VNConservativeNormalValue(tree->gtVNPair); + if (!vnStore->IsVNConstant(vn)) { - return info; + return false; } - if (optLocalAssertionProp && !optCrossBlockLocalAssertionProp) + // ValueNumber 'vn' indicates that this node evaluates to a constant + + var_types vnType = vnStore->TypeOfVN(vn); + if (vnType == TYP_INT) { - return NO_ASSERTION_INDEX; + *pConstant = vnStore->ConstantValue(vn); + *pFlags = vnStore->IsVNHandle(vn) ? vnStore->GetHandleFlags(vn) : GTF_EMPTY; + return true; } - - // Find assertion kind. - bool equals; - switch (relop->gtOper) +#ifdef TARGET_64BIT + else if (vnType == TYP_LONG) { - case GT_EQ: - equals = true; - break; - case GT_NE: - equals = false; - break; - default: - // TODO-CQ: add other relop operands. Disabled for now to measure perf - // and not occupy assertion table slots. We'll add them when used. - return NO_ASSERTION_INDEX; + *pConstant = vnStore->ConstantValue(vn); + *pFlags = vnStore->IsVNHandle(vn) ? vnStore->GetHandleFlags(vn) : GTF_EMPTY; + return true; } +#endif - // Look through any CSEs so we see the actual trees providing values, if possible. - // This is important for exact type assertions, which need to see the GT_IND. - // - GenTree* op1 = relop->AsOp()->gtOp1->gtCommaStoreVal(); - GenTree* op2 = relop->AsOp()->gtOp2->gtCommaStoreVal(); + return false; +} - // Avoid creating local assertions for float types. - // - if (optLocalAssertionProp && varTypeIsFloating(op1)) - { - return NO_ASSERTION_INDEX; - } +/***************************************************************************** + * + * Given an assertion add it to the assertion table + * + * If it is already in the assertion table return the assertionIndex that + * we use to refer to this element. + * Otherwise add it to the assertion table and return the assertionIndex that + * we use to refer to this element. + * If we need to add to the table and the table is full return the value zero + */ +AssertionIndex Compiler::optAddAssertion(const AssertionDsc& newAssertion) +{ + bool canAddNewAssertions = optAssertionCount < optMaxAssertionCount; - // See if we have IND(obj) ==/!= TypeHandle + // See if we already have this assertion in the table. // - if (!optLocalAssertionProp && op1->OperIs(GT_IND) && op1->gtGetOp1()->TypeIs(TYP_REF)) + // For local assertion prop we can speed things up by checking the dep vector. + // Note we only need check the op1 vector; copies get indexed on both op1 + // and op2, so searching the first will find any existing match. + // + if (optLocalAssertionProp) { - ValueNum objVN = optConservativeNormalVN(op1->gtGetOp1()); - ValueNum typeHndVN = optConservativeNormalVN(op2); + assert(newAssertion.GetOp1().KindIs(O1K_LCLVAR)); - if ((objVN != ValueNumStore::NoVN) && vnStore->IsVNTypeHandle(typeHndVN)) + unsigned lclNum = newAssertion.GetOp1().GetLclNum(); + BitVecOps::Iter iter(apTraits, GetAssertionDep(lclNum)); + unsigned bvIndex = 0; + while (iter.NextElem(&bvIndex)) { - AssertionDsc dsc = AssertionDsc::CreateSubtype(this, objVN, typeHndVN, /*exact*/ true); - AssertionIndex index = optAddAssertion(dsc); - - // We don't need to create a complementary assertion here. We're only interested - // in the assertion that the object is of a certain type. The opposite assertion - // (that the object is not of a certain type) is not useful (at least not yet). - // - // So if we have "if (obj->pMT != CNS) then create the assertion for the "else" edge. - if (relop->OperIs(GT_NE)) - { - return AssertionInfo::ForNextEdge(index); - } - return index; - } - } - - // Check for op1 or op2 to be lcl var and if so, keep it in op1. - if (!op1->OperIs(GT_LCL_VAR) && op2->OperIs(GT_LCL_VAR)) - { - std::swap(op1, op2); - } + AssertionIndex const index = GetAssertionIndex(bvIndex); + const AssertionDsc& curAssertion = optGetAssertion(index); - // If op1 is lcl and op2 is const or lcl, create assertion. - if (op1->OperIs(GT_LCL_VAR) && (op2->OperIsConst() || op2->OperIs(GT_LCL_VAR))) // Fix for Dev10 851483 - { - // Watch out for cases where long local(s) are implicitly truncated. - // - LclVarDsc* const lcl1Dsc = lvaGetDesc(op1->AsLclVarCommon()); - if (lcl1Dsc->TypeIs(TYP_LONG) && !op1->TypeIs(TYP_LONG)) - { - return NO_ASSERTION_INDEX; - } - if (op2->OperIs(GT_LCL_VAR)) - { - LclVarDsc* const lcl2Dsc = lvaGetDesc(op2->AsLclVarCommon()); - if (lcl2Dsc->TypeIs(TYP_LONG) && !op2->TypeIs(TYP_LONG)) + if (curAssertion.Equals(newAssertion, /* vnBased */ false)) { - return NO_ASSERTION_INDEX; + return index; } } - - return optCreateJtrueAssertions(op1, op2, equals); } - else if (!optLocalAssertionProp) + else { - ValueNum op1VN = vnStore->VNConservativeNormalValue(op1->gtVNPair); - ValueNum op2VN = vnStore->VNConservativeNormalValue(op2->gtVNPair); - - if (vnStore->IsVNCheckedBound(op1VN) && vnStore->IsVNInt32Constant(op2VN)) + bool mayHaveDuplicates = + optAssertionHasAssertionsForVN(newAssertion.GetOp1().GetVN(), /* addIfNotFound */ canAddNewAssertions); + // We need to register op2.vn too, even if we know for sure there are no duplicates + if (newAssertion.GetOp2().KindIs(O2K_CHECKED_BOUND_ADD_CNS)) { - assert(relop->OperIs(GT_EQ, GT_NE)); - return optCreateJtrueAssertions(op1, op2, equals); + mayHaveDuplicates |= optAssertionHasAssertionsForVN(newAssertion.GetOp2().GetCheckedBound(), + /* addIfNotFound */ canAddNewAssertions); + + // Additionally, check for the pattern of "VN + const == checkedBndVN" and register "VN" as well. + ValueNum addOpVN; + if (vnStore->IsVNBinFuncWithConst(newAssertion.GetOp1().GetVN(), VNF_ADD, &addOpVN, nullptr)) + { + mayHaveDuplicates |= optAssertionHasAssertionsForVN(addOpVN, /* addIfNotFound */ canAddNewAssertions); + } } - } - // Check op1 and op2 for an indirection of a GT_LCL_VAR and keep it in op1. - if ((!op1->OperIs(GT_IND) || !op1->AsOp()->gtOp1->OperIs(GT_LCL_VAR)) && - (op2->OperIs(GT_IND) && op2->AsOp()->gtOp1->OperIs(GT_LCL_VAR))) - { - std::swap(op1, op2); - } - // If op1 is ind, then extract op1's oper. - if (op1->OperIs(GT_IND) && op1->AsOp()->gtOp1->OperIs(GT_LCL_VAR)) - { - return optCreateJtrueAssertions(op1, op2, equals); + if (mayHaveDuplicates) + { + // For global prop we search the entire table. + // + // Check if exists already, so we can skip adding new one. Search backwards. + for (AssertionIndex index = optAssertionCount; index >= 1; index--) + { + const AssertionDsc& curAssertion = optGetAssertion(index); + if (curAssertion.Equals(newAssertion, /* vnBased */ true)) + { + return index; + } + } + } } - // Look for a call to an IsInstanceOf helper compared to a nullptr - if (!op2->OperIs(GT_CNS_INT) && op1->OperIs(GT_CNS_INT)) - { - std::swap(op1, op2); - } - // Validate op1 and op2 - if (!op1->OperIs(GT_CALL) || !op1->AsCall()->IsHelperCall() || !op1->TypeIs(TYP_REF) || // op1 - !op2->OperIs(GT_CNS_INT) || (op2->AsIntCon()->gtIconVal != 0)) // op2 + // Check if we are within max count. + if (!canAddNewAssertions) { + optAssertionOverflow++; return NO_ASSERTION_INDEX; } - if (optLocalAssertionProp) + optAssertionTabPrivate[optAssertionCount] = newAssertion; + optAssertionCount++; + +#ifdef DEBUG + if (verbose) { - // O1K_SUBTYPE is Global Assertion Prop only - return NO_ASSERTION_INDEX; + printf("GenTreeNode creates assertion:\n"); + gtDispTree(optAssertionPropCurrentTree, nullptr, nullptr, true); + printf(optLocalAssertionProp ? "In " FMT_BB " New Local " : "In " FMT_BB " New Global ", compCurBB->bbNum); + optPrintAssertion(newAssertion, optAssertionCount); } +#endif // DEBUG - GenTreeCall* const call = op1->AsCall(); + // Track the short-circuit criteria + optCanPropLclVar |= newAssertion.CanPropLclVar(); + optCanPropEqual |= newAssertion.CanPropEqualOrNotEqual(); + optCanPropNonNull |= newAssertion.CanPropNonNull(); + optCanPropSubRange |= newAssertion.CanPropSubRange(); + optCanPropBndsChk |= newAssertion.CanPropBndsCheck(); - // Note CORINFO_HELP_READYTORUN_ISINSTANCEOF does not have the same argument pattern. - // In particular, it is not possible to deduce what class is being tested from its args. - // - // Also note The CASTCLASS helpers won't appear in predicates as they throw on failure. - // So the helper list here is smaller than the one in optAssertionProp_Call. - // - CorInfoHelpFunc helper = eeGetHelperNum(call->gtCallMethHnd); - if ((helper == CORINFO_HELP_ISINSTANCEOFINTERFACE) || (helper == CORINFO_HELP_ISINSTANCEOFARRAY) || - (helper == CORINFO_HELP_ISINSTANCEOFCLASS) || (helper == CORINFO_HELP_ISINSTANCEOFANY)) + // Assertion mask bits are [index + 1]. + if (optLocalAssertionProp) { - GenTree* objectNode = call->gtArgs.GetUserArgByIndex(1)->GetNode(); - GenTree* methodTableNode = call->gtArgs.GetUserArgByIndex(0)->GetNode(); - - // objectNode can be TYP_I_IMPL in case if it's a constant handle - // (e.g. a string literal from frozen segments) - // - assert(objectNode->TypeIs(TYP_REF, TYP_I_IMPL)); - assert(methodTableNode->TypeIs(TYP_I_IMPL)); - - ValueNum objVN = optConservativeNormalVN(objectNode); - ValueNum typeHndVN = optConservativeNormalVN(methodTableNode); + assert(newAssertion.GetOp1().KindIs(O1K_LCLVAR)); - if ((objVN != ValueNumStore::NoVN) && vnStore->IsVNTypeHandle(typeHndVN)) + // Mark the variables this index depends on + unsigned lclNum = newAssertion.GetOp1().GetLclNum(); + BitVecOps::AddElemD(apTraits, GetAssertionDep(lclNum), optAssertionCount - 1); + if (newAssertion.GetOp2().KindIs(O2K_LCLVAR_COPY)) { - AssertionDsc dsc = AssertionDsc::CreateSubtype(this, objVN, typeHndVN, /*exact*/ false); - AssertionIndex index = optAddAssertion(dsc); - - // We don't need to create a complementary assertion here. We're only interested - // in the assertion that the object is of a certain type. The opposite assertion - // (that the object is not of a certain type) is not useful (at least not yet). - // - // So if we have "if (ISINST(obj, pMT) == null) then create the assertion for the "else" edge. - // - if (relop->OperIs(GT_EQ)) - { - return AssertionInfo::ForNextEdge(index); - } - return index; + lclNum = newAssertion.GetOp2().GetLclNum(); + BitVecOps::AddElemD(apTraits, GetAssertionDep(lclNum), optAssertionCount - 1); } } - return NO_ASSERTION_INDEX; +#ifdef DEBUG + optDebugCheckAssertions(optAssertionCount); +#endif + return optAssertionCount; } -/***************************************************************************** - * - * If this node creates an assertion then assign an index to the assertion - * by adding it to the lookup table, if necessary. - */ -void Compiler::optAssertionGen(GenTree* tree) +//------------------------------------------------------------------------ +// optAssertionHasAssertionsForVN: Check if we already have assertions for the given VN. +// If "addIfNotFound" is true, add the VN to the map if it's not already there. +// +// Arguments: +// vn - the VN to check for +// addIfNotFound - whether to add the VN to the map if it's not found +// +// Return Value: +// true if we already have assertions for the given VN, false otherwise. +// +bool Compiler::optAssertionHasAssertionsForVN(ValueNum vn, bool addIfNotFound) { - tree->ClearAssertion(); - - // If there are QMARKs in the IR, we won't generate assertions - // for conditionally executed code. - // - if (optLocalAssertionProp && ((tree->gtFlags & GTF_COLON_COND) != 0)) + assert(!optLocalAssertionProp); + if (vn == ValueNumStore::NoVN) { - return; + assert(!addIfNotFound); + return false; } -#ifdef DEBUG - optAssertionPropCurrentTree = tree; -#endif - - AssertionInfo assertionInfo; - switch (tree->OperGet()) + if (addIfNotFound) { - case GT_STORE_LCL_VAR: - // VN takes care of non local assertions for data flow. - if (optLocalAssertionProp) - { - assertionInfo = optCreateAssertion(tree, tree->AsLclVar()->Data(), /*equals*/ true); - } - break; - - case GT_IND: - case GT_XAND: - case GT_XORR: - case GT_XADD: - case GT_XCHG: - case GT_CMPXCHG: - case GT_BLK: - case GT_STOREIND: - case GT_STORE_BLK: - case GT_NULLCHECK: - case GT_ARR_LENGTH: - case GT_MDARR_LENGTH: - case GT_MDARR_LOWER_BOUND: - // These indirs (esp. GT_IND and GT_STOREIND) are the most popular sources of assertions. - if (tree->IndirMayFault(this)) - { - assertionInfo = optCreateAssertion(tree->GetIndirOrArrMetaDataAddr(), nullptr, /*equals*/ false); - } - break; - - case GT_INTRINSIC: - if (tree->AsIntrinsic()->gtIntrinsicName == NI_System_Object_GetType) - { - assertionInfo = optCreateAssertion(tree->AsIntrinsic()->gtGetOp1(), nullptr, /*equals*/ false); - } - break; - - case GT_BOUNDS_CHECK: - if (!optLocalAssertionProp) - { - ValueNum idxVN = optConservativeNormalVN(tree->AsBoundsChk()->GetIndex()); - ValueNum lenVN = optConservativeNormalVN(tree->AsBoundsChk()->GetArrayLength()); - if ((idxVN == ValueNumStore::NoVN) || (lenVN == ValueNumStore::NoVN)) - { - assertionInfo = NO_ASSERTION_INDEX; - } - else - { - // GT_BOUNDS_CHECK node provides the following contract: - // * idxVN < lenVN - // * lenVN is non-negative - assertionInfo = optAddAssertion(AssertionDsc::CreateNoThrowArrBnd(idxVN, lenVN)); - } - } - break; - - case GT_ARR_ELEM: - // An array element reference can create a non-null assertion - assertionInfo = optCreateAssertion(tree->AsArrElem()->gtArrObj, nullptr, /*equals*/ false); - break; + // Lazy initialize the map when we first need to add to it + if (optAssertionVNsMap == nullptr) + { + optAssertionVNsMap = new (this, CMK_AssertionProp) VNSet(getAllocator(CMK_AssertionProp)); + } - case GT_CALL: + // Avoid double lookup by using the return value of LookupPointerOrAdd to + // determine whether the VN was already in the map. + bool* pValue = optAssertionVNsMap->LookupPointerOrAdd(vn, false); + if (!*pValue) { - // A virtual call can create a non-null assertion. We transform some virtual calls into non-virtual calls - // with a GTF_CALL_NULLCHECK flag set. - // Ignore tail calls because they have 'this` pointer in the regular arg list and an implicit null check. - GenTreeCall* const call = tree->AsCall(); - if (call->NeedsNullCheck() || (call->IsVirtual() && !call->IsTailCall())) - { - // Retrieve the 'this' arg. - GenTree* thisArg = call->gtArgs.GetThisArg()->GetNode(); - assert(thisArg != nullptr); - assertionInfo = optCreateAssertion(thisArg, nullptr, /*equals*/ false); - } + *pValue = true; + return false; } - break; + return true; + } - case GT_JTRUE: - assertionInfo = optAssertionGenJtrue(tree); + // Otherwise just do a normal lookup + return (optAssertionVNsMap != nullptr) && optAssertionVNsMap->Lookup(vn); +} + +#ifdef DEBUG +void Compiler::optDebugCheckAssertion(const AssertionDsc& assertion) const +{ + switch (assertion.GetOp1().GetKind()) + { + case O1K_EXACT_TYPE: + case O1K_SUBTYPE: + case O1K_VN: + assert(!optLocalAssertionProp); break; - default: - // All other gtOper node kinds, leave 'assertionIndex' = NO_ASSERTION_INDEX break; } - if (assertionInfo.HasAssertion()) + switch (assertion.GetOp2().GetKind()) { - tree->SetAssertionInfo(assertionInfo); - } -} + case O2K_SUBRANGE: + case O2K_LCLVAR_COPY: + assert(optLocalAssertionProp); + break; -/***************************************************************************** - * - * Maps a complementary assertion to its original assertion so it can be - * retrieved faster. - */ -void Compiler::optMapComplementary(AssertionIndex assertionIndex, AssertionIndex index) -{ - if (assertionIndex == NO_ASSERTION_INDEX || index == NO_ASSERTION_INDEX) - { - return; - } + case O2K_ZEROOBJ: + // We only make these assertion for stores (not control flow). + assert(assertion.KindIs(OAK_EQUAL)); + // We use "optLocalAssertionIsEqualOrNotEqual" to find these. + break; - assert(assertionIndex <= optMaxAssertionCount); - assert(index <= optMaxAssertionCount); + case O2K_CONST_DOUBLE: + assert(!FloatingPointUtils::isNaN(assertion.GetOp2().GetDoubleConstant())); + break; - optComplementaryAssertionMap[assertionIndex] = index; - optComplementaryAssertionMap[index] = assertionIndex; + default: + // for all other 'assertion.GetOp2().GetKind()' values we don't check anything + break; + } } /***************************************************************************** * - * Given an assertion index, return the assertion index of the complementary - * assertion or 0 if one does not exist. + * Verify that assertion prop related assumptions are valid. If "index" + * is 0 (i.e., NO_ASSERTION_INDEX) then verify all assertions in the table. + * If "index" is between 1 and optAssertionCount, then verify the assertion + * desc corresponding to "index." */ -AssertionIndex Compiler::optFindComplementary(AssertionIndex assertIndex) +void Compiler::optDebugCheckAssertions(AssertionIndex index) { - if (assertIndex == NO_ASSERTION_INDEX) - { - return NO_ASSERTION_INDEX; - } - const AssertionDsc& inputAssertion = optGetAssertion(assertIndex); - - // Must be an equal or not equal assertion. - if (!AssertionDsc::IsReversible(inputAssertion.GetKind())) - { - return NO_ASSERTION_INDEX; - } - - AssertionIndex index = optComplementaryAssertionMap[assertIndex]; - if (index != NO_ASSERTION_INDEX && index <= optAssertionCount) - { - return index; - } - - for (AssertionIndex index = 1; index <= optAssertionCount; ++index) + AssertionIndex start = (index == NO_ASSERTION_INDEX) ? 1 : index; + AssertionIndex end = (index == NO_ASSERTION_INDEX) ? optAssertionCount : index; + for (AssertionIndex ind = start; ind <= end; ++ind) { - // Make sure assertion kinds are complementary and op1, op2 kinds match. - const AssertionDsc& curAssertion = optGetAssertion(index); - if (curAssertion.Complementary(inputAssertion, !optLocalAssertionProp)) - { - optMapComplementary(assertIndex, index); - return index; - } + const AssertionDsc& assertion = optGetAssertion(ind); + optDebugCheckAssertion(assertion); } - return NO_ASSERTION_INDEX; } +#endif //------------------------------------------------------------------------ -// optAssertionIsSubrange: Find a subrange assertion for the given range and tree. -// -// This function will return the index of the first assertion in "assertions" -// which claims that the value of "tree" is within the bounds of the provided -// "range" (i. e. "range.Contains(assertedRange)"). +// optCreateComplementaryAssertion: Create an assertion that is the complementary +// of the specified assertion. // // Arguments: -// tree - the tree for which to find the assertion -// range - range the subrange of which to look for -// assertions - the set of assertions +// assertionIndex - the index of the assertion // -// Return Value: -// Index of the found assertion, NO_ASSERTION_INDEX otherwise. +// Notes: +// The created complementary assertion is associated with the original +// assertion such that it can be found by optFindComplementary. // -AssertionIndex Compiler::optAssertionIsSubrange(GenTree* tree, IntegralRange range, ASSERT_VALARG_TP assertions) +void Compiler::optCreateComplementaryAssertion(AssertionIndex assertionIndex) { - assert(optLocalAssertionProp); // Subrange assertions are local only. - if (!optCanPropSubRange) + if (assertionIndex == NO_ASSERTION_INDEX) { - return NO_ASSERTION_INDEX; + return; } - BitVecOps::Iter iter(apTraits, assertions); - unsigned bvIndex = 0; - while (iter.NextElem(&bvIndex)) + const AssertionDsc& candidateAssertion = optGetAssertion(assertionIndex); + if (candidateAssertion.KindIs(OAK_EQUAL)) { - AssertionIndex const index = GetAssertionIndex(bvIndex); - const AssertionDsc& curAssertion = optGetAssertion(index); - if (curAssertion.CanPropSubRange()) + // Don't create useless OAK_NOT_EQUAL assertions + + if (candidateAssertion.GetOp1().KindIs(O1K_LCLVAR, O1K_VN)) { - if (curAssertion.GetOp1().GetLclNum() != tree->AsLclVarCommon()->GetLclNum()) + // "LCLVAR != CNS" is not a useful assertion (unless CNS is 0/1) + if (candidateAssertion.GetOp2().KindIs(O2K_CONST_INT) && + (candidateAssertion.GetOp2().GetIntConstant() != 0) && + (candidateAssertion.GetOp2().GetIntConstant() != 1)) { - continue; + return; } - if (range.Contains(curAssertion.GetOp2().GetIntegralRange())) + // "LCLVAR != LCLVAR_COPY" + if (candidateAssertion.GetOp2().KindIs(O2K_LCLVAR_COPY)) { - return index; + return; } } + + // "Object is not Class" is also not a useful assertion (at least for now) + if (candidateAssertion.GetOp1().KindIs(O1K_EXACT_TYPE, O1K_SUBTYPE)) + { + return; + } + AssertionDsc reversed = candidateAssertion.Reverse(); + optMapComplementary(optAddAssertion(reversed), assertionIndex); + } + else if (candidateAssertion.KindIs(OAK_LT_UN, OAK_LE_UN) && + candidateAssertion.GetOp2().KindIs(O2K_CHECKED_BOUND_ADD_CNS)) + { + // Assertions such as "X > checkedBndVN" aren't very useful. + return; + } + else if (AssertionDsc::IsReversible(candidateAssertion.GetKind())) + { + AssertionDsc reversed = candidateAssertion.Reverse(); + optMapComplementary(optAddAssertion(reversed), assertionIndex); } +} - return NO_ASSERTION_INDEX; +//------------------------------------------------------------------------ +// optCreateJtrueAssertions: Create assertions about a JTRUE's relop operands. +// +// Arguments: +// op1 - the first assertion operand +// op2 - the second assertion operand +// equals - the assertion kind (equals / not equals) +// +// Return Value: +// The new assertion index or NO_ASSERTION_INDEX if a new assertion +// was not created. +// +// Notes: +// Assertion creation may fail either because the provided assertion +// operands aren't supported or because the assertion table is full. +// If an assertion is created successfully then an attempt is made to also +// create a second, complementary assertion. This may too fail, for the +// same reasons as the first one. +// +AssertionIndex Compiler::optCreateJtrueAssertions(GenTree* op1, GenTree* op2, bool equals) +{ + AssertionIndex assertionIndex = optCreateAssertion(op1, op2, equals); + // Don't bother if we don't have an assertion on the JTrue False path. Current implementation + // allows for a complementary only if there is an assertion on the False path (tree->HasAssertion()). + if (assertionIndex != NO_ASSERTION_INDEX) + { + optCreateComplementaryAssertion(assertionIndex); + } + return assertionIndex; } -/********************************************************************************** - * - * Given a "tree" that is usually arg1 of a isinst/cast kind of GT_CALL (a class - * handle), and "methodTableArg" which is a const int (a class handle), then search - * if there is an assertion in "assertions", that asserts the equality of the two - * class handles and then returns the index of the assertion. If one such assertion - * could not be found, then it returns NO_ASSERTION_INDEX. - * - */ -AssertionIndex Compiler::optAssertionIsSubtype(GenTree* tree, GenTree* methodTableArg, ASSERT_VALARG_TP assertions) +AssertionInfo Compiler::optCreateJTrueBoundsAssertion(GenTree* tree) { - BitVecOps::Iter iter(apTraits, assertions); - unsigned bvIndex = 0; - while (iter.NextElem(&bvIndex)) + // These assertions are VN based, so not relevant for local prop + // + if (optLocalAssertionProp) { - AssertionIndex const index = GetAssertionIndex(bvIndex); - const AssertionDsc& curAssertion = optGetAssertion(index); - if (!curAssertion.KindIs(OAK_EQUAL) || !curAssertion.GetOp1().KindIs(O1K_SUBTYPE, O1K_EXACT_TYPE)) - { - // TODO-CQ: We might benefit from OAK_NOT_EQUAL assertion as well, e.g.: - // if (obj is not MyClass) // obj is known to be never of MyClass class - // { - // if (obj is MyClass) // can be folded to false - // { - // - continue; - } + return NO_ASSERTION_INDEX; + } - if ((curAssertion.GetOp1().GetVN() != vnStore->VNConservativeNormalValue(tree->gtVNPair) || - !curAssertion.GetOp2().KindIs(O2K_CONST_INT))) - { - continue; - } + GenTree* relop = tree->gtGetOp1(); + if (!relop->OperIsCompare()) + { + return NO_ASSERTION_INDEX; + } - ssize_t methodTableVal = 0; - GenTreeFlags iconFlags = GTF_EMPTY; - if (!optIsTreeKnownIntValue(!optLocalAssertionProp, methodTableArg, &methodTableVal, &iconFlags)) - { - continue; - } + ValueNum relopVN = optConservativeNormalVN(relop); + VNFuncApp relopFuncApp; + if (!vnStore->GetVNFunc(relopVN, &relopFuncApp)) + { + // We're expecting a relop here + return NO_ASSERTION_INDEX; + } - if (curAssertion.GetOp2().GetIntConstant() == methodTableVal) - { - // TODO-CQ: if they don't match, we might still be able to prove that the result is foldable via - // compareTypesForCast. - return index; - } + bool isUnsignedRelop; + if (relopFuncApp.FuncIs(VNF_LE, VNF_LT, VNF_GE, VNF_GT)) + { + isUnsignedRelop = false; + } + else if (relopFuncApp.FuncIs(VNF_LE_UN, VNF_LT_UN, VNF_GE_UN, VNF_GT_UN)) + { + isUnsignedRelop = true; + } + else + { + // Not a relop we're interested in. + // Assertions for NE/EQ are handled elsewhere. + return NO_ASSERTION_INDEX; } - return NO_ASSERTION_INDEX; -} - -//------------------------------------------------------------------------------ -// optVNBasedFoldExpr_Call_Memset: Unrolls NI_System_SpanHelpers_Fill for constant length. -// -// Arguments: -// call - NI_System_SpanHelpers_Fill call to unroll -// -// Return Value: -// Returns a new tree or nullptr if nothing is changed. -// -GenTree* Compiler::optVNBasedFoldExpr_Call_Memset(GenTreeCall* call) -{ - assert(call->IsSpecialIntrinsic(this, NI_System_SpanHelpers_Fill)); - CallArg* dstArg = call->gtArgs.GetUserArgByIndex(0); - CallArg* lenArg = call->gtArgs.GetUserArgByIndex(1); - CallArg* valArg = call->gtArgs.GetUserArgByIndex(2); - - var_types valType = valArg->GetSignatureType(); - unsigned lengthScale = genTypeSize(valType); + VNFunc relopFunc = relopFuncApp.m_func; + ValueNum op1VN = relopFuncApp.m_args[0]; + ValueNum op2VN = relopFuncApp.m_args[1]; - if (lengthScale == 1) + if ((genActualType(vnStore->TypeOfVN(op1VN)) != TYP_INT) || (genActualType(vnStore->TypeOfVN(op2VN)) != TYP_INT)) { - // Lower expands it slightly better. - JITDUMP("...value's type is byte - leave it for lower to expand.\n"); - return nullptr; + // For now, we don't have consumers for assertions derived from non-int32 comparisons + return NO_ASSERTION_INDEX; } - if (varTypeIsStruct(valType) || varTypeIsGC(valType)) + // "CheckedBnd X" + if (!isUnsignedRelop && vnStore->IsVNCheckedBound(op1VN)) { - JITDUMP("...value's type is not supported - bail out.\n"); - return nullptr; + // Move the checked bound to the right side for simplicity + relopFunc = ValueNumStore::SwapRelop(relopFunc); + AssertionDsc dsc = AssertionDsc::CreateCompareCheckedBound(relopFunc, op2VN, op1VN, 0); + AssertionIndex idx = optAddAssertion(dsc); + optCreateComplementaryAssertion(idx); + return idx; } - ValueNum lenVN = vnStore->VNConservativeNormalValue(lenArg->GetNode()->gtVNPair); - if (!vnStore->IsVNConstant(lenVN)) + // "X CheckedBnd" + if (!isUnsignedRelop && vnStore->IsVNCheckedBound(op2VN)) { - JITDUMP("...length is not a constant - bail out.\n"); - return nullptr; + AssertionDsc dsc = AssertionDsc::CreateCompareCheckedBound(relopFunc, op1VN, op2VN, 0); + AssertionIndex idx = optAddAssertion(dsc); + optCreateComplementaryAssertion(idx); + return idx; } - size_t len = vnStore->CoercedConstantValue(lenVN); - if ((len > getUnrollThreshold(Memset)) || - // The first condition prevents the overflow in the second condition. - // since both len and lengthScale are expected to be small at this point. - (len * lengthScale) > getUnrollThreshold(Memset)) + // "(CheckedBnd + CNS) X" + ValueNum checkedBnd; + int checkedBndCns; + if (!isUnsignedRelop && vnStore->IsVNCheckedBoundAddConst(op1VN, &checkedBnd, &checkedBndCns)) { - JITDUMP("...length is too big to unroll - bail out.\n"); - return nullptr; + // Move the (CheckedBnd + CNS) part to the right side for simplicity + relopFunc = ValueNumStore::SwapRelop(relopFunc); + AssertionDsc dsc = AssertionDsc::CreateCompareCheckedBound(relopFunc, op2VN, checkedBnd, checkedBndCns); + AssertionIndex idx = optAddAssertion(dsc); + optCreateComplementaryAssertion(idx); + return idx; } - // Some arbitrary threshold if the value is not a constant, - // since it is unlikely that we can optimize it further. - if (!valArg->GetNode()->OperIsConst() && (len >= 8)) + // "X (CheckedBnd + CNS)" + if (!isUnsignedRelop && vnStore->IsVNCheckedBoundAddConst(op2VN, &checkedBnd, &checkedBndCns)) { - JITDUMP("...length is too big to unroll for non-constant value - bail out.\n"); - return nullptr; + AssertionDsc dsc = AssertionDsc::CreateCompareCheckedBound(relopFunc, op1VN, checkedBnd, checkedBndCns); + AssertionIndex idx = optAddAssertion(dsc); + optCreateComplementaryAssertion(idx); + return idx; } - // Spill the side effects directly in the args, we're going to - // pick them up in the following gtExtractSideEffList - GenTree* dst = fgMakeMultiUse(&dstArg->NodeRef()); - GenTree* val = fgMakeMultiUse(&valArg->NodeRef()); + // Loop condition like "(uint)i < (uint)bnd" or equivalent + // Assertion: "no throw" since this condition guarantees that i is both >= 0 and < bnd (on the appropriate edge) + ValueNumStore::UnsignedCompareCheckedBoundInfo unsignedCompareBnd; + if (vnStore->IsVNUnsignedCompareCheckedBound(relopVN, &unsignedCompareBnd)) + { + ValueNum idxVN = vnStore->VNNormalValue(unsignedCompareBnd.vnIdx); + ValueNum lenVN = vnStore->VNNormalValue(unsignedCompareBnd.vnBound); - GenTree* result = nullptr; - gtExtractSideEffList(call, &result, GTF_ALL_EFFECT, true); + AssertionDsc dsc = AssertionDsc::CreateNoThrowArrBnd(idxVN, lenVN); + AssertionIndex index = optAddAssertion(dsc); + if (unsignedCompareBnd.cmpOper == VNF_GE_UN) + { + // By default JTRUE generated assertions hold on the "jump" edge. We have i >= bnd but we're really + // after i < bnd so we need to change the assertion edge to "next". + return AssertionInfo::ForNextEdge(index); + } + return index; + } - for (size_t offset = 0; offset < len; offset++) + // Create "X relop CNS" assertion (both signed and unsigned relops) + // Ignore non-positive constants for unsigned relops as they don't add any useful information. + ssize_t cns; + if (vnStore->IsVNIntegralConstant(op1VN, &cns) && (!isUnsignedRelop || (cns > 0))) { - // Clone dst and add offset if necessary. - GenTree* offsetNode = gtNewIconNode((ssize_t)(offset * lengthScale), TYP_I_IMPL); - GenTree* currDst = gtNewOperNode(GT_ADD, dst->TypeGet(), gtCloneExpr(dst), offsetNode); - GenTreeStoreInd* storeInd = - gtNewStoreIndNode(valType, currDst, gtCloneExpr(val), GTF_IND_UNALIGNED | GTF_IND_ALLOW_NON_ATOMIC); + relopFunc = ValueNumStore::SwapRelop(relopFunc); + AssertionDsc dsc = AssertionDsc::CreateConstantBound(this, relopFunc, op2VN, op1VN); + AssertionIndex idx = optAddAssertion(dsc); + optCreateComplementaryAssertion(idx); + return idx; + } - // Merge with the previous result. - result = result == nullptr ? storeInd : gtNewOperNode(GT_COMMA, TYP_VOID, result, storeInd); + if (vnStore->IsVNIntegralConstant(op2VN, &cns) && (!isUnsignedRelop || (cns > 0))) + { + AssertionDsc dsc = AssertionDsc::CreateConstantBound(this, relopFunc, op1VN, op2VN); + AssertionIndex idx = optAddAssertion(dsc); + optCreateComplementaryAssertion(idx); + return idx; } - JITDUMP("...optimized into STOREIND(s):\n"); - DISPTREE(result); - return result; + return NO_ASSERTION_INDEX; } -//------------------------------------------------------------------------------ -// optVNBasedFoldExpr_Call_Memmove: Unrolls NI_System_SpanHelpers_Memmove/CORINFO_HELP_MEMCPY -// if possible. This function effectively duplicates LowerCallMemmove. -// However, unlike LowerCallMemmove, it is able to optimize src into constants with help of VN. -// -// Arguments: -// call - NI_System_SpanHelpers_Memmove/CORINFO_HELP_MEMCPY call to unroll -// -// Return Value: -// Returns a new tree or nullptr if nothing is changed. -// -GenTree* Compiler::optVNBasedFoldExpr_Call_Memmove(GenTreeCall* call) +/***************************************************************************** + * + * Compute assertions for the JTrue node. + */ +AssertionInfo Compiler::optAssertionGenJtrue(GenTree* tree) { - JITDUMP("See if we can optimize NI_System_SpanHelpers_Memmove with help of VN...\n") - assert(call->IsSpecialIntrinsic(this, NI_System_SpanHelpers_Memmove) || - call->IsHelperCall(this, CORINFO_HELP_MEMCPY)); + GenTree* const relop = tree->AsOp()->gtOp1; + if (!relop->OperIsCompare()) + { + return NO_ASSERTION_INDEX; + } - CallArg* dstArg = call->gtArgs.GetUserArgByIndex(0); - CallArg* srcArg = call->gtArgs.GetUserArgByIndex(1); - CallArg* lenArg = call->gtArgs.GetUserArgByIndex(2); - ValueNum lenVN = vnStore->VNConservativeNormalValue(lenArg->GetNode()->gtVNPair); - if (!vnStore->IsVNConstant(lenVN)) + AssertionInfo info = optCreateJTrueBoundsAssertion(tree); + if (info.HasAssertion()) { - JITDUMP("...length is not a constant - bail out.\n"); - return nullptr; + return info; } - size_t len = vnStore->CoercedConstantValue(lenVN); - if (len == 0) + if (optLocalAssertionProp && !optCrossBlockLocalAssertionProp) { - // Memmove(dst, src, 0) -> no-op. - // Memmove doesn't dereference src/dst pointers if length is 0. - JITDUMP("...length is 0 -> optimize to no-op.\n"); - return gtWrapWithSideEffects(gtNewNothingNode(), call, GTF_ALL_EFFECT, true); + return NO_ASSERTION_INDEX; } - if (len > getUnrollThreshold(Memcpy)) + // Find assertion kind. + bool equals; + switch (relop->gtOper) { - JITDUMP("...length is too big to unroll - bail out.\n"); - return nullptr; + case GT_EQ: + equals = true; + break; + case GT_NE: + equals = false; + break; + default: + // TODO-CQ: add other relop operands. Disabled for now to measure perf + // and not occupy assertion table slots. We'll add them when used. + return NO_ASSERTION_INDEX; + } + + // Look through any CSEs so we see the actual trees providing values, if possible. + // This is important for exact type assertions, which need to see the GT_IND. + // + GenTree* op1 = relop->AsOp()->gtOp1->gtCommaStoreVal(); + GenTree* op2 = relop->AsOp()->gtOp2->gtCommaStoreVal(); + + // Avoid creating local assertions for float types. + // + if (optLocalAssertionProp && varTypeIsFloating(op1)) + { + return NO_ASSERTION_INDEX; } - // if GetImmutableDataFromAddress returns true, it means that the src is a read-only constant. - // Thus, dst and src do not overlap (if they do - it's an UB). - uint8_t* buffer = new (this, CMK_AssertionProp) uint8_t[len]; - if (!GetImmutableDataFromAddress(srcArg->GetNode(), (int)len, buffer)) + // See if we have IND(obj) ==/!= TypeHandle + // + if (!optLocalAssertionProp && op1->OperIs(GT_IND) && op1->gtGetOp1()->TypeIs(TYP_REF)) { - JITDUMP("...src is not a constant - fallback to LowerCallMemmove.\n"); - return nullptr; + ValueNum objVN = optConservativeNormalVN(op1->gtGetOp1()); + ValueNum typeHndVN = optConservativeNormalVN(op2); + + if ((objVN != ValueNumStore::NoVN) && vnStore->IsVNTypeHandle(typeHndVN)) + { + AssertionDsc dsc = AssertionDsc::CreateSubtype(this, objVN, typeHndVN, /*exact*/ true); + AssertionIndex index = optAddAssertion(dsc); + + // We don't need to create a complementary assertion here. We're only interested + // in the assertion that the object is of a certain type. The opposite assertion + // (that the object is not of a certain type) is not useful (at least not yet). + // + // So if we have "if (obj->pMT != CNS) then create the assertion for the "else" edge. + if (relop->OperIs(GT_NE)) + { + return AssertionInfo::ForNextEdge(index); + } + return index; + } } - // if dstArg is not simple, we replace the arg directly with a temp assignment and - // continue using that temp - it allows us reliably extract all side effects. - GenTree* dst = fgMakeMultiUse(&dstArg->NodeRef()); + // Check for op1 or op2 to be lcl var and if so, keep it in op1. + if (!op1->OperIs(GT_LCL_VAR) && op2->OperIs(GT_LCL_VAR)) + { + std::swap(op1, op2); + } - // Now we're going to emit a chain of STOREIND via COMMA nodes. - // the very first tree is expected to be side-effects from the original call (including all args) - GenTree* result = nullptr; - gtExtractSideEffList(call, &result, GTF_ALL_EFFECT, true); + // If op1 is lcl and op2 is const or lcl, create assertion. + if (op1->OperIs(GT_LCL_VAR) && (op2->OperIsConst() || op2->OperIs(GT_LCL_VAR))) // Fix for Dev10 851483 + { + // Watch out for cases where long local(s) are implicitly truncated. + // + LclVarDsc* const lcl1Dsc = lvaGetDesc(op1->AsLclVarCommon()); + if (lcl1Dsc->TypeIs(TYP_LONG) && !op1->TypeIs(TYP_LONG)) + { + return NO_ASSERTION_INDEX; + } + if (op2->OperIs(GT_LCL_VAR)) + { + LclVarDsc* const lcl2Dsc = lvaGetDesc(op2->AsLclVarCommon()); + if (lcl2Dsc->TypeIs(TYP_LONG) && !op2->TypeIs(TYP_LONG)) + { + return NO_ASSERTION_INDEX; + } + } - unsigned lenRemaining = (unsigned)len; - while (lenRemaining > 0) + return optCreateJtrueAssertions(op1, op2, equals); + } + else if (!optLocalAssertionProp) { - const ssize_t offset = (ssize_t)len - (ssize_t)lenRemaining; + ValueNum op1VN = vnStore->VNConservativeNormalValue(op1->gtVNPair); + ValueNum op2VN = vnStore->VNConservativeNormalValue(op2->gtVNPair); - // Clone dst and add offset if necessary. - GenTree* currDst = gtCloneExpr(dst); - if (offset != 0) + if (vnStore->IsVNCheckedBound(op1VN) && vnStore->IsVNInt32Constant(op2VN)) { - currDst = gtNewOperNode(GT_ADD, dst->TypeGet(), currDst, gtNewIconNode(offset, TYP_I_IMPL)); + assert(relop->OperIs(GT_EQ, GT_NE)); + return optCreateJtrueAssertions(op1, op2, equals); } + } - // Create an unaligned STOREIND node using the largest possible word size. - var_types type = roundDownMaxType(lenRemaining); - GenTree* srcCns = gtNewGenericCon(type, buffer + offset); - GenTreeStoreInd* storeInd = gtNewStoreIndNode(type, currDst, srcCns, GTF_IND_UNALIGNED); - fgUpdateConstTreeValueNumber(srcCns); + // Check op1 and op2 for an indirection of a GT_LCL_VAR and keep it in op1. + if ((!op1->OperIs(GT_IND) || !op1->AsOp()->gtOp1->OperIs(GT_LCL_VAR)) && + (op2->OperIs(GT_IND) && op2->AsOp()->gtOp1->OperIs(GT_LCL_VAR))) + { + std::swap(op1, op2); + } + // If op1 is ind, then extract op1's oper. + if (op1->OperIs(GT_IND) && op1->AsOp()->gtOp1->OperIs(GT_LCL_VAR)) + { + return optCreateJtrueAssertions(op1, op2, equals); + } - // Merge with the previous result. - result = result == nullptr ? storeInd : gtNewOperNode(GT_COMMA, TYP_VOID, result, storeInd); + // Look for a call to an IsInstanceOf helper compared to a nullptr + if (!op2->OperIs(GT_CNS_INT) && op1->OperIs(GT_CNS_INT)) + { + std::swap(op1, op2); + } + // Validate op1 and op2 + if (!op1->OperIs(GT_CALL) || !op1->AsCall()->IsHelperCall() || !op1->TypeIs(TYP_REF) || // op1 + !op2->OperIs(GT_CNS_INT) || (op2->AsIntCon()->gtIconVal != 0)) // op2 + { + return NO_ASSERTION_INDEX; + } - lenRemaining -= genTypeSize(type); + if (optLocalAssertionProp) + { + // O1K_SUBTYPE is Global Assertion Prop only + return NO_ASSERTION_INDEX; } - JITDUMP("...optimized into STOREIND(s)!:\n"); - DISPTREE(result); - return result; -} + GenTreeCall* const call = op1->AsCall(); -//------------------------------------------------------------------------------ -// optVNBasedFoldExpr_Call: Folds given call using VN to a simpler tree. -// -// Arguments: -// block - The block containing the tree. -// parent - The parent node of the tree. -// call - The call to fold -// -// Return Value: -// Returns a new tree or nullptr if nothing is changed. -// -GenTree* Compiler::optVNBasedFoldExpr_Call(BasicBlock* block, GenTree* parent, GenTreeCall* call) -{ - switch (call->GetHelperNum()) - { - case CORINFO_HELP_CHKCASTARRAY: - case CORINFO_HELP_CHKCASTANY: - case CORINFO_HELP_CHKCASTINTERFACE: - case CORINFO_HELP_CHKCASTCLASS: - case CORINFO_HELP_ISINSTANCEOFARRAY: - case CORINFO_HELP_ISINSTANCEOFCLASS: - case CORINFO_HELP_ISINSTANCEOFANY: - case CORINFO_HELP_ISINSTANCEOFINTERFACE: - { - CallArg* castClsCallArg = call->gtArgs.GetUserArgByIndex(0); - CallArg* castObjCallArg = call->gtArgs.GetUserArgByIndex(1); - GenTree* castClsArg = castClsCallArg->GetNode(); - GenTree* castObjArg = castObjCallArg->GetNode(); - - // If object has the same VN as the cast, then the cast is effectively a no-op. - // - if (castObjArg->gtVNPair == call->gtVNPair) - { - // if castObjArg is not simple, we replace the arg with a temp assignment and - // continue using that temp - it allows us reliably extract all side effects - castObjArg = fgMakeMultiUse(&castObjCallArg->NodeRef()); - return gtWrapWithSideEffects(castObjArg, call, GTF_ALL_EFFECT, true); - } + // Note CORINFO_HELP_READYTORUN_ISINSTANCEOF does not have the same argument pattern. + // In particular, it is not possible to deduce what class is being tested from its args. + // + // Also note The CASTCLASS helpers won't appear in predicates as they throw on failure. + // So the helper list here is smaller than the one in optAssertionProp_Call. + // + CorInfoHelpFunc helper = eeGetHelperNum(call->gtCallMethHnd); + if ((helper == CORINFO_HELP_ISINSTANCEOFINTERFACE) || (helper == CORINFO_HELP_ISINSTANCEOFARRAY) || + (helper == CORINFO_HELP_ISINSTANCEOFCLASS) || (helper == CORINFO_HELP_ISINSTANCEOFANY)) + { + GenTree* objectNode = call->gtArgs.GetUserArgByIndex(1)->GetNode(); + GenTree* methodTableNode = call->gtArgs.GetUserArgByIndex(0)->GetNode(); + + // objectNode can be TYP_I_IMPL in case if it's a constant handle + // (e.g. a string literal from frozen segments) + // + assert(objectNode->TypeIs(TYP_REF, TYP_I_IMPL)); + assert(methodTableNode->TypeIs(TYP_I_IMPL)); + + ValueNum objVN = optConservativeNormalVN(objectNode); + ValueNum typeHndVN = optConservativeNormalVN(methodTableNode); + + if ((objVN != ValueNumStore::NoVN) && vnStore->IsVNTypeHandle(typeHndVN)) + { + AssertionDsc dsc = AssertionDsc::CreateSubtype(this, objVN, typeHndVN, /*exact*/ false); + AssertionIndex index = optAddAssertion(dsc); - // Let's see if gtGetClassHandle may help us to fold the cast (since VNForCast did not). - if (castClsArg->IsIconHandle(GTF_ICON_CLASS_HDL)) + // We don't need to create a complementary assertion here. We're only interested + // in the assertion that the object is of a certain type. The opposite assertion + // (that the object is not of a certain type) is not useful (at least not yet). + // + // So if we have "if (ISINST(obj, pMT) == null) then create the assertion for the "else" edge. + // + if (relop->OperIs(GT_EQ)) { - bool isExact; - bool isNonNull; - CORINFO_CLASS_HANDLE castFrom = gtGetClassHandle(castObjArg, &isExact, &isNonNull); - if (castFrom != NO_CLASS_HANDLE) - { - CORINFO_CLASS_HANDLE castTo = gtGetHelperArgClassHandle(castClsArg); - // Constant prop may fail to propagate compile time class handles, so verify we have - // a handle before invoking the runtime. - if ((castTo != NO_CLASS_HANDLE) && - info.compCompHnd->compareTypesForCast(castFrom, castTo) == TypeCompareState::Must) - { - // if castObjArg is not simple, we replace the arg with a temp assignment and - // continue using that temp - it allows us reliably extract all side effects - castObjArg = fgMakeMultiUse(&castObjCallArg->NodeRef()); - return gtWrapWithSideEffects(castObjArg, call, GTF_ALL_EFFECT, true); - } - } + return AssertionInfo::ForNextEdge(index); } + return index; } - break; - - default: - break; - } - - if (call->IsSpecialIntrinsic(this, NI_System_SpanHelpers_Memmove) || call->IsHelperCall(this, CORINFO_HELP_MEMCPY)) - { - return optVNBasedFoldExpr_Call_Memmove(call); } - if (call->IsSpecialIntrinsic(this, NI_System_SpanHelpers_Fill)) - { - return optVNBasedFoldExpr_Call_Memset(call); - } - - return nullptr; + return NO_ASSERTION_INDEX; } -//------------------------------------------------------------------------------ -// optVNBasedFoldExpr: Folds given tree using VN to a constant or a simpler tree. -// -// Arguments: -// block - The block containing the tree. -// parent - The parent node of the tree. -// tree - The tree to fold. -// -// Return Value: -// Returns a new tree or nullptr if nothing is changed. -// -GenTree* Compiler::optVNBasedFoldExpr(BasicBlock* block, GenTree* parent, GenTree* tree) +/***************************************************************************** + * + * If this node creates an assertion then assign an index to the assertion + * by adding it to the lookup table, if necessary. + */ +void Compiler::optAssertionGen(GenTree* tree) { - // First, attempt to fold it to a constant if possible. - GenTree* foldedToCns = optVNBasedFoldConstExpr(block, parent, tree); - if (foldedToCns != nullptr) - { - return foldedToCns; - } - - switch (tree->OperGet()) - { - case GT_CALL: - return optVNBasedFoldExpr_Call(block, parent, tree->AsCall()); - - // We can add more VN-based foldings here. - - default: - break; - } - return nullptr; -} + tree->ClearAssertion(); -//------------------------------------------------------------------------------ -// optVNBasedFoldConstExpr: Substitutes tree with an evaluated constant while -// managing side-effects. -// -// Arguments: -// block - The block containing the tree. -// parent - The parent node of the tree. -// tree - The tree node whose value is known at compile time. -// The tree should have a constant value number. -// -// Return Value: -// Returns a potentially new or a transformed tree node. -// Returns nullptr when no transformation is possible. -// -// Description: -// Transforms a tree node if its result evaluates to a constant. The -// transformation can be a "ChangeOper" to a constant or a new constant node -// with extracted side-effects. -// -// Before replacing or substituting the "tree" with a constant, extracts any -// side effects from the "tree" and creates a comma separated side effect list -// and then appends the transformed node at the end of the list. -// This comma separated list is then returned. -// -// For JTrue nodes, side effects are not put into a comma separated list. If -// the relop will evaluate to "true" or "false" statically, then the side-effects -// will be put into new statements, presuming the JTrue will be folded away. -// -GenTree* Compiler::optVNBasedFoldConstExpr(BasicBlock* block, GenTree* parent, GenTree* tree) -{ - if (tree->OperIs(GT_JTRUE)) - { - // Treat JTRUE separately to extract side effects into respective statements rather - // than using a COMMA separated op1. - return optVNConstantPropOnJTrue(block, tree); - } - // If relop is part of JTRUE, this should be optimized as part of the parent JTRUE. - // Or if relop is part of QMARK or anything else, we simply bail here. - else if (tree->OperIsCompare() && (tree->gtFlags & GTF_RELOP_JMP_USED)) + // If there are QMARKs in the IR, we won't generate assertions + // for conditionally executed code. + // + if (optLocalAssertionProp && ((tree->gtFlags & GTF_COLON_COND) != 0)) { - return nullptr; + return; } - // We want to use the Normal ValueNumber when checking for constants. - ValueNumPair vnPair = tree->gtVNPair; - ValueNum vnCns = vnStore->VNConservativeNormalValue(vnPair); - - // Check if node evaluates to a constant - if (!vnStore->IsVNConstant(vnCns)) - { - // Last chance - propagate VNF_PtrToLoc(lcl, offset) as GT_LCL_ADDR node - VNFuncApp funcApp; - if (((tree->gtFlags & GTF_SIDE_EFFECT) == 0) && vnStore->GetVNFunc(vnCns, &funcApp) && - (funcApp.m_func == VNF_PtrToLoc)) - { - unsigned lcl = (unsigned)vnStore->CoercedConstantValue(funcApp.m_args[0]); - unsigned offs = (unsigned)vnStore->CoercedConstantValue(funcApp.m_args[1]); - return gtNewLclAddrNode(lcl, offs, tree->TypeGet()); - } - - return nullptr; - } +#ifdef DEBUG + optAssertionPropCurrentTree = tree; +#endif - GenTree* conValTree = nullptr; - switch (vnStore->TypeOfVN(vnCns)) + AssertionInfo assertionInfo; + switch (tree->OperGet()) { - case TYP_FLOAT: - { - float value = vnStore->ConstantValue(vnCns); - - if (tree->TypeIs(TYP_INT)) - { - // Same sized reinterpretation of bits to integer - conValTree = gtNewIconNode(*(reinterpret_cast(&value))); - } - else + case GT_STORE_LCL_VAR: + // VN takes care of non local assertions for data flow. + if (optLocalAssertionProp) { - // Implicit conversion to float or double - assert(varTypeIsFloating(tree->TypeGet())); - conValTree = gtNewDconNode(FloatingPointUtils::convertToDouble(value), tree->TypeGet()); + assertionInfo = optCreateAssertion(tree, tree->AsLclVar()->Data(), /*equals*/ true); } break; - } - - case TYP_DOUBLE: - { - double value = vnStore->ConstantValue(vnCns); - if (tree->TypeIs(TYP_LONG)) - { - conValTree = gtNewLconNode(*(reinterpret_cast(&value))); - } - else + case GT_IND: + case GT_XAND: + case GT_XORR: + case GT_XADD: + case GT_XCHG: + case GT_CMPXCHG: + case GT_BLK: + case GT_STOREIND: + case GT_STORE_BLK: + case GT_NULLCHECK: + case GT_ARR_LENGTH: + case GT_MDARR_LENGTH: + case GT_MDARR_LOWER_BOUND: + // These indirs (esp. GT_IND and GT_STOREIND) are the most popular sources of assertions. + if (tree->IndirMayFault(this)) { - // Implicit conversion to float or double - assert(varTypeIsFloating(tree->TypeGet())); - conValTree = gtNewDconNode(value, tree->TypeGet()); + assertionInfo = optCreateAssertion(tree->GetIndirOrArrMetaDataAddr(), nullptr, /*equals*/ false); } break; - } - - case TYP_LONG: - { - INT64 value = vnStore->ConstantValue(vnCns); -#ifdef TARGET_64BIT - if (vnStore->IsVNHandle(vnCns)) - { - // Don't perform constant folding that involves a handle that needs - // to be recorded as a relocation with the VM. - if (!opts.compReloc) - { - conValTree = gtNewIconHandleNode(value, vnStore->GetHandleFlags(vnCns)); - } - } - else -#endif + case GT_INTRINSIC: + if (tree->AsIntrinsic()->gtIntrinsicName == NI_System_Object_GetType) { - switch (tree->TypeGet()) - { - case TYP_INT: - // Implicit conversion to smaller integer - conValTree = gtNewIconNode(static_cast(value)); - break; - - case TYP_LONG: - // Same type no conversion required - conValTree = gtNewLconNode(value); - break; - - case TYP_FLOAT: - // No implicit conversions from long to float and value numbering will - // not propagate through memory reinterpretations of different size. - unreached(); - break; - - case TYP_DOUBLE: - // Same sized reinterpretation of bits to double - conValTree = gtNewDconNodeD(*(reinterpret_cast(&value))); - break; - - default: - // Do not support such optimization. - break; - } + assertionInfo = optCreateAssertion(tree->AsIntrinsic()->gtGetOp1(), nullptr, /*equals*/ false); } - } - break; + break; - case TYP_REF: - { - if (tree->TypeIs(TYP_REF)) + case GT_BOUNDS_CHECK: + if (!optLocalAssertionProp) { - const size_t value = vnStore->ConstantValue(vnCns); - if (value == 0) + ValueNum idxVN = optConservativeNormalVN(tree->AsBoundsChk()->GetIndex()); + ValueNum lenVN = optConservativeNormalVN(tree->AsBoundsChk()->GetArrayLength()); + if ((idxVN == ValueNumStore::NoVN) || (lenVN == ValueNumStore::NoVN)) { - conValTree = gtNewNull(); + assertionInfo = NO_ASSERTION_INDEX; } else { - assert(vnStore->IsVNObjHandle(vnCns)); - conValTree = gtNewIconHandleNode(value, GTF_ICON_OBJ_HDL); + // GT_BOUNDS_CHECK node provides the following contract: + // * idxVN < lenVN + // * lenVN is non-negative + assertionInfo = optAddAssertion(AssertionDsc::CreateNoThrowArrBnd(idxVN, lenVN)); } } - } - break; + break; + + case GT_ARR_ELEM: + // An array element reference can create a non-null assertion + assertionInfo = optCreateAssertion(tree->AsArrElem()->gtArrObj, nullptr, /*equals*/ false); + break; - case TYP_INT: + case GT_CALL: { - int value = vnStore->ConstantValue(vnCns); -#ifndef TARGET_64BIT - if (vnStore->IsVNHandle(vnCns)) - { - // Don't perform constant folding that involves a handle that needs - // to be recorded as a relocation with the VM. - if (!opts.compReloc) - { - conValTree = gtNewIconHandleNode(value, vnStore->GetHandleFlags(vnCns)); - } - } - else -#endif + // A virtual call can create a non-null assertion. We transform some virtual calls into non-virtual calls + // with a GTF_CALL_NULLCHECK flag set. + // Ignore tail calls because they have 'this` pointer in the regular arg list and an implicit null check. + GenTreeCall* const call = tree->AsCall(); + if (call->NeedsNullCheck() || (call->IsVirtual() && !call->IsTailCall())) { - switch (tree->TypeGet()) - { - case TYP_REF: - case TYP_INT: - // Same type no conversion required - conValTree = gtNewIconNode(value); - break; - - case TYP_LONG: - // Implicit conversion to larger integer - conValTree = gtNewLconNode(value); - break; - - case TYP_FLOAT: - // Same sized reinterpretation of bits to float - conValTree = gtNewDconNodeF(BitOperations::UInt32BitsToSingle((uint32_t)value)); - break; - - case TYP_DOUBLE: - // No implicit conversions from int to double and value numbering will - // not propagate through memory reinterpretations of different size. - unreached(); - break; - - case TYP_BYTE: - case TYP_UBYTE: - case TYP_SHORT: - case TYP_USHORT: - assert(FitsIn(tree->TypeGet(), value)); - conValTree = gtNewIconNode(value); - break; - - default: - // Do not support (e.g. byref(const int)). - break; - } + // Retrieve the 'this' arg. + GenTree* thisArg = call->gtArgs.GetThisArg()->GetNode(); + assert(thisArg != nullptr); + assertionInfo = optCreateAssertion(thisArg, nullptr, /*equals*/ false); } } break; -#if defined(FEATURE_SIMD) - case TYP_SIMD8: - { - simd8_t value = vnStore->ConstantValue(vnCns); - - GenTreeVecCon* vecCon = gtNewVconNode(tree->TypeGet()); - memcpy(&vecCon->gtSimdVal, &value, sizeof(simd8_t)); - - conValTree = vecCon; - break; - } - - case TYP_SIMD12: - { - simd12_t value = vnStore->ConstantValue(vnCns); - - GenTreeVecCon* vecCon = gtNewVconNode(tree->TypeGet()); - memcpy(&vecCon->gtSimdVal, &value, sizeof(simd12_t)); - - conValTree = vecCon; - break; - } - - case TYP_SIMD16: - { - simd16_t value = vnStore->ConstantValue(vnCns); - - GenTreeVecCon* vecCon = gtNewVconNode(tree->TypeGet()); - memcpy(&vecCon->gtSimdVal, &value, sizeof(simd16_t)); - - conValTree = vecCon; + case GT_JTRUE: + assertionInfo = optAssertionGenJtrue(tree); break; - } - -#if defined(TARGET_XARCH) - case TYP_SIMD32: - { - simd32_t value = vnStore->ConstantValue(vnCns); - - GenTreeVecCon* vecCon = gtNewVconNode(tree->TypeGet()); - memcpy(&vecCon->gtSimdVal, &value, sizeof(simd32_t)); - conValTree = vecCon; + default: + // All other gtOper node kinds, leave 'assertionIndex' = NO_ASSERTION_INDEX break; - } - - case TYP_SIMD64: - { - simd64_t value = vnStore->ConstantValue(vnCns); - - GenTreeVecCon* vecCon = gtNewVconNode(tree->TypeGet()); - memcpy(&vecCon->gtSimdVal, &value, sizeof(simd64_t)); + } - conValTree = vecCon; - break; - } - break; + if (assertionInfo.HasAssertion()) + { + tree->SetAssertionInfo(assertionInfo); + } +} -#endif // TARGET_XARCH -#endif // FEATURE_SIMD +/***************************************************************************** + * + * Maps a complementary assertion to its original assertion so it can be + * retrieved faster. + */ +void Compiler::optMapComplementary(AssertionIndex assertionIndex, AssertionIndex index) +{ + if (assertionIndex == NO_ASSERTION_INDEX || index == NO_ASSERTION_INDEX) + { + return; + } -#if defined(FEATURE_MASKED_HW_INTRINSICS) - case TYP_MASK: - { - simdmask_t value = vnStore->ConstantValue(vnCns); + assert(assertionIndex <= optMaxAssertionCount); + assert(index <= optMaxAssertionCount); - GenTreeMskCon* mskCon = gtNewMskConNode(tree->TypeGet()); - memcpy(&mskCon->gtSimdMaskVal, &value, sizeof(simdmask_t)); + optComplementaryAssertionMap[assertionIndex] = index; + optComplementaryAssertionMap[index] = assertionIndex; +} - conValTree = mskCon; - break; - } - break; -#endif // FEATURE_MASKED_HW_INTRINSICS +/***************************************************************************** + * + * Given an assertion index, return the assertion index of the complementary + * assertion or 0 if one does not exist. + */ +AssertionIndex Compiler::optFindComplementary(AssertionIndex assertIndex) +{ + if (assertIndex == NO_ASSERTION_INDEX) + { + return NO_ASSERTION_INDEX; + } + const AssertionDsc& inputAssertion = optGetAssertion(assertIndex); - case TYP_BYREF: - // Do not support const byref optimization. - break; + // Must be an equal or not equal assertion. + if (!AssertionDsc::IsReversible(inputAssertion.GetKind())) + { + return NO_ASSERTION_INDEX; + } - default: - // We do not record constants of other types. - unreached(); - break; + AssertionIndex index = optComplementaryAssertionMap[assertIndex]; + if (index != NO_ASSERTION_INDEX && index <= optAssertionCount) + { + return index; } - if (conValTree != nullptr) + for (AssertionIndex index = 1; index <= optAssertionCount; ++index) { - if (!optIsProfitableToSubstitute(tree, block, parent, conValTree)) + // Make sure assertion kinds are complementary and op1, op2 kinds match. + const AssertionDsc& curAssertion = optGetAssertion(index); + if (curAssertion.Complementary(inputAssertion, !optLocalAssertionProp)) { - // Not profitable to substitute - return nullptr; + optMapComplementary(assertIndex, index); + return index; } - - // Were able to optimize. - conValTree->gtVNPair = vnPair; - return gtWrapWithSideEffects(conValTree, tree, GTF_SIDE_EFFECT, true); - } - else - { - // Was not able to optimize. - return nullptr; } + return NO_ASSERTION_INDEX; } -//------------------------------------------------------------------------------ -// optIsProfitableToSubstitute: Checks if value worth substituting to dest +//------------------------------------------------------------------------ +// optAssertionIsSubrange: Find a subrange assertion for the given range and tree. +// +// This function will return the index of the first assertion in "assertions" +// which claims that the value of "tree" is within the bounds of the provided +// "range" (i. e. "range.Contains(assertedRange)"). // // Arguments: -// dest - destination to substitute value to -// destBlock - Basic block of destination -// destParent - Parent of destination -// value - value we plan to substitute +// tree - the tree for which to find the assertion +// range - range the subrange of which to look for +// assertions - the set of assertions // -// Returns: -// False if it's likely not profitable to do substitution, True otherwise +// Return Value: +// Index of the found assertion, NO_ASSERTION_INDEX otherwise. // -bool Compiler::optIsProfitableToSubstitute(GenTree* dest, BasicBlock* destBlock, GenTree* destParent, GenTree* value) +AssertionIndex Compiler::optAssertionIsSubrange(GenTree* tree, IntegralRange range, ASSERT_VALARG_TP assertions) { - // Giving up on these kinds of handles demonstrated size improvements - if (value->IsIconHandle(GTF_ICON_STATIC_HDL, GTF_ICON_CLASS_HDL)) - { - return false; - } - - // A simple heuristic: If the constant is defined outside of a loop (not far from its head) - // and is used inside it - don't propagate. - // - // TODO: Extend on more kinds of trees - - if (!dest->OperIs(GT_LCL_VAR)) + assert(optLocalAssertionProp); // Subrange assertions are local only. + if (!optCanPropSubRange) { - return true; + return NO_ASSERTION_INDEX; } - const GenTreeLclVar* lcl = dest->AsLclVar(); - - if (value->IsCnsVec()) + BitVecOps::Iter iter(apTraits, assertions); + unsigned bvIndex = 0; + while (iter.NextElem(&bvIndex)) { -#if defined(FEATURE_HW_INTRINSICS) - // Many hwintrinsics can't benefit from constant prop because they don't support - // constant folding nor do they support any specialized encodings. So, we want to - // skip constant prop and preserve any user-defined locals in that scenario. - // - // However, if the local is only referenced once then we want to allow propagation - // regardless since we can then contain the only actual usage and save a needless - // instruction. - // - // To determine number of uses, we prefer checking SSA first since it is more exact - // and can account for patterns where a local is reassigned later. However, if we - // can't find an SSA then we fallback to the naive ref count of the local, noting - // that we need to check for greater than 2 since it includes both the def and use. - - bool inspectIntrinsic = false; - - if ((destParent != nullptr) && destParent->OperIsHWIntrinsic()) + AssertionIndex const index = GetAssertionIndex(bvIndex); + const AssertionDsc& curAssertion = optGetAssertion(index); + if (curAssertion.CanPropSubRange()) { - LclVarDsc* varDsc = lvaGetDesc(lcl); - - if (lcl->HasSsaName()) - { - inspectIntrinsic = varDsc->GetPerSsaData(lcl->GetSsaNum())->GetNumUses() > 1; - } - else + if (curAssertion.GetOp1().GetLclNum() != tree->AsLclVarCommon()->GetLclNum()) { - inspectIntrinsic = varDsc->lvRefCnt() > 2; + continue; } - } - - if (inspectIntrinsic) - { - GenTreeHWIntrinsic* parent = destParent->AsHWIntrinsic(); - NamedIntrinsic intrinsicId = parent->GetHWIntrinsicId(); - if (!HWIntrinsicInfo::CanBenefitFromConstantProp(intrinsicId)) + if (range.Contains(curAssertion.GetOp2().GetIntegralRange())) { - return false; + return index; } - - // For several of the scenarios we may skip the costing logic - // since we know that the operand is always containable and therefore - // is always cost effective to propagate. - - return parent->ShouldConstantProp(dest, value->AsVecCon()); } -#endif // FEATURE_HW_INTRINSICS - } - else if (!value->IsCnsFltOrDbl() && !value->IsCnsMsk()) - { - return true; } - gtPrepareCost(value); + return NO_ASSERTION_INDEX; +} - if ((value->GetCostEx() > 1) && (value->GetCostSz() > 1)) +/********************************************************************************** + * + * Given a "tree" that is usually arg1 of a isinst/cast kind of GT_CALL (a class + * handle), and "methodTableArg" which is a const int (a class handle), then search + * if there is an assertion in "assertions", that asserts the equality of the two + * class handles and then returns the index of the assertion. If one such assertion + * could not be found, then it returns NO_ASSERTION_INDEX. + * + */ +AssertionIndex Compiler::optAssertionIsSubtype(GenTree* tree, GenTree* methodTableArg, ASSERT_VALARG_TP assertions) +{ + BitVecOps::Iter iter(apTraits, assertions); + unsigned bvIndex = 0; + while (iter.NextElem(&bvIndex)) { - // Try to find the block this constant was originally defined in - if (lcl->HasSsaName()) + AssertionIndex const index = GetAssertionIndex(bvIndex); + const AssertionDsc& curAssertion = optGetAssertion(index); + if (!curAssertion.KindIs(OAK_EQUAL) || !curAssertion.GetOp1().KindIs(O1K_SUBTYPE, O1K_EXACT_TYPE)) { - BasicBlock* defBlock = lvaGetDesc(lcl)->GetPerSsaData(lcl->GetSsaNum())->GetBlock(); - if (defBlock != nullptr) - { - // Avoid propagating if the weighted use cost is significantly greater than the def cost. - // NOTE: this currently does not take "a float living across a call" case into account - // where we might end up with spill/restore on ABIs without callee-saved registers - const weight_t defBlockWeight = defBlock->getBBWeight(this); - const weight_t lclblockWeight = destBlock->getBBWeight(this); + // TODO-CQ: We might benefit from OAK_NOT_EQUAL assertion as well, e.g.: + // if (obj is not MyClass) // obj is known to be never of MyClass class + // { + // if (obj is MyClass) // can be folded to false + // { + // + continue; + } - if ((defBlockWeight > 0) && ((lclblockWeight / defBlockWeight) >= BB_LOOP_WEIGHT_SCALE)) - { - JITDUMP("Constant propagation inside loop " FMT_BB " is not profitable\n", destBlock->bbNum); - return false; - } - } + if ((curAssertion.GetOp1().GetVN() != vnStore->VNConservativeNormalValue(tree->gtVNPair) || + !curAssertion.GetOp2().KindIs(O2K_CONST_INT))) + { + continue; + } + + ssize_t methodTableVal = 0; + GenTreeFlags iconFlags = GTF_EMPTY; + if (!optIsTreeKnownIntValue(!optLocalAssertionProp, methodTableArg, &methodTableVal, &iconFlags)) + { + continue; + } + + if (curAssertion.GetOp2().GetIntConstant() == methodTableVal) + { + // TODO-CQ: if they don't match, we might still be able to prove that the result is foldable via + // compareTypesForCast. + return index; } } - return true; + return NO_ASSERTION_INDEX; } + //------------------------------------------------------------------------------ // optConstantAssertionProp: Possibly substitute a constant for a local use // @@ -5808,302 +4505,6 @@ ASSERT_TP* Compiler::optInitAssertionDataflowFlags() return jumpDestOut; } -// Callback data for the VN based constant prop visitor. -struct VNAssertionPropVisitorInfo -{ - Compiler* m_compiler; - Statement* stmt; - BasicBlock* block; - VNAssertionPropVisitorInfo(Compiler* pThis, BasicBlock* block, Statement* stmt) - : m_compiler(pThis) - , stmt(stmt) - , block(block) - { - } -}; - -//------------------------------------------------------------------------------ -// optVNConstantPropOnJTrue -// Constant propagate on the JTrue node. -// -// Arguments: -// block - The block that contains the JTrue. -// test - The JTrue node whose relop evaluates to 0 or non-zero value. -// -// Return Value: -// nullptr if no constant propagation is done, else the modified JTrue node -// containing "0==0" or "0!=0" relop node -// (where op1 is wrapped with side effects if any). -// -GenTree* Compiler::optVNConstantPropOnJTrue(BasicBlock* block, GenTree* test) -{ - GenTree* relop = test->gtGetOp1(); - - // VN based assertion non-null on this relop has been performed. - if (!relop->OperIsCompare()) - { - return nullptr; - } - - // - // Make sure GTF_RELOP_JMP_USED flag is set so that we can later skip constant - // prop'ing a JTRUE's relop child node for a second time in the pre-order - // tree walk. - // - assert((relop->gtFlags & GTF_RELOP_JMP_USED) != 0); - - // We want to use the Normal ValueNumber when checking for constants. - ValueNum vnCns = vnStore->VNConservativeNormalValue(relop->gtVNPair); - if (!vnStore->IsVNConstant(vnCns)) - { - return nullptr; - } - - GenTree* sideEffects = gtWrapWithSideEffects(gtNewNothingNode(), relop); - if (!sideEffects->IsNothingNode()) - { - // Insert side effects before the JTRUE stmt. - Statement* newStmt = fgNewStmtNearEnd(block, sideEffects); - fgMorphBlockStmt(block, newStmt DEBUGARG(__FUNCTION__)); - } - - // Let's maintain the invariant that JTRUE's operand is always a relop. - // and if we have side effects, we wrap one of the operands with them, not the relop. - const bool evalsToTrue = (vnStore->CoercedConstantValue(vnCns) != 0); - test->AsOp()->gtOp1 = gtNewOperNode(evalsToTrue ? GT_EQ : GT_NE, relop->TypeGet(), gtNewFalse(), gtNewFalse()); - return test; -} - -//------------------------------------------------------------------------------ -// optVNBasedFoldCurStmt: Performs VN-based folding -// on the current statement's tree nodes using VN. -// -// Assumption: -// This function is called as part of a post-order tree walk. -// -// Arguments: -// tree - The currently visited tree node. -// stmt - The statement node in which the "tree" is present. -// parent - The parent node of the tree. -// block - The block that contains the statement that contains the tree. -// -// Return Value: -// Returns the standard visitor walk result. -// -Compiler::fgWalkResult Compiler::optVNBasedFoldCurStmt(BasicBlock* block, - Statement* stmt, - GenTree* parent, - GenTree* tree) -{ - // Don't try and fold expressions marked with GTF_DONT_CSE - // TODO-ASG: delete. - if (!tree->CanCSE()) - { - return WALK_CONTINUE; - } - - // Don't propagate floating-point constants into a TYP_STRUCT LclVar - // This can occur for HFA return values (see hfa_sf3E_r.exe) - if (tree->TypeIs(TYP_STRUCT)) - { - return WALK_CONTINUE; - } - - switch (tree->OperGet()) - { - // Make sure we have an R-value. - case GT_ADD: - case GT_SUB: - case GT_DIV: - case GT_MOD: - case GT_UDIV: - case GT_UMOD: - case GT_EQ: - case GT_NE: - case GT_LT: - case GT_LE: - case GT_GE: - case GT_GT: - case GT_OR: - case GT_XOR: - case GT_AND: - case GT_LSH: - case GT_RSH: - case GT_RSZ: - case GT_NEG: - case GT_CAST: - case GT_BITCAST: - case GT_INTRINSIC: -#ifdef FEATURE_HW_INTRINSICS - case GT_HWINTRINSIC: -#endif // FEATURE_HW_INTRINSICS - case GT_ARR_LENGTH: - break; - - case GT_BLK: - case GT_IND: - { - const ValueNum vn = tree->GetVN(VNK_Conservative); - if (vnStore->VNNormalValue(vn) != vn) - { - return WALK_CONTINUE; - } - } - break; - - case GT_JTRUE: - break; - - case GT_MUL: - // Don't transform long multiplies. - if (tree->gtFlags & GTF_MUL_64RSLT) - { - return WALK_CONTINUE; - } - break; - - case GT_LCL_VAR: - case GT_LCL_FLD: - // Let's not conflict with CSE (to save the movw/movt). - if (lclNumIsCSE(tree->AsLclVarCommon()->GetLclNum())) - { - return WALK_CONTINUE; - } - break; - - case GT_CALL: - // The checks aren't for correctness, but to avoid unnecessary work. - if (!tree->AsCall()->IsPure(this) && !tree->AsCall()->IsSpecialIntrinsic()) - { - return WALK_CONTINUE; - } - break; - - default: - // Unknown node, continue to walk. - return WALK_CONTINUE; - } - - // Perform the VN-based folding: - GenTree* newTree = optVNBasedFoldExpr(block, parent, tree); - - if (newTree == nullptr) - { - // Not propagated, keep going. - return WALK_CONTINUE; - } - - optAssertionProp_Update(newTree, tree, stmt); - - JITDUMP("After VN-based fold of [%06u]:\n", tree->gtTreeID); - DBEXEC(VERBOSE, gtDispStmt(stmt)); - - return WALK_CONTINUE; -} - -//------------------------------------------------------------------------------ -// optVnNonNullPropCurStmt -// Performs VN based non-null propagation on the tree node. -// -// Assumption: -// This function is called as part of a pre-order tree walk. -// -// Arguments: -// block - The block that contains the statement that contains the tree. -// stmt - The statement node in which the "tree" is present. -// tree - The currently visited tree node. -// -// Return Value: -// None. -// -// Description: -// Performs value number based non-null propagation on GT_CALL and -// indirections. This is different from flow based assertions and helps -// unify VN based constant prop and non-null prop in a single pre-order walk. -// -void Compiler::optVnNonNullPropCurStmt(BasicBlock* block, Statement* stmt, GenTree* tree) -{ - ASSERT_TP empty = BitVecOps::UninitVal(); - GenTree* newTree = nullptr; - if (tree->OperIs(GT_CALL)) - { - newTree = optNonNullAssertionProp_Call(empty, tree->AsCall()); - } - else if (tree->OperIsIndir()) - { - newTree = optAssertionProp_Ind(empty, tree, stmt); - } - if (newTree) - { - assert(newTree == tree); - optAssertionProp_Update(newTree, tree, stmt); - } -} - -//------------------------------------------------------------------------------ -// optVNAssertionPropCurStmtVisitor -// Unified Value Numbering based assertion propagation visitor. -// -// Assumption: -// This function is called as part of a post-order tree walk. -// -// Return Value: -// WALK_RESULTs. -// -// Description: -// An unified value numbering based assertion prop visitor that -// performs non-null and constant assertion propagation based on -// value numbers. -// -/* static */ -Compiler::fgWalkResult Compiler::optVNAssertionPropCurStmtVisitor(GenTree** ppTree, fgWalkData* data) -{ - VNAssertionPropVisitorInfo* pData = (VNAssertionPropVisitorInfo*)data->pCallbackData; - Compiler* pThis = pData->m_compiler; - - pThis->optVnNonNullPropCurStmt(pData->block, pData->stmt, *ppTree); - - return pThis->optVNBasedFoldCurStmt(pData->block, pData->stmt, data->parent, *ppTree); -} - -/***************************************************************************** - * - * Perform VN based i.e., data flow based assertion prop first because - * even if we don't gen new control flow assertions, we still propagate - * these first. - * - * Returns the skipped next stmt if the current statement or next few - * statements got removed, else just returns the incoming stmt. - */ -Statement* Compiler::optVNAssertionPropCurStmt(BasicBlock* block, Statement* stmt) -{ - // TODO-Review: EH successor/predecessor iteration seems broken. - // See: SELF_HOST_TESTS_ARM\jit\Directed\ExcepFilters\fault\fault.exe - if (block->bbCatchTyp == BBCT_FAULT) - { - return stmt; - } - - // Preserve the prev link before the propagation and morph. - Statement* prev = (stmt == block->firstStmt()) ? nullptr : stmt->GetPrevStmt(); - - // Perform VN based assertion prop first, in case we don't find - // anything in assertion gen. - optAssertionPropagatedCurrentStmt = false; - - VNAssertionPropVisitorInfo data(this, block, stmt); - fgWalkTreePost(stmt->GetRootNodePointer(), Compiler::optVNAssertionPropCurStmtVisitor, &data); - - if (optAssertionPropagatedCurrentStmt) - { - fgMorphBlockStmt(block, stmt DEBUGARG("optVNAssertionPropCurStmt")); - } - - // Check if propagation removed statements starting from current stmt. - // If so, advance to the next good statement. - Statement* nextStmt = (prev == nullptr) ? block->firstStmt() : prev->GetNextStmt(); - return nextStmt; -} //------------------------------------------------------------------------------ // optAssertionPropMain: assertion propagation phase diff --git a/src/coreclr/jit/integralrange.cpp b/src/coreclr/jit/integralrange.cpp new file mode 100644 index 00000000000000..38bb61cbe61414 --- /dev/null +++ b/src/coreclr/jit/integralrange.cpp @@ -0,0 +1,596 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +/*XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +XX XX +XX IntegralRange XX +XX XX +XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +*/ + +#include "jitpch.h" +#include "rangecheck.h" +#ifdef _MSC_VER +#pragma hdrstop +#endif + +//------------------------------------------------------------------------ +// Contains: Whether the range contains a given integral value, inclusive. +// +// Arguments: +// value - the integral value in question +// +// Return Value: +// "true" if the value is within the range's bounds, "false" otherwise. +// +bool IntegralRange::Contains(int64_t value) const +{ + int64_t lowerBound = SymbolicToRealValue(m_lowerBound); + int64_t upperBound = SymbolicToRealValue(m_upperBound); + + return (lowerBound <= value) && (value <= upperBound); +} + +//------------------------------------------------------------------------ +// SymbolicToRealValue: Convert a symbolic value to a 64-bit signed integer. +// +// Arguments: +// value - the symbolic value in question +// +// Return Value: +// Integer corresponding to the symbolic value. +// +/* static */ int64_t IntegralRange::SymbolicToRealValue(SymbolicIntegerValue value) +{ + static const int64_t SymbolicToRealMap[]{ + INT64_MIN, // SymbolicIntegerValue::LongMin + INT32_MIN, // SymbolicIntegerValue::IntMin + INT16_MIN, // SymbolicIntegerValue::ShortMin + INT8_MIN, // SymbolicIntegerValue::ByteMin + 0, // SymbolicIntegerValue::Zero + 1, // SymbolicIntegerValue::One + INT8_MAX, // SymbolicIntegerValue::ByteMax + UINT8_MAX, // SymbolicIntegerValue::UByteMax + INT16_MAX, // SymbolicIntegerValue::ShortMax + UINT16_MAX, // SymbolicIntegerValue::UShortMax + CORINFO_Array_MaxLength, // SymbolicIntegerValue::ArrayLenMax + INT32_MAX, // SymbolicIntegerValue::IntMax + UINT32_MAX, // SymbolicIntegerValue::UIntMax + INT64_MAX // SymbolicIntegerValue::LongMax + }; + + assert(sizeof(SymbolicIntegerValue) == sizeof(int32_t)); + assert(SymbolicToRealMap[static_cast(SymbolicIntegerValue::LongMin)] == INT64_MIN); + assert(SymbolicToRealMap[static_cast(SymbolicIntegerValue::Zero)] == 0); + assert(SymbolicToRealMap[static_cast(SymbolicIntegerValue::LongMax)] == INT64_MAX); + + return SymbolicToRealMap[static_cast(value)]; +} + +//------------------------------------------------------------------------ +// LowerBoundForType: Get the symbolic lower bound for a type. +// +// Arguments: +// type - the integral type in question +// +// Return Value: +// Symbolic value representing the smallest possible value "type" can represent. +// +/* static */ SymbolicIntegerValue IntegralRange::LowerBoundForType(var_types type) +{ + switch (type) + { + case TYP_UBYTE: + case TYP_USHORT: + return SymbolicIntegerValue::Zero; + case TYP_BYTE: + return SymbolicIntegerValue::ByteMin; + case TYP_SHORT: + return SymbolicIntegerValue::ShortMin; + case TYP_INT: + return SymbolicIntegerValue::IntMin; + case TYP_LONG: + return SymbolicIntegerValue::LongMin; + default: + unreached(); + } +} + +//------------------------------------------------------------------------ +// UpperBoundForType: Get the symbolic upper bound for a type. +// +// Arguments: +// type - the integral type in question +// +// Return Value: +// Symbolic value representing the largest possible value "type" can represent. +// +/* static */ SymbolicIntegerValue IntegralRange::UpperBoundForType(var_types type) +{ + switch (type) + { + case TYP_BYTE: + return SymbolicIntegerValue::ByteMax; + case TYP_UBYTE: + return SymbolicIntegerValue::UByteMax; + case TYP_SHORT: + return SymbolicIntegerValue::ShortMax; + case TYP_USHORT: + return SymbolicIntegerValue::UShortMax; + case TYP_INT: + return SymbolicIntegerValue::IntMax; + case TYP_UINT: + return SymbolicIntegerValue::UIntMax; + case TYP_LONG: + return SymbolicIntegerValue::LongMax; + default: + unreached(); + } +} + +//------------------------------------------------------------------------ +// ForNode: Compute the integral range for a node. +// +// Arguments: +// node - the node, of an integral type, in question +// compiler - the Compiler, used to retrieve additional info +// +// Return Value: +// The integral range this node produces. +// +/* static */ IntegralRange IntegralRange::ForNode(GenTree* node, Compiler* compiler) +{ + assert(varTypeIsIntegral(node)); + + var_types rangeType = node->TypeGet(); + + switch (node->OperGet()) + { + case GT_EQ: + case GT_NE: + case GT_LT: + case GT_LE: + case GT_GE: + case GT_GT: + return {SymbolicIntegerValue::Zero, SymbolicIntegerValue::One}; + + case GT_AND: + { + IntegralRange leftRange = IntegralRange::ForNode(node->gtGetOp1(), compiler); + IntegralRange rightRange = IntegralRange::ForNode(node->gtGetOp2(), compiler); + if (leftRange.IsNonNegative() && rightRange.IsNonNegative()) + { + // If both sides are known to be non-negative, the result is non-negative. + // Further, the top end of the range cannot exceed the min of the two upper bounds. + return {SymbolicIntegerValue::Zero, min(leftRange.GetUpperBound(), rightRange.GetUpperBound())}; + } + + if (leftRange.IsNonNegative() || rightRange.IsNonNegative()) + { + // If only one side is known to be non-negative, however it is harder to + // reason about the upper bound. + return {SymbolicIntegerValue::Zero, UpperBoundForType(rangeType)}; + } + + break; + } + + case GT_ARR_LENGTH: + case GT_MDARR_LENGTH: + return {SymbolicIntegerValue::Zero, SymbolicIntegerValue::ArrayLenMax}; + + case GT_CALL: + if (node->AsCall()->NormalizesSmallTypesOnReturn()) + { + rangeType = static_cast(node->AsCall()->gtReturnType); + } + break; + + case GT_IND: + { + GenTree* const addr = node->AsIndir()->Addr(); + + if (node->TypeIs(TYP_INT) && addr->OperIs(GT_ADD) && addr->gtGetOp1()->OperIs(GT_LCL_VAR) && + addr->gtGetOp2()->IsIntegralConst(OFFSETOF__CORINFO_Span__length)) + { + GenTreeLclVar* const lclVar = addr->gtGetOp1()->AsLclVar(); + + if (compiler->lvaGetDesc(lclVar->GetLclNum())->IsSpan()) + { + assert(compiler->lvaIsImplicitByRefLocal(lclVar->GetLclNum())); + return {SymbolicIntegerValue::Zero, UpperBoundForType(rangeType)}; + } + } + break; + } + + case GT_LCL_FLD: + { + GenTreeLclFld* const lclFld = node->AsLclFld(); + LclVarDsc* const varDsc = compiler->lvaGetDesc(lclFld); + + if (node->TypeIs(TYP_INT) && varDsc->IsSpan() && lclFld->GetLclOffs() == OFFSETOF__CORINFO_Span__length) + { + return {SymbolicIntegerValue::Zero, UpperBoundForType(rangeType)}; + } + + break; + } + + case GT_LCL_VAR: + { + LclVarDsc* const varDsc = compiler->lvaGetDesc(node->AsLclVar()); + + if (varDsc->lvNormalizeOnStore()) + { + rangeType = compiler->lvaGetDesc(node->AsLclVar())->TypeGet(); + } + + if (varDsc->IsNeverNegative()) + { + return {SymbolicIntegerValue::Zero, UpperBoundForType(rangeType)}; + } + break; + } + + case GT_CNS_INT: + { + if (node->IsIntegralConst(0) || node->IsIntegralConst(1)) + { + return {SymbolicIntegerValue::Zero, SymbolicIntegerValue::One}; + } + + int64_t constValue = node->AsIntCon()->IntegralValue(); + if (constValue >= 0) + { + return {SymbolicIntegerValue::Zero, UpperBoundForType(rangeType)}; + } + + break; + } + + case GT_QMARK: + return Union(ForNode(node->AsQmark()->ThenNode(), compiler), + ForNode(node->AsQmark()->ElseNode(), compiler)); + + case GT_CAST: + return ForCastOutput(node->AsCast(), compiler); + +#if defined(FEATURE_HW_INTRINSICS) + case GT_HWINTRINSIC: + switch (node->AsHWIntrinsic()->GetHWIntrinsicId()) + { +#if defined(TARGET_XARCH) + case NI_Vector128_op_Equality: + case NI_Vector128_op_Inequality: + case NI_Vector256_op_Equality: + case NI_Vector256_op_Inequality: + case NI_Vector512_op_Equality: + case NI_Vector512_op_Inequality: + case NI_X86Base_CompareScalarOrderedEqual: + case NI_X86Base_CompareScalarOrderedNotEqual: + case NI_X86Base_CompareScalarOrderedLessThan: + case NI_X86Base_CompareScalarOrderedLessThanOrEqual: + case NI_X86Base_CompareScalarOrderedGreaterThan: + case NI_X86Base_CompareScalarOrderedGreaterThanOrEqual: + case NI_X86Base_CompareScalarUnorderedEqual: + case NI_X86Base_CompareScalarUnorderedNotEqual: + case NI_X86Base_CompareScalarUnorderedLessThanOrEqual: + case NI_X86Base_CompareScalarUnorderedLessThan: + case NI_X86Base_CompareScalarUnorderedGreaterThanOrEqual: + case NI_X86Base_CompareScalarUnorderedGreaterThan: + case NI_X86Base_TestC: + case NI_X86Base_TestZ: + case NI_X86Base_TestNotZAndNotC: + case NI_AVX_TestC: + case NI_AVX_TestZ: + case NI_AVX_TestNotZAndNotC: + return {SymbolicIntegerValue::Zero, SymbolicIntegerValue::One}; + + case NI_X86Base_Extract: + case NI_X86Base_X64_Extract: + case NI_Vector128_ToScalar: + case NI_Vector256_ToScalar: + case NI_Vector512_ToScalar: + case NI_Vector128_GetElement: + case NI_Vector256_GetElement: + case NI_Vector512_GetElement: + if (varTypeIsSmall(node->AsHWIntrinsic()->GetSimdBaseType())) + { + return ForType(node->AsHWIntrinsic()->GetSimdBaseType()); + } + break; + + case NI_AVX2_LeadingZeroCount: + case NI_AVX2_TrailingZeroCount: + case NI_AVX2_X64_LeadingZeroCount: + case NI_AVX2_X64_TrailingZeroCount: + case NI_X86Base_PopCount: + case NI_X86Base_X64_PopCount: + // Note: No advantage in using a precise range for IntegralRange. + // Example: IntCns = 42 gives [0..127] with a non -precise range, [42,42] with a precise range. + return {SymbolicIntegerValue::Zero, SymbolicIntegerValue::ByteMax}; +#elif defined(TARGET_ARM64) + case NI_Vector64_op_Equality: + case NI_Vector64_op_Inequality: + case NI_Vector128_op_Equality: + case NI_Vector128_op_Inequality: + return {SymbolicIntegerValue::Zero, SymbolicIntegerValue::One}; + + case NI_AdvSimd_Extract: + case NI_Vector64_ToScalar: + case NI_Vector128_ToScalar: + case NI_Vector64_GetElement: + case NI_Vector128_GetElement: + if (varTypeIsSmall(node->AsHWIntrinsic()->GetSimdBaseType())) + { + return ForType(node->AsHWIntrinsic()->GetSimdBaseType()); + } + break; + + case NI_AdvSimd_PopCount: + case NI_AdvSimd_LeadingZeroCount: + case NI_AdvSimd_LeadingSignCount: + case NI_ArmBase_LeadingZeroCount: + case NI_ArmBase_Arm64_LeadingZeroCount: + case NI_ArmBase_Arm64_LeadingSignCount: + // Note: No advantage in using a precise range for IntegralRange. + // Example: IntCns = 42 gives [0..127] with a non -precise range, [42,42] with a precise range. + return {SymbolicIntegerValue::Zero, SymbolicIntegerValue::ByteMax}; +#else +#error Unsupported platform +#endif + default: + break; + } + break; +#endif // defined(FEATURE_HW_INTRINSICS) + + default: + break; + } + + return ForType(rangeType); +} + +//------------------------------------------------------------------------ +// ForCastInput: Get the non-overflowing input range for a cast. +// +// This routine computes the input range for a cast from +// an integer to an integer for which it will not overflow. +// See also the specification comment for IntegralRange. +// +// Arguments: +// cast - the cast node for which the range will be computed +// +// Return Value: +// The range this cast consumes without overflowing - see description. +// +/* static */ IntegralRange IntegralRange::ForCastInput(GenTreeCast* cast) +{ + var_types fromType = genActualType(cast->CastOp()); + var_types toType = cast->CastToType(); + bool fromUnsigned = cast->IsUnsigned(); + + assert((fromType == TYP_INT) || (fromType == TYP_LONG) || varTypeIsGC(fromType)); + assert(varTypeIsIntegral(toType)); + + // Cast from a GC type is the same as a cast from TYP_I_IMPL for our purposes. + if (varTypeIsGC(fromType)) + { + fromType = TYP_I_IMPL; + } + + if (!cast->gtOverflow()) + { + // CAST(small type <- uint/int/ulong/long) - [TO_TYPE_MIN..TO_TYPE_MAX] + if (varTypeIsSmall(toType)) + { + return {LowerBoundForType(toType), UpperBoundForType(toType)}; + } + + // We choose to say here that representation-changing casts never overflow. + // It does not really matter what we do here because representation-changing + // non-overflowing casts cannot be deleted from the IR in any case. + // CAST(uint/int <- uint/int) - [INT_MIN..INT_MAX] + // CAST(uint/int <- ulong/long) - [LONG_MIN..LONG_MAX] + // CAST(ulong/long <- uint/int) - [INT_MIN..INT_MAX] + // CAST(ulong/long <- ulong/long) - [LONG_MIN..LONG_MAX] + return ForType(fromType); + } + + SymbolicIntegerValue lowerBound; + SymbolicIntegerValue upperBound; + + // CAST_OVF(small type <- int/long) - [TO_TYPE_MIN..TO_TYPE_MAX] + // CAST_OVF(small type <- uint/ulong) - [0..TO_TYPE_MAX] + if (varTypeIsSmall(toType)) + { + lowerBound = fromUnsigned ? SymbolicIntegerValue::Zero : LowerBoundForType(toType); + upperBound = UpperBoundForType(toType); + } + else + { + switch (toType) + { + // CAST_OVF(uint <- uint) - [INT_MIN..INT_MAX] + // CAST_OVF(uint <- int) - [0..INT_MAX] + // CAST_OVF(uint <- ulong/long) - [0..UINT_MAX] + case TYP_UINT: + if (fromType == TYP_LONG) + { + lowerBound = SymbolicIntegerValue::Zero; + upperBound = SymbolicIntegerValue::UIntMax; + } + else + { + lowerBound = fromUnsigned ? SymbolicIntegerValue::IntMin : SymbolicIntegerValue::Zero; + upperBound = SymbolicIntegerValue::IntMax; + } + break; + + // CAST_OVF(int <- uint/ulong) - [0..INT_MAX] + // CAST_OVF(int <- int/long) - [INT_MIN..INT_MAX] + case TYP_INT: + lowerBound = fromUnsigned ? SymbolicIntegerValue::Zero : SymbolicIntegerValue::IntMin; + upperBound = SymbolicIntegerValue::IntMax; + break; + + // CAST_OVF(ulong <- uint) - [INT_MIN..INT_MAX] + // CAST_OVF(ulong <- int) - [0..INT_MAX] + // CAST_OVF(ulong <- ulong) - [LONG_MIN..LONG_MAX] + // CAST_OVF(ulong <- long) - [0..LONG_MAX] + case TYP_ULONG: + lowerBound = fromUnsigned ? LowerBoundForType(fromType) : SymbolicIntegerValue::Zero; + upperBound = UpperBoundForType(fromType); + break; + + // CAST_OVF(long <- uint/int) - [INT_MIN..INT_MAX] + // CAST_OVF(long <- ulong) - [0..LONG_MAX] + // CAST_OVF(long <- long) - [LONG_MIN..LONG_MAX] + case TYP_LONG: + if (fromUnsigned && (fromType == TYP_LONG)) + { + lowerBound = SymbolicIntegerValue::Zero; + } + else + { + lowerBound = LowerBoundForType(fromType); + } + upperBound = UpperBoundForType(fromType); + break; + + default: + unreached(); + } + } + + return {lowerBound, upperBound}; +} + +//------------------------------------------------------------------------ +// ForCastOutput: Get the output range for a cast. +// +// This method is the "output" counterpart to ForCastInput, it returns +// a range produced by a cast (by definition, non-overflowing one). +// The output range is the same for representation-preserving casts, but +// can be different for others. One example is CAST_OVF(uint <- long). +// The input range is [0..UINT_MAX], while the output is [INT_MIN..INT_MAX]. +// Unlike ForCastInput, this method supports casts from floating point types. +// +// Arguments: +// cast - the cast node for which the range will be computed +// compiler - Compiler object +// +// Return Value: +// The range this cast produces - see description. +// +/* static */ IntegralRange IntegralRange::ForCastOutput(GenTreeCast* cast, Compiler* compiler) +{ + var_types fromType = genActualType(cast->CastOp()); + var_types toType = cast->CastToType(); + bool fromUnsigned = cast->IsUnsigned(); + + assert((fromType == TYP_INT) || (fromType == TYP_LONG) || varTypeIsFloating(fromType) || varTypeIsGC(fromType)); + assert(varTypeIsIntegral(toType)); + + // CAST/CAST_OVF(small type <- float/double) - [TO_TYPE_MIN..TO_TYPE_MAX] + // CAST/CAST_OVF(uint/int <- float/double) - [INT_MIN..INT_MAX] + // CAST/CAST_OVF(ulong/long <- float/double) - [LONG_MIN..LONG_MAX] + if (varTypeIsFloating(fromType)) + { + if (!varTypeIsSmall(toType)) + { + toType = genActualType(toType); + } + + return IntegralRange::ForType(toType); + } + + // Cast from a GC type is the same as a cast from TYP_I_IMPL for our purposes. + if (varTypeIsGC(fromType)) + { + fromType = TYP_I_IMPL; + } + + if (varTypeIsSmall(toType) || (genActualType(toType) == fromType)) + { + return ForCastInput(cast); + } + + // if we're upcasting and the cast op is a known non-negative - consider + // this cast unsigned + if (!fromUnsigned && (genTypeSize(toType) >= genTypeSize(fromType))) + { + fromUnsigned = cast->CastOp()->IsNeverNegative(compiler); + } + + // CAST(uint/int <- ulong/long) - [INT_MIN..INT_MAX] + // CAST(ulong/long <- uint) - [0..UINT_MAX] + // CAST(ulong/long <- int) - [INT_MIN..INT_MAX] + if (!cast->gtOverflow()) + { + if ((fromType == TYP_INT) && fromUnsigned) + { + return {SymbolicIntegerValue::Zero, SymbolicIntegerValue::UIntMax}; + } + + return {SymbolicIntegerValue::IntMin, SymbolicIntegerValue::IntMax}; + } + + SymbolicIntegerValue lowerBound; + SymbolicIntegerValue upperBound; + switch (toType) + { + // CAST_OVF(uint <- ulong) - [INT_MIN..INT_MAX] + // CAST_OVF(uint <- long) - [INT_MIN..INT_MAX] + case TYP_UINT: + lowerBound = SymbolicIntegerValue::IntMin; + upperBound = SymbolicIntegerValue::IntMax; + break; + + // CAST_OVF(int <- ulong) - [0..INT_MAX] + // CAST_OVF(int <- long) - [INT_MIN..INT_MAX] + case TYP_INT: + lowerBound = fromUnsigned ? SymbolicIntegerValue::Zero : SymbolicIntegerValue::IntMin; + upperBound = SymbolicIntegerValue::IntMax; + break; + + // CAST_OVF(ulong <- uint) - [0..UINT_MAX] + // CAST_OVF(ulong <- int) - [0..INT_MAX] + case TYP_ULONG: + lowerBound = SymbolicIntegerValue::Zero; + upperBound = fromUnsigned ? SymbolicIntegerValue::UIntMax : SymbolicIntegerValue::IntMax; + break; + + // CAST_OVF(long <- uint) - [0..UINT_MAX] + // CAST_OVF(long <- int) - [INT_MIN..INT_MAX] + case TYP_LONG: + lowerBound = fromUnsigned ? SymbolicIntegerValue::Zero : SymbolicIntegerValue::IntMin; + upperBound = fromUnsigned ? SymbolicIntegerValue::UIntMax : SymbolicIntegerValue::IntMax; + break; + + default: + unreached(); + } + + return {lowerBound, upperBound}; +} + +/* static */ IntegralRange IntegralRange::Union(IntegralRange range1, IntegralRange range2) +{ + return IntegralRange(min(range1.GetLowerBound(), range2.GetLowerBound()), + max(range1.GetUpperBound(), range2.GetUpperBound())); +} + +#ifdef DEBUG +/* static */ void IntegralRange::Print(IntegralRange range) +{ + printf("[%lld", SymbolicToRealValue(range.m_lowerBound)); + printf(".."); + printf("%lld]", SymbolicToRealValue(range.m_upperBound)); +} +#endif // DEBUG + diff --git a/src/coreclr/jit/vnbasedmorph.cpp b/src/coreclr/jit/vnbasedmorph.cpp new file mode 100644 index 00000000000000..e76802748ab7c8 --- /dev/null +++ b/src/coreclr/jit/vnbasedmorph.cpp @@ -0,0 +1,1040 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +/*XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +XX XX +XX VN-Based Assertion Prop XX +XX XX +XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +*/ + +#include "jitpch.h" +#include "rangecheck.h" +#ifdef _MSC_VER +#pragma hdrstop +#endif + +//------------------------------------------------------------------------------ +// optVNBasedFoldExpr_Call_Memset: Unrolls NI_System_SpanHelpers_Fill for constant length. +// +// Arguments: +// call - NI_System_SpanHelpers_Fill call to unroll +// +// Return Value: +// Returns a new tree or nullptr if nothing is changed. +// +GenTree* Compiler::optVNBasedFoldExpr_Call_Memset(GenTreeCall* call) +{ + assert(call->IsSpecialIntrinsic(this, NI_System_SpanHelpers_Fill)); + + CallArg* dstArg = call->gtArgs.GetUserArgByIndex(0); + CallArg* lenArg = call->gtArgs.GetUserArgByIndex(1); + CallArg* valArg = call->gtArgs.GetUserArgByIndex(2); + + var_types valType = valArg->GetSignatureType(); + unsigned lengthScale = genTypeSize(valType); + + if (lengthScale == 1) + { + // Lower expands it slightly better. + JITDUMP("...value's type is byte - leave it for lower to expand.\n"); + return nullptr; + } + + if (varTypeIsStruct(valType) || varTypeIsGC(valType)) + { + JITDUMP("...value's type is not supported - bail out.\n"); + return nullptr; + } + + ValueNum lenVN = vnStore->VNConservativeNormalValue(lenArg->GetNode()->gtVNPair); + if (!vnStore->IsVNConstant(lenVN)) + { + JITDUMP("...length is not a constant - bail out.\n"); + return nullptr; + } + + size_t len = vnStore->CoercedConstantValue(lenVN); + if ((len > getUnrollThreshold(Memset)) || + // The first condition prevents the overflow in the second condition. + // since both len and lengthScale are expected to be small at this point. + (len * lengthScale) > getUnrollThreshold(Memset)) + { + JITDUMP("...length is too big to unroll - bail out.\n"); + return nullptr; + } + + // Some arbitrary threshold if the value is not a constant, + // since it is unlikely that we can optimize it further. + if (!valArg->GetNode()->OperIsConst() && (len >= 8)) + { + JITDUMP("...length is too big to unroll for non-constant value - bail out.\n"); + return nullptr; + } + + // Spill the side effects directly in the args, we're going to + // pick them up in the following gtExtractSideEffList + GenTree* dst = fgMakeMultiUse(&dstArg->NodeRef()); + GenTree* val = fgMakeMultiUse(&valArg->NodeRef()); + + GenTree* result = nullptr; + gtExtractSideEffList(call, &result, GTF_ALL_EFFECT, true); + + for (size_t offset = 0; offset < len; offset++) + { + // Clone dst and add offset if necessary. + GenTree* offsetNode = gtNewIconNode((ssize_t)(offset * lengthScale), TYP_I_IMPL); + GenTree* currDst = gtNewOperNode(GT_ADD, dst->TypeGet(), gtCloneExpr(dst), offsetNode); + GenTreeStoreInd* storeInd = + gtNewStoreIndNode(valType, currDst, gtCloneExpr(val), GTF_IND_UNALIGNED | GTF_IND_ALLOW_NON_ATOMIC); + + // Merge with the previous result. + result = result == nullptr ? storeInd : gtNewOperNode(GT_COMMA, TYP_VOID, result, storeInd); + } + + JITDUMP("...optimized into STOREIND(s):\n"); + DISPTREE(result); + return result; +} + +//------------------------------------------------------------------------------ +// optVNBasedFoldExpr_Call_Memmove: Unrolls NI_System_SpanHelpers_Memmove/CORINFO_HELP_MEMCPY +// if possible. This function effectively duplicates LowerCallMemmove. +// However, unlike LowerCallMemmove, it is able to optimize src into constants with help of VN. +// +// Arguments: +// call - NI_System_SpanHelpers_Memmove/CORINFO_HELP_MEMCPY call to unroll +// +// Return Value: +// Returns a new tree or nullptr if nothing is changed. +// +GenTree* Compiler::optVNBasedFoldExpr_Call_Memmove(GenTreeCall* call) +{ + JITDUMP("See if we can optimize NI_System_SpanHelpers_Memmove with help of VN...\n") + assert(call->IsSpecialIntrinsic(this, NI_System_SpanHelpers_Memmove) || + call->IsHelperCall(this, CORINFO_HELP_MEMCPY)); + + CallArg* dstArg = call->gtArgs.GetUserArgByIndex(0); + CallArg* srcArg = call->gtArgs.GetUserArgByIndex(1); + CallArg* lenArg = call->gtArgs.GetUserArgByIndex(2); + ValueNum lenVN = vnStore->VNConservativeNormalValue(lenArg->GetNode()->gtVNPair); + if (!vnStore->IsVNConstant(lenVN)) + { + JITDUMP("...length is not a constant - bail out.\n"); + return nullptr; + } + + size_t len = vnStore->CoercedConstantValue(lenVN); + if (len == 0) + { + // Memmove(dst, src, 0) -> no-op. + // Memmove doesn't dereference src/dst pointers if length is 0. + JITDUMP("...length is 0 -> optimize to no-op.\n"); + return gtWrapWithSideEffects(gtNewNothingNode(), call, GTF_ALL_EFFECT, true); + } + + if (len > getUnrollThreshold(Memcpy)) + { + JITDUMP("...length is too big to unroll - bail out.\n"); + return nullptr; + } + + // if GetImmutableDataFromAddress returns true, it means that the src is a read-only constant. + // Thus, dst and src do not overlap (if they do - it's an UB). + uint8_t* buffer = new (this, CMK_AssertionProp) uint8_t[len]; + if (!GetImmutableDataFromAddress(srcArg->GetNode(), (int)len, buffer)) + { + JITDUMP("...src is not a constant - fallback to LowerCallMemmove.\n"); + return nullptr; + } + + // if dstArg is not simple, we replace the arg directly with a temp assignment and + // continue using that temp - it allows us reliably extract all side effects. + GenTree* dst = fgMakeMultiUse(&dstArg->NodeRef()); + + // Now we're going to emit a chain of STOREIND via COMMA nodes. + // the very first tree is expected to be side-effects from the original call (including all args) + GenTree* result = nullptr; + gtExtractSideEffList(call, &result, GTF_ALL_EFFECT, true); + + unsigned lenRemaining = (unsigned)len; + while (lenRemaining > 0) + { + const ssize_t offset = (ssize_t)len - (ssize_t)lenRemaining; + + // Clone dst and add offset if necessary. + GenTree* currDst = gtCloneExpr(dst); + if (offset != 0) + { + currDst = gtNewOperNode(GT_ADD, dst->TypeGet(), currDst, gtNewIconNode(offset, TYP_I_IMPL)); + } + + // Create an unaligned STOREIND node using the largest possible word size. + var_types type = roundDownMaxType(lenRemaining); + GenTree* srcCns = gtNewGenericCon(type, buffer + offset); + GenTreeStoreInd* storeInd = gtNewStoreIndNode(type, currDst, srcCns, GTF_IND_UNALIGNED); + fgUpdateConstTreeValueNumber(srcCns); + + // Merge with the previous result. + result = result == nullptr ? storeInd : gtNewOperNode(GT_COMMA, TYP_VOID, result, storeInd); + + lenRemaining -= genTypeSize(type); + } + + JITDUMP("...optimized into STOREIND(s)!:\n"); + DISPTREE(result); + return result; +} + +//------------------------------------------------------------------------------ +// optVNBasedFoldExpr_Call: Folds given call using VN to a simpler tree. +// +// Arguments: +// block - The block containing the tree. +// parent - The parent node of the tree. +// call - The call to fold +// +// Return Value: +// Returns a new tree or nullptr if nothing is changed. +// +GenTree* Compiler::optVNBasedFoldExpr_Call(BasicBlock* block, GenTree* parent, GenTreeCall* call) +{ + switch (call->GetHelperNum()) + { + case CORINFO_HELP_CHKCASTARRAY: + case CORINFO_HELP_CHKCASTANY: + case CORINFO_HELP_CHKCASTINTERFACE: + case CORINFO_HELP_CHKCASTCLASS: + case CORINFO_HELP_ISINSTANCEOFARRAY: + case CORINFO_HELP_ISINSTANCEOFCLASS: + case CORINFO_HELP_ISINSTANCEOFANY: + case CORINFO_HELP_ISINSTANCEOFINTERFACE: + { + CallArg* castClsCallArg = call->gtArgs.GetUserArgByIndex(0); + CallArg* castObjCallArg = call->gtArgs.GetUserArgByIndex(1); + GenTree* castClsArg = castClsCallArg->GetNode(); + GenTree* castObjArg = castObjCallArg->GetNode(); + + // If object has the same VN as the cast, then the cast is effectively a no-op. + // + if (castObjArg->gtVNPair == call->gtVNPair) + { + // if castObjArg is not simple, we replace the arg with a temp assignment and + // continue using that temp - it allows us reliably extract all side effects + castObjArg = fgMakeMultiUse(&castObjCallArg->NodeRef()); + return gtWrapWithSideEffects(castObjArg, call, GTF_ALL_EFFECT, true); + } + + // Let's see if gtGetClassHandle may help us to fold the cast (since VNForCast did not). + if (castClsArg->IsIconHandle(GTF_ICON_CLASS_HDL)) + { + bool isExact; + bool isNonNull; + CORINFO_CLASS_HANDLE castFrom = gtGetClassHandle(castObjArg, &isExact, &isNonNull); + if (castFrom != NO_CLASS_HANDLE) + { + CORINFO_CLASS_HANDLE castTo = gtGetHelperArgClassHandle(castClsArg); + // Constant prop may fail to propagate compile time class handles, so verify we have + // a handle before invoking the runtime. + if ((castTo != NO_CLASS_HANDLE) && + info.compCompHnd->compareTypesForCast(castFrom, castTo) == TypeCompareState::Must) + { + // if castObjArg is not simple, we replace the arg with a temp assignment and + // continue using that temp - it allows us reliably extract all side effects + castObjArg = fgMakeMultiUse(&castObjCallArg->NodeRef()); + return gtWrapWithSideEffects(castObjArg, call, GTF_ALL_EFFECT, true); + } + } + } + } + break; + + default: + break; + } + + if (call->IsSpecialIntrinsic(this, NI_System_SpanHelpers_Memmove) || call->IsHelperCall(this, CORINFO_HELP_MEMCPY)) + { + return optVNBasedFoldExpr_Call_Memmove(call); + } + + if (call->IsSpecialIntrinsic(this, NI_System_SpanHelpers_Fill)) + { + return optVNBasedFoldExpr_Call_Memset(call); + } + + return nullptr; +} + +//------------------------------------------------------------------------------ +// optVNBasedFoldExpr: Folds given tree using VN to a constant or a simpler tree. +// +// Arguments: +// block - The block containing the tree. +// parent - The parent node of the tree. +// tree - The tree to fold. +// +// Return Value: +// Returns a new tree or nullptr if nothing is changed. +// +GenTree* Compiler::optVNBasedFoldExpr(BasicBlock* block, GenTree* parent, GenTree* tree) +{ + // First, attempt to fold it to a constant if possible. + GenTree* foldedToCns = optVNBasedFoldConstExpr(block, parent, tree); + if (foldedToCns != nullptr) + { + return foldedToCns; + } + + switch (tree->OperGet()) + { + case GT_CALL: + return optVNBasedFoldExpr_Call(block, parent, tree->AsCall()); + + // We can add more VN-based foldings here. + + default: + break; + } + return nullptr; +} + +//------------------------------------------------------------------------------ +// optVNBasedFoldConstExpr: Substitutes tree with an evaluated constant while +// managing side-effects. +// +// Arguments: +// block - The block containing the tree. +// parent - The parent node of the tree. +// tree - The tree node whose value is known at compile time. +// The tree should have a constant value number. +// +// Return Value: +// Returns a potentially new or a transformed tree node. +// Returns nullptr when no transformation is possible. +// +// Description: +// Transforms a tree node if its result evaluates to a constant. The +// transformation can be a "ChangeOper" to a constant or a new constant node +// with extracted side-effects. +// +// Before replacing or substituting the "tree" with a constant, extracts any +// side effects from the "tree" and creates a comma separated side effect list +// and then appends the transformed node at the end of the list. +// This comma separated list is then returned. +// +// For JTrue nodes, side effects are not put into a comma separated list. If +// the relop will evaluate to "true" or "false" statically, then the side-effects +// will be put into new statements, presuming the JTrue will be folded away. +// +GenTree* Compiler::optVNBasedFoldConstExpr(BasicBlock* block, GenTree* parent, GenTree* tree) +{ + if (tree->OperIs(GT_JTRUE)) + { + // Treat JTRUE separately to extract side effects into respective statements rather + // than using a COMMA separated op1. + return optVNConstantPropOnJTrue(block, tree); + } + // If relop is part of JTRUE, this should be optimized as part of the parent JTRUE. + // Or if relop is part of QMARK or anything else, we simply bail here. + else if (tree->OperIsCompare() && (tree->gtFlags & GTF_RELOP_JMP_USED)) + { + return nullptr; + } + + // We want to use the Normal ValueNumber when checking for constants. + ValueNumPair vnPair = tree->gtVNPair; + ValueNum vnCns = vnStore->VNConservativeNormalValue(vnPair); + + // Check if node evaluates to a constant + if (!vnStore->IsVNConstant(vnCns)) + { + // Last chance - propagate VNF_PtrToLoc(lcl, offset) as GT_LCL_ADDR node + VNFuncApp funcApp; + if (((tree->gtFlags & GTF_SIDE_EFFECT) == 0) && vnStore->GetVNFunc(vnCns, &funcApp) && + (funcApp.m_func == VNF_PtrToLoc)) + { + unsigned lcl = (unsigned)vnStore->CoercedConstantValue(funcApp.m_args[0]); + unsigned offs = (unsigned)vnStore->CoercedConstantValue(funcApp.m_args[1]); + return gtNewLclAddrNode(lcl, offs, tree->TypeGet()); + } + + return nullptr; + } + + GenTree* conValTree = nullptr; + switch (vnStore->TypeOfVN(vnCns)) + { + case TYP_FLOAT: + { + float value = vnStore->ConstantValue(vnCns); + + if (tree->TypeIs(TYP_INT)) + { + // Same sized reinterpretation of bits to integer + conValTree = gtNewIconNode(*(reinterpret_cast(&value))); + } + else + { + // Implicit conversion to float or double + assert(varTypeIsFloating(tree->TypeGet())); + conValTree = gtNewDconNode(FloatingPointUtils::convertToDouble(value), tree->TypeGet()); + } + break; + } + + case TYP_DOUBLE: + { + double value = vnStore->ConstantValue(vnCns); + + if (tree->TypeIs(TYP_LONG)) + { + conValTree = gtNewLconNode(*(reinterpret_cast(&value))); + } + else + { + // Implicit conversion to float or double + assert(varTypeIsFloating(tree->TypeGet())); + conValTree = gtNewDconNode(value, tree->TypeGet()); + } + break; + } + + case TYP_LONG: + { + INT64 value = vnStore->ConstantValue(vnCns); + +#ifdef TARGET_64BIT + if (vnStore->IsVNHandle(vnCns)) + { + // Don't perform constant folding that involves a handle that needs + // to be recorded as a relocation with the VM. + if (!opts.compReloc) + { + conValTree = gtNewIconHandleNode(value, vnStore->GetHandleFlags(vnCns)); + } + } + else +#endif + { + switch (tree->TypeGet()) + { + case TYP_INT: + // Implicit conversion to smaller integer + conValTree = gtNewIconNode(static_cast(value)); + break; + + case TYP_LONG: + // Same type no conversion required + conValTree = gtNewLconNode(value); + break; + + case TYP_FLOAT: + // No implicit conversions from long to float and value numbering will + // not propagate through memory reinterpretations of different size. + unreached(); + break; + + case TYP_DOUBLE: + // Same sized reinterpretation of bits to double + conValTree = gtNewDconNodeD(*(reinterpret_cast(&value))); + break; + + default: + // Do not support such optimization. + break; + } + } + } + break; + + case TYP_REF: + { + if (tree->TypeIs(TYP_REF)) + { + const size_t value = vnStore->ConstantValue(vnCns); + if (value == 0) + { + conValTree = gtNewNull(); + } + else + { + assert(vnStore->IsVNObjHandle(vnCns)); + conValTree = gtNewIconHandleNode(value, GTF_ICON_OBJ_HDL); + } + } + } + break; + + case TYP_INT: + { + int value = vnStore->ConstantValue(vnCns); +#ifndef TARGET_64BIT + if (vnStore->IsVNHandle(vnCns)) + { + // Don't perform constant folding that involves a handle that needs + // to be recorded as a relocation with the VM. + if (!opts.compReloc) + { + conValTree = gtNewIconHandleNode(value, vnStore->GetHandleFlags(vnCns)); + } + } + else +#endif + { + switch (tree->TypeGet()) + { + case TYP_REF: + case TYP_INT: + // Same type no conversion required + conValTree = gtNewIconNode(value); + break; + + case TYP_LONG: + // Implicit conversion to larger integer + conValTree = gtNewLconNode(value); + break; + + case TYP_FLOAT: + // Same sized reinterpretation of bits to float + conValTree = gtNewDconNodeF(BitOperations::UInt32BitsToSingle((uint32_t)value)); + break; + + case TYP_DOUBLE: + // No implicit conversions from int to double and value numbering will + // not propagate through memory reinterpretations of different size. + unreached(); + break; + + case TYP_BYTE: + case TYP_UBYTE: + case TYP_SHORT: + case TYP_USHORT: + assert(FitsIn(tree->TypeGet(), value)); + conValTree = gtNewIconNode(value); + break; + + default: + // Do not support (e.g. byref(const int)). + break; + } + } + } + break; + +#if defined(FEATURE_SIMD) + case TYP_SIMD8: + { + simd8_t value = vnStore->ConstantValue(vnCns); + + GenTreeVecCon* vecCon = gtNewVconNode(tree->TypeGet()); + memcpy(&vecCon->gtSimdVal, &value, sizeof(simd8_t)); + + conValTree = vecCon; + break; + } + + case TYP_SIMD12: + { + simd12_t value = vnStore->ConstantValue(vnCns); + + GenTreeVecCon* vecCon = gtNewVconNode(tree->TypeGet()); + memcpy(&vecCon->gtSimdVal, &value, sizeof(simd12_t)); + + conValTree = vecCon; + break; + } + + case TYP_SIMD16: + { + simd16_t value = vnStore->ConstantValue(vnCns); + + GenTreeVecCon* vecCon = gtNewVconNode(tree->TypeGet()); + memcpy(&vecCon->gtSimdVal, &value, sizeof(simd16_t)); + + conValTree = vecCon; + break; + } + +#if defined(TARGET_XARCH) + case TYP_SIMD32: + { + simd32_t value = vnStore->ConstantValue(vnCns); + + GenTreeVecCon* vecCon = gtNewVconNode(tree->TypeGet()); + memcpy(&vecCon->gtSimdVal, &value, sizeof(simd32_t)); + + conValTree = vecCon; + break; + } + + case TYP_SIMD64: + { + simd64_t value = vnStore->ConstantValue(vnCns); + + GenTreeVecCon* vecCon = gtNewVconNode(tree->TypeGet()); + memcpy(&vecCon->gtSimdVal, &value, sizeof(simd64_t)); + + conValTree = vecCon; + break; + } + break; + +#endif // TARGET_XARCH +#endif // FEATURE_SIMD + +#if defined(FEATURE_MASKED_HW_INTRINSICS) + case TYP_MASK: + { + simdmask_t value = vnStore->ConstantValue(vnCns); + + GenTreeMskCon* mskCon = gtNewMskConNode(tree->TypeGet()); + memcpy(&mskCon->gtSimdMaskVal, &value, sizeof(simdmask_t)); + + conValTree = mskCon; + break; + } + break; +#endif // FEATURE_MASKED_HW_INTRINSICS + + case TYP_BYREF: + // Do not support const byref optimization. + break; + + default: + // We do not record constants of other types. + unreached(); + break; + } + + if (conValTree != nullptr) + { + if (!optIsProfitableToSubstitute(tree, block, parent, conValTree)) + { + // Not profitable to substitute + return nullptr; + } + + // Were able to optimize. + conValTree->gtVNPair = vnPair; + return gtWrapWithSideEffects(conValTree, tree, GTF_SIDE_EFFECT, true); + } + else + { + // Was not able to optimize. + return nullptr; + } +} + +//------------------------------------------------------------------------------ +// optIsProfitableToSubstitute: Checks if value worth substituting to dest +// +// Arguments: +// dest - destination to substitute value to +// destBlock - Basic block of destination +// destParent - Parent of destination +// value - value we plan to substitute +// +// Returns: +// False if it's likely not profitable to do substitution, True otherwise +// +bool Compiler::optIsProfitableToSubstitute(GenTree* dest, BasicBlock* destBlock, GenTree* destParent, GenTree* value) +{ + // Giving up on these kinds of handles demonstrated size improvements + if (value->IsIconHandle(GTF_ICON_STATIC_HDL, GTF_ICON_CLASS_HDL)) + { + return false; + } + + // A simple heuristic: If the constant is defined outside of a loop (not far from its head) + // and is used inside it - don't propagate. + // + // TODO: Extend on more kinds of trees + + if (!dest->OperIs(GT_LCL_VAR)) + { + return true; + } + + const GenTreeLclVar* lcl = dest->AsLclVar(); + + if (value->IsCnsVec()) + { +#if defined(FEATURE_HW_INTRINSICS) + // Many hwintrinsics can't benefit from constant prop because they don't support + // constant folding nor do they support any specialized encodings. So, we want to + // skip constant prop and preserve any user-defined locals in that scenario. + // + // However, if the local is only referenced once then we want to allow propagation + // regardless since we can then contain the only actual usage and save a needless + // instruction. + // + // To determine number of uses, we prefer checking SSA first since it is more exact + // and can account for patterns where a local is reassigned later. However, if we + // can't find an SSA then we fallback to the naive ref count of the local, noting + // that we need to check for greater than 2 since it includes both the def and use. + + bool inspectIntrinsic = false; + + if ((destParent != nullptr) && destParent->OperIsHWIntrinsic()) + { + LclVarDsc* varDsc = lvaGetDesc(lcl); + + if (lcl->HasSsaName()) + { + inspectIntrinsic = varDsc->GetPerSsaData(lcl->GetSsaNum())->GetNumUses() > 1; + } + else + { + inspectIntrinsic = varDsc->lvRefCnt() > 2; + } + } + + if (inspectIntrinsic) + { + GenTreeHWIntrinsic* parent = destParent->AsHWIntrinsic(); + NamedIntrinsic intrinsicId = parent->GetHWIntrinsicId(); + + if (!HWIntrinsicInfo::CanBenefitFromConstantProp(intrinsicId)) + { + return false; + } + + // For several of the scenarios we may skip the costing logic + // since we know that the operand is always containable and therefore + // is always cost effective to propagate. + + return parent->ShouldConstantProp(dest, value->AsVecCon()); + } +#endif // FEATURE_HW_INTRINSICS + } + else if (!value->IsCnsFltOrDbl() && !value->IsCnsMsk()) + { + return true; + } + + gtPrepareCost(value); + + if ((value->GetCostEx() > 1) && (value->GetCostSz() > 1)) + { + // Try to find the block this constant was originally defined in + if (lcl->HasSsaName()) + { + BasicBlock* defBlock = lvaGetDesc(lcl)->GetPerSsaData(lcl->GetSsaNum())->GetBlock(); + if (defBlock != nullptr) + { + // Avoid propagating if the weighted use cost is significantly greater than the def cost. + // NOTE: this currently does not take "a float living across a call" case into account + // where we might end up with spill/restore on ABIs without callee-saved registers + const weight_t defBlockWeight = defBlock->getBBWeight(this); + const weight_t lclblockWeight = destBlock->getBBWeight(this); + + if ((defBlockWeight > 0) && ((lclblockWeight / defBlockWeight) >= BB_LOOP_WEIGHT_SCALE)) + { + JITDUMP("Constant propagation inside loop " FMT_BB " is not profitable\n", destBlock->bbNum); + return false; + } + } + } + } + return true; +} + +// Callback data for the VN based constant prop visitor. +struct VNAssertionPropVisitorInfo +{ + Compiler* m_compiler; + Statement* stmt; + BasicBlock* block; + VNAssertionPropVisitorInfo(Compiler* pThis, BasicBlock* block, Statement* stmt) + : m_compiler(pThis) + , stmt(stmt) + , block(block) + { + } +}; + +//------------------------------------------------------------------------------ +// optVNConstantPropOnJTrue +// Constant propagate on the JTrue node. +// +// Arguments: +// block - The block that contains the JTrue. +// test - The JTrue node whose relop evaluates to 0 or non-zero value. +// +// Return Value: +// nullptr if no constant propagation is done, else the modified JTrue node +// containing "0==0" or "0!=0" relop node +// (where op1 is wrapped with side effects if any). +// +GenTree* Compiler::optVNConstantPropOnJTrue(BasicBlock* block, GenTree* test) +{ + GenTree* relop = test->gtGetOp1(); + + // VN based assertion non-null on this relop has been performed. + if (!relop->OperIsCompare()) + { + return nullptr; + } + + // + // Make sure GTF_RELOP_JMP_USED flag is set so that we can later skip constant + // prop'ing a JTRUE's relop child node for a second time in the pre-order + // tree walk. + // + assert((relop->gtFlags & GTF_RELOP_JMP_USED) != 0); + + // We want to use the Normal ValueNumber when checking for constants. + ValueNum vnCns = vnStore->VNConservativeNormalValue(relop->gtVNPair); + if (!vnStore->IsVNConstant(vnCns)) + { + return nullptr; + } + + GenTree* sideEffects = gtWrapWithSideEffects(gtNewNothingNode(), relop); + if (!sideEffects->IsNothingNode()) + { + // Insert side effects before the JTRUE stmt. + Statement* newStmt = fgNewStmtNearEnd(block, sideEffects); + fgMorphBlockStmt(block, newStmt DEBUGARG(__FUNCTION__)); + } + + // Let's maintain the invariant that JTRUE's operand is always a relop. + // and if we have side effects, we wrap one of the operands with them, not the relop. + const bool evalsToTrue = (vnStore->CoercedConstantValue(vnCns) != 0); + test->AsOp()->gtOp1 = gtNewOperNode(evalsToTrue ? GT_EQ : GT_NE, relop->TypeGet(), gtNewFalse(), gtNewFalse()); + return test; +} + +//------------------------------------------------------------------------------ +// optVNBasedFoldCurStmt: Performs VN-based folding +// on the current statement's tree nodes using VN. +// +// Assumption: +// This function is called as part of a post-order tree walk. +// +// Arguments: +// tree - The currently visited tree node. +// stmt - The statement node in which the "tree" is present. +// parent - The parent node of the tree. +// block - The block that contains the statement that contains the tree. +// +// Return Value: +// Returns the standard visitor walk result. +// +Compiler::fgWalkResult Compiler::optVNBasedFoldCurStmt(BasicBlock* block, + Statement* stmt, + GenTree* parent, + GenTree* tree) +{ + // Don't try and fold expressions marked with GTF_DONT_CSE + // TODO-ASG: delete. + if (!tree->CanCSE()) + { + return WALK_CONTINUE; + } + + // Don't propagate floating-point constants into a TYP_STRUCT LclVar + // This can occur for HFA return values (see hfa_sf3E_r.exe) + if (tree->TypeIs(TYP_STRUCT)) + { + return WALK_CONTINUE; + } + + switch (tree->OperGet()) + { + // Make sure we have an R-value. + case GT_ADD: + case GT_SUB: + case GT_DIV: + case GT_MOD: + case GT_UDIV: + case GT_UMOD: + case GT_EQ: + case GT_NE: + case GT_LT: + case GT_LE: + case GT_GE: + case GT_GT: + case GT_OR: + case GT_XOR: + case GT_AND: + case GT_LSH: + case GT_RSH: + case GT_RSZ: + case GT_NEG: + case GT_CAST: + case GT_BITCAST: + case GT_INTRINSIC: +#ifdef FEATURE_HW_INTRINSICS + case GT_HWINTRINSIC: +#endif // FEATURE_HW_INTRINSICS + case GT_ARR_LENGTH: + break; + + case GT_BLK: + case GT_IND: + { + const ValueNum vn = tree->GetVN(VNK_Conservative); + if (vnStore->VNNormalValue(vn) != vn) + { + return WALK_CONTINUE; + } + } + break; + + case GT_JTRUE: + break; + + case GT_MUL: + // Don't transform long multiplies. + if (tree->gtFlags & GTF_MUL_64RSLT) + { + return WALK_CONTINUE; + } + break; + + case GT_LCL_VAR: + case GT_LCL_FLD: + // Let's not conflict with CSE (to save the movw/movt). + if (lclNumIsCSE(tree->AsLclVarCommon()->GetLclNum())) + { + return WALK_CONTINUE; + } + break; + + case GT_CALL: + // The checks aren't for correctness, but to avoid unnecessary work. + if (!tree->AsCall()->IsPure(this) && !tree->AsCall()->IsSpecialIntrinsic()) + { + return WALK_CONTINUE; + } + break; + + default: + // Unknown node, continue to walk. + return WALK_CONTINUE; + } + + // Perform the VN-based folding: + GenTree* newTree = optVNBasedFoldExpr(block, parent, tree); + + if (newTree == nullptr) + { + // Not propagated, keep going. + return WALK_CONTINUE; + } + + optAssertionProp_Update(newTree, tree, stmt); + + JITDUMP("After VN-based fold of [%06u]:\n", tree->gtTreeID); + DBEXEC(VERBOSE, gtDispStmt(stmt)); + + return WALK_CONTINUE; +} + +//------------------------------------------------------------------------------ +// optVnNonNullPropCurStmt +// Performs VN based non-null propagation on the tree node. +// +// Assumption: +// This function is called as part of a pre-order tree walk. +// +// Arguments: +// block - The block that contains the statement that contains the tree. +// stmt - The statement node in which the "tree" is present. +// tree - The currently visited tree node. +// +// Return Value: +// None. +// +// Description: +// Performs value number based non-null propagation on GT_CALL and +// indirections. This is different from flow based assertions and helps +// unify VN based constant prop and non-null prop in a single pre-order walk. +// +void Compiler::optVnNonNullPropCurStmt(BasicBlock* block, Statement* stmt, GenTree* tree) +{ + ASSERT_TP empty = BitVecOps::UninitVal(); + GenTree* newTree = nullptr; + if (tree->OperIs(GT_CALL)) + { + newTree = optNonNullAssertionProp_Call(empty, tree->AsCall()); + } + else if (tree->OperIsIndir()) + { + newTree = optAssertionProp_Ind(empty, tree, stmt); + } + if (newTree) + { + assert(newTree == tree); + optAssertionProp_Update(newTree, tree, stmt); + } +} + +//------------------------------------------------------------------------------ +// optVNAssertionPropCurStmtVisitor +// Unified Value Numbering based assertion propagation visitor. +// +// Assumption: +// This function is called as part of a post-order tree walk. +// +// Return Value: +// WALK_RESULTs. +// +// Description: +// An unified value numbering based assertion prop visitor that +// performs non-null and constant assertion propagation based on +// value numbers. +// +/* static */ +Compiler::fgWalkResult Compiler::optVNAssertionPropCurStmtVisitor(GenTree** ppTree, fgWalkData* data) +{ + VNAssertionPropVisitorInfo* pData = (VNAssertionPropVisitorInfo*)data->pCallbackData; + Compiler* pThis = pData->m_compiler; + + pThis->optVnNonNullPropCurStmt(pData->block, pData->stmt, *ppTree); + + return pThis->optVNBasedFoldCurStmt(pData->block, pData->stmt, data->parent, *ppTree); +} + +/***************************************************************************** + * + * Perform VN based i.e., data flow based assertion prop first because + * even if we don't gen new control flow assertions, we still propagate + * these first. + * + * Returns the skipped next stmt if the current statement or next few + * statements got removed, else just returns the incoming stmt. + */ +Statement* Compiler::optVNAssertionPropCurStmt(BasicBlock* block, Statement* stmt) +{ + // TODO-Review: EH successor/predecessor iteration seems broken. + // See: SELF_HOST_TESTS_ARM\jit\Directed\ExcepFilters\fault\fault.exe + if (block->bbCatchTyp == BBCT_FAULT) + { + return stmt; + } + + // Preserve the prev link before the propagation and morph. + Statement* prev = (stmt == block->firstStmt()) ? nullptr : stmt->GetPrevStmt(); + + // Perform VN based assertion prop first, in case we don't find + // anything in assertion gen. + optAssertionPropagatedCurrentStmt = false; + + VNAssertionPropVisitorInfo data(this, block, stmt); + fgWalkTreePost(stmt->GetRootNodePointer(), Compiler::optVNAssertionPropCurStmtVisitor, &data); + + if (optAssertionPropagatedCurrentStmt) + { + fgMorphBlockStmt(block, stmt DEBUGARG("optVNAssertionPropCurStmt")); + } + + // Check if propagation removed statements starting from current stmt. + // If so, advance to the next good statement. + Statement* nextStmt = (prev == nullptr) ? block->firstStmt() : prev->GetNextStmt(); + return nextStmt; +}