diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 74ebf2548df805..25ca1a7e764016 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -3422,6 +3422,12 @@ class Compiler GenTree* gtNewSimdCreateSequenceNode( var_types type, GenTree* op1, GenTree* op2, var_types simdBaseType, unsigned simdSize); + GenTree* gtNewSimdCreateGeometricSequenceNode( + var_types type, GenTree* op1, GenTree* op2, var_types simdBaseType, unsigned simdSize); + + GenTree* gtNewSimdCreateAlternatingSequenceNode( + var_types type, GenTree* op1, GenTree* op2, var_types simdBaseType, unsigned simdSize); + GenTree* gtNewSimdDotProdNode(var_types type, GenTree* op1, GenTree* op2, @@ -3552,6 +3558,30 @@ class Compiler var_types simdBaseType, unsigned simdSize); + GenTree* gtNewSimdConcatNode(var_types type, + GenTree* op1, + GenTree* op2, + var_types simdBaseType, + unsigned simdSize, + bool leftUpper, + bool rightUpper); + + GenTree* gtNewSimdZipNode(var_types type, + GenTree* op1, + GenTree* op2, + var_types simdBaseType, + unsigned simdSize, + bool upper); + + GenTree* gtNewSimdUnzipNode(var_types type, + GenTree* op1, + GenTree* op2, + var_types simdBaseType, + unsigned simdSize, + bool odd); + + GenTree* gtNewSimdReverseNode(var_types type, GenTree* op1, var_types simdBaseType, unsigned simdSize); + GenTree* gtNewSimdRoundNode( var_types type, GenTree* op1, var_types simdBaseType, unsigned simdSize); diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 1b846460e89c8b..742094d4a0dd10 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -26761,6 +26761,854 @@ GenTree* Compiler::gtNewSimdNarrowNode( #endif // !TARGET_XARCH && !TARGET_ARM64 } +//---------------------------------------------------------------------------------------------- +// Compiler::gtNewSimdCreateGeometricSequenceNode: Creates a new simd CreateGeometricSequence node +// +// Arguments: +// type - The return type of SIMD node being created +// op1 - The initial value +// op2 - The multiplier value +// simdBaseType - The base type of SIMD type of the intrinsic +// simdSize - The size of the SIMD type of the intrinsic +// +// Returns: +// The created CreateGeometricSequence node +// +GenTree* Compiler::gtNewSimdCreateGeometricSequenceNode( + var_types type, GenTree* op1, GenTree* op2, var_types simdBaseType, unsigned simdSize) +{ + assert(varTypeIsSIMD(type)); + assert(getSIMDTypeForSize(simdSize) == type); + assert(varTypeIsArithmetic(simdBaseType)); + assert(op2->OperIsConst()); + + // op2 is expected to be constant. When op1 is also constant the whole sequence can be folded + // to a constant; otherwise build the constant multiplier vector and leave one broadcast+multiply. + + GenTreeVecCon* vecCon = gtNewVconNode(type); + uint32_t simdCount = getSIMDVectorLength(simdSize, simdBaseType); + bool isPartial = !op1->OperIsConst(); + + switch (simdBaseType) + { + case TYP_BYTE: + case TYP_UBYTE: + { + uint64_t initial = isPartial ? 1 : static_cast(op1->AsIntConCommon()->IntegralValue()); + uint64_t multiplier = static_cast(op2->AsIntConCommon()->IntegralValue()); + + for (uint32_t index = 0; index < simdCount; index++) + { + vecCon->gtSimdVal.u8[index] = static_cast(initial); + initial *= multiplier; + } + break; + } + + case TYP_SHORT: + case TYP_USHORT: + { + uint64_t initial = isPartial ? 1 : static_cast(op1->AsIntConCommon()->IntegralValue()); + uint64_t multiplier = static_cast(op2->AsIntConCommon()->IntegralValue()); + + for (uint32_t index = 0; index < simdCount; index++) + { + vecCon->gtSimdVal.u16[index] = static_cast(initial); + initial *= multiplier; + } + break; + } + + case TYP_INT: + case TYP_UINT: + { + uint64_t initial = isPartial ? 1 : static_cast(op1->AsIntConCommon()->IntegralValue()); + uint64_t multiplier = static_cast(op2->AsIntConCommon()->IntegralValue()); + + for (uint32_t index = 0; index < simdCount; index++) + { + vecCon->gtSimdVal.u32[index] = static_cast(initial); + initial *= multiplier; + } + break; + } + + case TYP_LONG: + case TYP_ULONG: + { + uint64_t initial = isPartial ? 1 : static_cast(op1->AsIntConCommon()->IntegralValue()); + uint64_t multiplier = static_cast(op2->AsIntConCommon()->IntegralValue()); + + for (uint32_t index = 0; index < simdCount; index++) + { + vecCon->gtSimdVal.u64[index] = initial; + initial *= multiplier; + } + break; + } + + case TYP_FLOAT: + { + float initial = isPartial ? 1.0f : static_cast(op1->AsDblCon()->DconValue()); + float multiplier = static_cast(op2->AsDblCon()->DconValue()); + + for (uint32_t index = 0; index < simdCount; index++) + { + vecCon->gtSimdVal.f32[index] = initial; + initial *= multiplier; + } + break; + } + + case TYP_DOUBLE: + { + double initial = isPartial ? 1.0 : op1->AsDblCon()->DconValue(); + double multiplier = op2->AsDblCon()->DconValue(); + + for (uint32_t index = 0; index < simdCount; index++) + { + vecCon->gtSimdVal.f64[index] = initial; + initial *= multiplier; + } + break; + } + + default: + { + unreached(); + } + } + + GenTree* result = vecCon; + + if (isPartial) + { + GenTree* initial = gtNewSimdCreateBroadcastNode(type, op1, simdBaseType, simdSize); + result = gtNewSimdBinOpNode(GT_MUL, type, result, initial, simdBaseType, simdSize); + } + + return result; +} + +//---------------------------------------------------------------------------------------------- +// Compiler::gtNewSimdCreateAlternatingSequenceNode: Creates a new simd CreateAlternatingSequence node +// +// Arguments: +// type - The return type of SIMD node being created +// op1 - The even-indexed value +// op2 - The odd-indexed value +// simdBaseType - The base type of SIMD type of the intrinsic +// simdSize - The size of the SIMD type of the intrinsic +// +// Returns: +// The created CreateAlternatingSequence node +// +GenTree* Compiler::gtNewSimdCreateAlternatingSequenceNode( + var_types type, GenTree* op1, GenTree* op2, var_types simdBaseType, unsigned simdSize) +{ + assert(varTypeIsSIMD(type)); + assert(getSIMDTypeForSize(simdSize) == type); + assert(varTypeIsArithmetic(simdBaseType)); + + // Fold constant pairs directly. Otherwise build two broadcasts and zip them, except where + // the target has a better way to broadcast the two-lane pattern directly. + + uint32_t simdCount = getSIMDVectorLength(simdSize, simdBaseType); + + if (simdCount == 1) + { + // Only the even-indexed value contributes to the result, but op2 still needs to be evaluated for side effects. + GenTree* result = gtNewSimdCreateBroadcastNode(type, op1, simdBaseType, simdSize); + GenTree* resultDup = fgMakeMultiUse(&result); + return gtNewOperNode(GT_COMMA, type, result, gtWrapWithSideEffects(resultDup, op2, GTF_ALL_EFFECT)); + } + + if (op1->OperIsConst() && op2->OperIsConst()) + { + GenTreeVecCon* vecCon = gtNewVconNode(type); + + switch (simdBaseType) + { + case TYP_BYTE: + case TYP_UBYTE: + { + uint64_t even = static_cast(op1->AsIntConCommon()->IntegralValue()); + uint64_t odd = static_cast(op2->AsIntConCommon()->IntegralValue()); + + for (uint32_t index = 0; index < simdCount; index++) + { + vecCon->gtSimdVal.u8[index] = static_cast(((index & 1) == 0) ? even : odd); + } + break; + } + + case TYP_SHORT: + case TYP_USHORT: + { + uint64_t even = static_cast(op1->AsIntConCommon()->IntegralValue()); + uint64_t odd = static_cast(op2->AsIntConCommon()->IntegralValue()); + + for (uint32_t index = 0; index < simdCount; index++) + { + vecCon->gtSimdVal.u16[index] = static_cast(((index & 1) == 0) ? even : odd); + } + break; + } + + case TYP_INT: + case TYP_UINT: + { + uint64_t even = static_cast(op1->AsIntConCommon()->IntegralValue()); + uint64_t odd = static_cast(op2->AsIntConCommon()->IntegralValue()); + + for (uint32_t index = 0; index < simdCount; index++) + { + vecCon->gtSimdVal.u32[index] = static_cast(((index & 1) == 0) ? even : odd); + } + break; + } + + case TYP_LONG: + case TYP_ULONG: + { + uint64_t even = static_cast(op1->AsIntConCommon()->IntegralValue()); + uint64_t odd = static_cast(op2->AsIntConCommon()->IntegralValue()); + + for (uint32_t index = 0; index < simdCount; index++) + { + vecCon->gtSimdVal.u64[index] = ((index & 1) == 0) ? even : odd; + } + break; + } + + case TYP_FLOAT: + { + double even = op1->AsDblCon()->DconValue(); + double odd = op2->AsDblCon()->DconValue(); + + for (uint32_t index = 0; index < simdCount; index++) + { + vecCon->gtSimdVal.f32[index] = static_cast(((index & 1) == 0) ? even : odd); + } + break; + } + + case TYP_DOUBLE: + { + double even = op1->AsDblCon()->DconValue(); + double odd = op2->AsDblCon()->DconValue(); + + for (uint32_t index = 0; index < simdCount; index++) + { + vecCon->gtSimdVal.f64[index] = ((index & 1) == 0) ? even : odd; + } + break; + } + + default: + { + unreached(); + } + } + + return vecCon; + } + +#if defined(TARGET_XARCH) + if (((simdBaseType == TYP_INT) || (simdBaseType == TYP_UINT)) && + ((compOpportunisticallyDependsOn(InstructionSet_AVX2) && ((simdSize == 16) || (simdSize == 32))) || + (compOpportunisticallyDependsOn(InstructionSet_AVX512) && (simdSize == 64)))) + { + // var pattern = Vector128.CreateScalarUnsafe(op1).WithElement(1, op2); + // return Broadcast(pattern.AsInt64()).As(); + + GenTree* pattern = gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, op1, simdBaseType, 16); + pattern = gtNewSimdWithElementNode(TYP_SIMD16, pattern, gtNewIconNode(1), op2, simdBaseType, 16); + + if (simdSize == 64) + { + return gtNewSimdHWIntrinsicNode(type, pattern, NI_AVX512_BroadcastPairScalarToVector512, simdBaseType, + simdSize); + } + + var_types broadcastBaseType = (simdBaseType == TYP_INT) ? TYP_LONG : TYP_ULONG; + NamedIntrinsic broadcast = + (simdSize == 16) ? NI_AVX2_BroadcastScalarToVector128 : NI_AVX2_BroadcastScalarToVector256; + return gtNewSimdHWIntrinsicNode(type, pattern, broadcast, broadcastBaseType, simdSize); + } +#endif // TARGET_XARCH + + GenTree* even = gtNewSimdCreateBroadcastNode(type, op1, simdBaseType, simdSize); + GenTree* odd = gtNewSimdCreateBroadcastNode(type, op2, simdBaseType, simdSize); + + return gtNewSimdZipNode(type, even, odd, simdBaseType, simdSize, false); +} + +//---------------------------------------------------------------------------------------------- +// Compiler::gtNewSimdConcatNode: Creates a new simd ConcatLowerLower/... node +// +// Arguments: +// type - The return type of SIMD node being created +// op1 - The vector that supplies the lower half +// op2 - The vector that supplies the upper half +// simdBaseType - The base type of SIMD type of the intrinsic +// simdSize - The size of the SIMD type of the intrinsic +// leftUpper - Whether the lower result half comes from the upper half of op1 +// rightUpper - Whether the upper result half comes from the upper half of op2 +// +// Returns: +// The created concat node +// +GenTree* Compiler::gtNewSimdConcatNode(var_types type, + GenTree* op1, + GenTree* op2, + var_types simdBaseType, + unsigned simdSize, + bool leftUpper, + bool rightUpper) +{ + assert(varTypeIsSIMD(type)); + assert(getSIMDTypeForSize(simdSize) == type); + assert(varTypeIsArithmetic(simdBaseType)); + + uint32_t simdCount = getSIMDVectorLength(simdSize, simdBaseType); + uint32_t lowerCount = (simdCount + 1) / 2; + + if (simdCount == 1) + { + GenTree* result = op1; + GenTree* resultDup = fgMakeMultiUse(&result); + return gtNewOperNode(GT_COMMA, type, result, gtWrapWithSideEffects(resultDup, op2, GTF_ALL_EFFECT)); + } + +#if defined(TARGET_ARM64) + if ((simdSize == 8) && (simdCount == 2) && + ((simdBaseType == TYP_INT) || (simdBaseType == TYP_UINT) || (simdBaseType == TYP_FLOAT))) + { + // var result = op1; + // if (leftUpper) + // { + // result = result.WithElement(0, result.GetElement(1)); + // } + // return result.WithElement(1, op2.GetElement(rightUpper ? 1 : 0)); + + GenTree* result = op1; + + if (leftUpper) + { + GenTree* resultDup = fgMakeMultiUse(&result); + result = gtNewSimdHWIntrinsicNode(type, result, gtNewIconNode(0), resultDup, gtNewIconNode(1), + NI_AdvSimd_Arm64_InsertSelectedScalar, simdBaseType, simdSize); + } + + return gtNewSimdHWIntrinsicNode(type, result, gtNewIconNode(1), op2, gtNewIconNode(rightUpper ? 1 : 0), + NI_AdvSimd_Arm64_InsertSelectedScalar, simdBaseType, simdSize); + } + + if (simdSize == 16) + { + // var result = op1.AsUInt64(); + // if (leftUpper) + // { + // result = result.WithElement(0, result.GetElement(1)); + // } + // return result.WithElement(1, op2.AsUInt64().GetElement(rightUpper ? 1 : 0)).As(); + + GenTree* result = op1; + + if (leftUpper) + { + GenTree* resultDup = fgMakeMultiUse(&result); + result = gtNewSimdHWIntrinsicNode(type, result, gtNewIconNode(0), resultDup, gtNewIconNode(1), + NI_AdvSimd_Arm64_InsertSelectedScalar, TYP_ULONG, simdSize); + } + + return gtNewSimdHWIntrinsicNode(type, result, gtNewIconNode(1), op2, gtNewIconNode(rightUpper ? 1 : 0), + NI_AdvSimd_Arm64_InsertSelectedScalar, TYP_ULONG, simdSize); + } +#endif // TARGET_ARM64 + +#if defined(TARGET_XARCH) + if (simdSize == 16) +#elif defined(TARGET_ARM64) + if (simdSize == 8) +#else +#error Unsupported platform +#endif // !TARGET_XARCH && !TARGET_ARM64 + { +#if defined(TARGET_XARCH) + if ((simdBaseType == TYP_INT) || (simdBaseType == TYP_UINT) || (simdBaseType == TYP_FLOAT)) + { + // return Sse.Shuffle(op1.AsSingle(), op2.AsSingle(), immediate).As(); + + uint32_t leftStart = leftUpper ? simdCount - lowerCount : 0; + uint32_t rightStart = rightUpper ? simdCount - lowerCount : 0; + unsigned immediate = 0; + + for (uint32_t index = 0; index < simdCount; index++) + { + uint32_t shuffleIndex = (index < lowerCount) ? leftStart + index : rightStart + index - lowerCount; + immediate |= shuffleIndex << (index * 2); + } + + return gtNewSimdHWIntrinsicNode(type, op1, op2, gtNewIconNode(immediate), NI_X86Base_Shuffle, TYP_FLOAT, + simdSize); + } +#endif // TARGET_XARCH + + // var tmp = op1.ToVectorUnsafe().WithUpper(op2); + // return Shuffle(tmp, indices).GetLower(); + + unsigned wideSimdSize = simdSize * 2; + var_types wideType = getSIMDTypeForSize(wideSimdSize); + GenTreeVecCon* shuffle = gtNewVconNode(wideType); + uint32_t wideCount = getSIMDVectorLength(wideSimdSize, simdBaseType); + uint32_t leftStart = leftUpper ? simdCount - lowerCount : 0; + uint32_t rightStart = rightUpper ? simdCount - lowerCount : 0; + + for (uint32_t index = 0; index < wideCount; index++) + { + uint32_t shuffleIndex = 0; + + if (index < simdCount) + { + shuffleIndex = (index < lowerCount) ? leftStart + index : simdCount + rightStart + index - lowerCount; + } + + switch (simdBaseType) + { + case TYP_BYTE: + case TYP_UBYTE: + shuffle->gtSimdVal.u8[index] = static_cast(shuffleIndex); + break; + + case TYP_SHORT: + case TYP_USHORT: + shuffle->gtSimdVal.u16[index] = static_cast(shuffleIndex); + break; + + case TYP_INT: + case TYP_UINT: + case TYP_FLOAT: + shuffle->gtSimdVal.u32[index] = shuffleIndex; + break; + + case TYP_LONG: + case TYP_ULONG: + case TYP_DOUBLE: + shuffle->gtSimdVal.u64[index] = shuffleIndex; + break; + + default: + unreached(); + } + } + + assert(IsValidForShuffle(shuffle, wideSimdSize, simdBaseType, nullptr, false)); + +#if defined(TARGET_XARCH) + GenTree* result = + gtNewSimdHWIntrinsicNode(wideType, op1, NI_Vector128_ToVector256Unsafe, simdBaseType, simdSize); +#elif defined(TARGET_ARM64) + GenTree* result = + gtNewSimdHWIntrinsicNode(wideType, op1, NI_Vector64_ToVector128Unsafe, simdBaseType, simdSize); +#else +#error Unsupported platform +#endif // !TARGET_XARCH && !TARGET_ARM64 + + result = gtNewSimdWithUpperNode(wideType, result, op2, simdBaseType, wideSimdSize); + result = gtNewSimdShuffleNode(wideType, result, shuffle, simdBaseType, wideSimdSize, false); + + return gtNewSimdGetLowerNode(type, result, simdBaseType, wideSimdSize); + } + + // var lower = leftUpper ? op1.GetUpper() : op1.GetLower(); + // var upper = rightUpper ? op2.GetUpper() : op2.GetLower(); + // return lower.ToVectorUnsafe().WithUpper(upper); + + var_types halfType = getSIMDTypeForSize(simdSize / 2); + GenTree* lower = leftUpper ? gtNewSimdGetUpperNode(halfType, op1, simdBaseType, simdSize) + : gtNewSimdGetLowerNode(halfType, op1, simdBaseType, simdSize); + GenTree* upper = rightUpper ? gtNewSimdGetUpperNode(halfType, op2, simdBaseType, simdSize) + : gtNewSimdGetLowerNode(halfType, op2, simdBaseType, simdSize); + +#if defined(TARGET_XARCH) + GenTree* result = + (simdSize == 32) + ? gtNewSimdHWIntrinsicNode(type, lower, NI_Vector128_ToVector256Unsafe, simdBaseType, simdSize / 2) + : gtNewSimdHWIntrinsicNode(type, lower, NI_Vector256_ToVector512Unsafe, simdBaseType, simdSize / 2); +#elif defined(TARGET_ARM64) + GenTree* result = gtNewSimdHWIntrinsicNode(type, lower, NI_Vector64_ToVector128Unsafe, simdBaseType, simdSize / 2); +#else +#error Unsupported platform +#endif // !TARGET_XARCH && !TARGET_ARM64 + + return gtNewSimdWithUpperNode(type, result, upper, simdBaseType, simdSize); +} + +//---------------------------------------------------------------------------------------------- +// Compiler::gtNewSimdZipNode: Creates a new simd ZipLower/ZipUpper node +// +// Arguments: +// type - The return type of SIMD node being created +// op1 - The vector that supplies even-indexed elements +// op2 - The vector that supplies odd-indexed elements +// simdBaseType - The base type of SIMD type of the intrinsic +// simdSize - The size of the SIMD type of the intrinsic +// upper - Whether to zip the upper halves +// +// Returns: +// The created zip node +// +GenTree* Compiler::gtNewSimdZipNode( + var_types type, GenTree* op1, GenTree* op2, var_types simdBaseType, unsigned simdSize, bool upper) +{ + assert(varTypeIsSIMD(type)); + assert(getSIMDTypeForSize(simdSize) == type); + assert(varTypeIsArithmetic(simdBaseType)); + + if (getSIMDVectorLength(simdSize, simdBaseType) == 1) + { + GenTree* result = op1; + GenTree* resultDup = fgMakeMultiUse(&result); + return gtNewOperNode(GT_COMMA, type, result, gtWrapWithSideEffects(resultDup, op2, GTF_ALL_EFFECT)); + } + +#if defined(TARGET_XARCH) + if (simdSize == 16) + { + NamedIntrinsic intrinsic = upper ? NI_X86Base_UnpackHigh : NI_X86Base_UnpackLow; + return gtNewSimdHWIntrinsicNode(type, op1, op2, intrinsic, simdBaseType, simdSize); + } + + if ((simdSize == 64) && ((simdBaseType == TYP_INT) || (simdBaseType == TYP_UINT)) && + compOpportunisticallyDependsOn(InstructionSet_AVX512)) + { + // var lower = Avx512F.UnpackLow(op1, op2); + // var upper = Avx512F.UnpackHigh(op1, op2); + // return Shuffle4x128(Shuffle4x128(lower, upper, lanes), ..., SHUFFLE_WYZX); + + GenTree* op1Dup = fgMakeMultiUse(&op1); + GenTree* op2Dup = fgMakeMultiUse(&op2); + + GenTree* lower = gtNewSimdHWIntrinsicNode(type, op1, op2, NI_AVX512_UnpackLow, simdBaseType, simdSize); + GenTree* higher = gtNewSimdHWIntrinsicNode(type, op1Dup, op2Dup, NI_AVX512_UnpackHigh, simdBaseType, simdSize); + + unsigned lanes = upper ? 0xEE : 0x44; + GenTree* result = gtNewSimdHWIntrinsicNode(type, lower, higher, gtNewIconNode(lanes), NI_AVX512_Shuffle4x128, + simdBaseType, simdSize); + GenTree* resultDup = fgMakeMultiUse(&result); + + return gtNewSimdHWIntrinsicNode(type, result, resultDup, gtNewIconNode(SHUFFLE_WYZX), NI_AVX512_Shuffle4x128, + simdBaseType, simdSize); + } +#elif defined(TARGET_ARM64) + NamedIntrinsic intrinsic = upper ? NI_AdvSimd_Arm64_ZipHigh : NI_AdvSimd_Arm64_ZipLow; + return gtNewSimdHWIntrinsicNode(type, op1, op2, intrinsic, simdBaseType, simdSize); +#else +#error Unsupported platform +#endif // !TARGET_XARCH && !TARGET_ARM64 + +#if defined(TARGET_XARCH) + // var lower = Zip(left, right, upper: false); + // var upper = Zip(left, right, upper: true); + // return lower.ToVectorUnsafe().WithUpper(upper); + + var_types halfType = getSIMDTypeForSize(simdSize / 2); + GenTree* left = upper ? gtNewSimdGetUpperNode(halfType, op1, simdBaseType, simdSize) + : gtNewSimdGetLowerNode(halfType, op1, simdBaseType, simdSize); + GenTree* right = upper ? gtNewSimdGetUpperNode(halfType, op2, simdBaseType, simdSize) + : gtNewSimdGetLowerNode(halfType, op2, simdBaseType, simdSize); + + GenTree* leftDup = fgMakeMultiUse(&left); + GenTree* rightDup = fgMakeMultiUse(&right); + + GenTree* lower = gtNewSimdZipNode(halfType, left, right, simdBaseType, simdSize / 2, false); + GenTree* higher = gtNewSimdZipNode(halfType, leftDup, rightDup, simdBaseType, simdSize / 2, true); + + GenTree* result = + (simdSize == 32) + ? gtNewSimdHWIntrinsicNode(type, lower, NI_Vector128_ToVector256Unsafe, simdBaseType, simdSize / 2) + : gtNewSimdHWIntrinsicNode(type, lower, NI_Vector256_ToVector512Unsafe, simdBaseType, simdSize / 2); + return gtNewSimdWithUpperNode(type, result, higher, simdBaseType, simdSize); +#endif // TARGET_XARCH +} + +//---------------------------------------------------------------------------------------------- +// Compiler::gtNewSimdUnzipNode: Creates a new simd UnzipEven/UnzipOdd node +// +// Arguments: +// type - The return type of SIMD node being created +// op1 - The vector that supplies the lower half +// op2 - The vector that supplies the upper half +// simdBaseType - The base type of SIMD type of the intrinsic +// simdSize - The size of the SIMD type of the intrinsic +// odd - Whether to unzip odd-indexed elements +// +// Returns: +// The created unzip node +// +GenTree* Compiler::gtNewSimdUnzipNode( + var_types type, GenTree* op1, GenTree* op2, var_types simdBaseType, unsigned simdSize, bool odd) +{ + assert(varTypeIsSIMD(type)); + assert(getSIMDTypeForSize(simdSize) == type); + assert(varTypeIsArithmetic(simdBaseType)); + + uint32_t simdCount = getSIMDVectorLength(simdSize, simdBaseType); + + if (simdCount == 1) + { + if (odd) + { + GenTree* result = gtWrapWithSideEffects(gtNewZeroConNode(type), op2, GTF_ALL_EFFECT); + return gtWrapWithSideEffects(result, op1, GTF_ALL_EFFECT); + } + + GenTree* result = op1; + GenTree* resultDup = fgMakeMultiUse(&result); + return gtNewOperNode(GT_COMMA, type, result, gtWrapWithSideEffects(resultDup, op2, GTF_ALL_EFFECT)); + } + +#if defined(TARGET_ARM64) + // return odd ? AdvSimd.Arm64.UnzipOdd(op1, op2) : AdvSimd.Arm64.UnzipEven(op1, op2); + + NamedIntrinsic intrinsic = odd ? NI_AdvSimd_Arm64_UnzipOdd : NI_AdvSimd_Arm64_UnzipEven; + return gtNewSimdHWIntrinsicNode(type, op1, op2, intrinsic, simdBaseType, simdSize); +#elif defined(TARGET_XARCH) + if (simdSize == 16) + { + if ((simdBaseType == TYP_INT) || (simdBaseType == TYP_UINT) || (simdBaseType == TYP_FLOAT)) + { + // return Sse.Shuffle(op1.AsSingle(), op2.AsSingle(), odd ? 0xDD : 0x88).As(); + + return gtNewSimdHWIntrinsicNode(type, op1, op2, gtNewIconNode(odd ? 0xDD : 0x88), NI_X86Base_Shuffle, + TYP_FLOAT, simdSize); + } + + // var tmp = op1.ToVectorUnsafe().WithUpper(op2); + // return Shuffle(tmp, indices).GetLower(); + + unsigned wideSimdSize = simdSize * 2; + var_types wideType = getSIMDTypeForSize(wideSimdSize); + GenTreeVecCon* shuffle = gtNewVconNode(wideType); + uint32_t wideCount = getSIMDVectorLength(wideSimdSize, simdBaseType); + uint32_t start = odd ? 1 : 0; + uint32_t lowerCount = (simdCount - start + 1) / 2; + + for (uint32_t index = 0; index < wideCount; index++) + { + uint32_t shuffleIndex = 0; + + if (index < simdCount) + { + shuffleIndex = + (index < lowerCount) ? start + (index * 2) : simdCount + start + ((index - lowerCount) * 2); + } + + switch (simdBaseType) + { + case TYP_BYTE: + case TYP_UBYTE: + shuffle->gtSimdVal.u8[index] = static_cast(shuffleIndex); + break; + + case TYP_SHORT: + case TYP_USHORT: + shuffle->gtSimdVal.u16[index] = static_cast(shuffleIndex); + break; + + case TYP_INT: + case TYP_UINT: + case TYP_FLOAT: + shuffle->gtSimdVal.u32[index] = shuffleIndex; + break; + + case TYP_LONG: + case TYP_ULONG: + case TYP_DOUBLE: + shuffle->gtSimdVal.u64[index] = shuffleIndex; + break; + + default: + unreached(); + } + } + + assert(IsValidForShuffle(shuffle, wideSimdSize, simdBaseType, nullptr, false)); + + GenTree* result = + gtNewSimdHWIntrinsicNode(wideType, op1, NI_Vector128_ToVector256Unsafe, simdBaseType, simdSize); + result = gtNewSimdWithUpperNode(wideType, result, op2, simdBaseType, wideSimdSize); + result = gtNewSimdShuffleNode(wideType, result, shuffle, simdBaseType, wideSimdSize, false); + + return gtNewSimdGetLowerNode(type, result, simdBaseType, wideSimdSize); + } + + if ((simdSize == 32) && ((simdBaseType == TYP_INT) || (simdBaseType == TYP_UINT)) && + compOpportunisticallyDependsOn(InstructionSet_AVX2)) + { + // var left = Shuffle(op1, indices); + // var right = Shuffle(op2, indices); + // return left.GetLower().ToVector256Unsafe().WithUpper(right.GetLower()); + + GenTreeVecCon* shuffle = gtNewVconNode(type); + uint32_t start = odd ? 1 : 0; + + for (uint32_t index = 0; index < simdCount; index++) + { + shuffle->gtSimdVal.u32[index] = start + (2 * (index % (simdCount / 2))); + } + + assert(IsValidForShuffle(shuffle, simdSize, simdBaseType, nullptr, false)); + + GenTree* leftShuffle = shuffle; + GenTree* rightShuffle = gtCloneExpr(shuffle); + GenTree* left = gtNewSimdShuffleNode(type, op1, leftShuffle, simdBaseType, simdSize, false); + GenTree* right = gtNewSimdShuffleNode(type, op2, rightShuffle, simdBaseType, simdSize, false); + + var_types halfType = getSIMDTypeForSize(simdSize / 2); + GenTree* lower = gtNewSimdGetLowerNode(halfType, left, simdBaseType, simdSize); + GenTree* upper = gtNewSimdGetLowerNode(halfType, right, simdBaseType, simdSize); + + GenTree* result = + gtNewSimdHWIntrinsicNode(type, lower, NI_Vector128_ToVector256Unsafe, simdBaseType, simdSize / 2); + return gtNewSimdWithUpperNode(type, result, upper, simdBaseType, simdSize); + } + + if ((simdSize == 64) && ((simdBaseType == TYP_INT) || (simdBaseType == TYP_UINT)) && + compOpportunisticallyDependsOn(InstructionSet_AVX512)) + { + // return Avx512F.PermuteVar16x32x2(op1, indices, op2); + + GenTreeVecCon* shuffle = gtNewVconNode(type); + uint32_t start = odd ? 1 : 0; + + for (uint32_t index = 0; index < simdCount; index++) + { + shuffle->gtSimdVal.u32[index] = + (index < (simdCount / 2)) ? start + (2 * index) : simdCount + start + (2 * (index - (simdCount / 2))); + } + + return gtNewSimdHWIntrinsicNode(type, op1, shuffle, op2, NI_AVX512_PermuteVar16x32x2, simdBaseType, simdSize); + } + + // var lower = Unzip(op1.GetLower(), op1.GetUpper()); + // var upper = Unzip(op2.GetLower(), op2.GetUpper()); + // return lower.ToVectorUnsafe().WithUpper(upper); + + GenTree* op1Dup = fgMakeMultiUse(&op1); + GenTree* op2Dup = fgMakeMultiUse(&op2); + + var_types halfType = getSIMDTypeForSize(simdSize / 2); + GenTree* op1Lower = gtNewSimdGetLowerNode(halfType, op1, simdBaseType, simdSize); + GenTree* op1Upper = gtNewSimdGetUpperNode(halfType, op1Dup, simdBaseType, simdSize); + GenTree* op2Lower = gtNewSimdGetLowerNode(halfType, op2, simdBaseType, simdSize); + GenTree* op2Upper = gtNewSimdGetUpperNode(halfType, op2Dup, simdBaseType, simdSize); + + GenTree* lower = gtNewSimdUnzipNode(halfType, op1Lower, op1Upper, simdBaseType, simdSize / 2, odd); + GenTree* higher = gtNewSimdUnzipNode(halfType, op2Lower, op2Upper, simdBaseType, simdSize / 2, odd); + + GenTree* result = + (simdSize == 32) + ? gtNewSimdHWIntrinsicNode(type, lower, NI_Vector128_ToVector256Unsafe, simdBaseType, simdSize / 2) + : gtNewSimdHWIntrinsicNode(type, lower, NI_Vector256_ToVector512Unsafe, simdBaseType, simdSize / 2); + return gtNewSimdWithUpperNode(type, result, higher, simdBaseType, simdSize); +#else +#error Unsupported platform +#endif // !TARGET_XARCH && !TARGET_ARM64 +} + +//---------------------------------------------------------------------------------------------- +// Compiler::gtNewSimdReverseNode: Creates a new simd Reverse node +// +// Arguments: +// type - The return type of SIMD node being created +// op1 - The vector to reverse +// simdBaseType - The base type of SIMD type of the intrinsic +// simdSize - The size of the SIMD type of the intrinsic +// +// Returns: +// The created reverse node +// +GenTree* Compiler::gtNewSimdReverseNode(var_types type, GenTree* op1, var_types simdBaseType, unsigned simdSize) +{ + assert(varTypeIsSIMD(type)); + assert(getSIMDTypeForSize(simdSize) == type); + assert(varTypeIsArithmetic(simdBaseType)); + + uint32_t simdCount = getSIMDVectorLength(simdSize, simdBaseType); + + if (simdCount == 1) + { + return op1; + } + +#if defined(TARGET_XARCH) + if ((simdSize == 32) && ((simdBaseType == TYP_INT) || (simdBaseType == TYP_UINT)) && + compOpportunisticallyDependsOn(InstructionSet_AVX2)) + { + // var tmp = Avx2.Shuffle(op1, SHUFFLE_XYZW); + // return Avx2.Permute2x128(tmp, tmp, 1); + + GenTree* reverseInLane = + gtNewSimdHWIntrinsicNode(type, op1, gtNewIconNode(SHUFFLE_XYZW), NI_AVX2_Shuffle, simdBaseType, simdSize); + GenTree* reverseInLaneDup = fgMakeMultiUse(&reverseInLane); + + return gtNewSimdHWIntrinsicNode(type, reverseInLane, reverseInLaneDup, gtNewIconNode(1), NI_AVX2_Permute2x128, + simdBaseType, simdSize); + } +#elif defined(TARGET_ARM64) + if ((simdSize == 8) && ((simdBaseType == TYP_INT) || (simdBaseType == TYP_UINT) || (simdBaseType == TYP_FLOAT))) + { + // return AdvSimd.ReverseElement32(op1.AsInt64()).As(); + + return gtNewSimdHWIntrinsicNode(type, op1, NI_AdvSimd_ReverseElement32, TYP_LONG, simdSize); + } +#endif // TARGET_XARCH || TARGET_ARM64 + + // return Shuffle(op1, indices); + + GenTreeVecCon* shuffle = gtNewVconNode(type); + + for (uint32_t index = 0; index < simdCount; index++) + { + uint32_t shuffleIndex = simdCount - 1 - index; + + switch (simdBaseType) + { + case TYP_BYTE: + case TYP_UBYTE: + shuffle->gtSimdVal.u8[index] = static_cast(shuffleIndex); + break; + + case TYP_SHORT: + case TYP_USHORT: + shuffle->gtSimdVal.u16[index] = static_cast(shuffleIndex); + break; + + case TYP_INT: + case TYP_UINT: + case TYP_FLOAT: + shuffle->gtSimdVal.u32[index] = shuffleIndex; + break; + + case TYP_LONG: + case TYP_ULONG: + case TYP_DOUBLE: + shuffle->gtSimdVal.u64[index] = shuffleIndex; + break; + + default: + unreached(); + } + } + + assert(IsValidForShuffle(shuffle, simdSize, simdBaseType, nullptr, false)); + + return gtNewSimdShuffleNode(type, op1, shuffle, simdBaseType, simdSize, false); +} + //------------------------------------------------------------------------ // gtNewSimdRoundNode: Creates a new simd Round node // diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index 7275c2ffe4f305..ad82209de9c9c9 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -1334,6 +1334,54 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, break; } + case NI_Vector64_CreateGeometricSequence: + case NI_Vector128_CreateGeometricSequence: + { + assert(sig->numArgs == 2); + + if (!impStackTop(0).val->OperIsConst()) + { + break; + } + + if (varTypeIsFloating(simdBaseType) && !impStackTop(1).val->OperIsConst() && + (getSIMDVectorLength(simdSize, simdBaseType) > 2)) + { + // Floating-point multiplication is not associative; use the fallback to preserve recurrence order. + break; + } + + if (varTypeIsLong(simdBaseType) && !impStackTop(1).val->OperIsConst() && (simdSize != 8)) + { + // TODO-ARM64-CQ: We should support long/ulong multiplication. + break; + } + + impSpillSideEffect(true, stackState.esStackDepth - + 2 DEBUGARG("Spilling op1 side effects for vector CreateGeometricSequence")); + + op2 = impPopStack().val; + op1 = impPopStack().val; + + retNode = gtNewSimdCreateGeometricSequenceNode(retType, op1, op2, simdBaseType, simdSize); + break; + } + + case NI_Vector64_CreateAlternatingSequence: + case NI_Vector128_CreateAlternatingSequence: + { + assert(sig->numArgs == 2); + + impSpillSideEffect(true, stackState.esStackDepth - + 2 DEBUGARG("Spilling op1 side effects for vector CreateAlternatingSequence")); + + op2 = impPopStack().val; + op1 = impPopStack().val; + + retNode = gtNewSimdCreateAlternatingSequenceNode(retType, op1, op2, simdBaseType, simdSize); + break; + } + case NI_Vector64_CreateScalarUnsafe: case NI_Vector128_CreateScalarUnsafe: { @@ -1519,6 +1567,20 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, break; } + case NI_Vector64_get_SignSequence: + case NI_Vector128_get_SignSequence: + { + assert(sig->numArgs == 0); + + var_types scalarType = genActualType(simdBaseType); + GenTree* one = gtNewOneConNode(scalarType); + GenTree* negativeOne = varTypeIsFloating(simdBaseType) ? gtNewDconNode(-1.0, simdBaseType) + : gtNewAllBitsSetConNode(scalarType); + + retNode = gtNewSimdCreateAlternatingSequenceNode(retType, one, negativeOne, simdBaseType, simdSize); + break; + } + case NI_Vector64_get_NaN: case NI_Vector128_get_NaN: { @@ -2844,6 +2906,71 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, break; } + case NI_Vector64_ConcatLowerLower: + case NI_Vector128_ConcatLowerLower: + case NI_Vector64_ConcatLowerUpper: + case NI_Vector128_ConcatLowerUpper: + case NI_Vector64_ConcatUpperLower: + case NI_Vector128_ConcatUpperLower: + case NI_Vector64_ConcatUpperUpper: + case NI_Vector128_ConcatUpperUpper: + { + assert(sig->numArgs == 2); + + op2 = impSIMDPopStack(); + op1 = impSIMDPopStack(); + + bool leftUpper = + (intrinsic == NI_Vector64_ConcatUpperLower) || (intrinsic == NI_Vector128_ConcatUpperLower) || + (intrinsic == NI_Vector64_ConcatUpperUpper) || (intrinsic == NI_Vector128_ConcatUpperUpper); + bool rightUpper = + (intrinsic == NI_Vector64_ConcatLowerUpper) || (intrinsic == NI_Vector128_ConcatLowerUpper) || + (intrinsic == NI_Vector64_ConcatUpperUpper) || (intrinsic == NI_Vector128_ConcatUpperUpper); + + retNode = gtNewSimdConcatNode(retType, op1, op2, simdBaseType, simdSize, leftUpper, rightUpper); + break; + } + + case NI_Vector64_ZipLower: + case NI_Vector128_ZipLower: + case NI_Vector64_ZipUpper: + case NI_Vector128_ZipUpper: + { + assert(sig->numArgs == 2); + + op2 = impSIMDPopStack(); + op1 = impSIMDPopStack(); + + bool upper = (intrinsic == NI_Vector64_ZipUpper) || (intrinsic == NI_Vector128_ZipUpper); + retNode = gtNewSimdZipNode(retType, op1, op2, simdBaseType, simdSize, upper); + break; + } + + case NI_Vector64_UnzipEven: + case NI_Vector128_UnzipEven: + case NI_Vector64_UnzipOdd: + case NI_Vector128_UnzipOdd: + { + assert(sig->numArgs == 2); + + op2 = impSIMDPopStack(); + op1 = impSIMDPopStack(); + + bool odd = (intrinsic == NI_Vector64_UnzipOdd) || (intrinsic == NI_Vector128_UnzipOdd); + retNode = gtNewSimdUnzipNode(retType, op1, op2, simdBaseType, simdSize, odd); + break; + } + + case NI_Vector64_Reverse: + case NI_Vector128_Reverse: + { + assert(sig->numArgs == 1); + + op1 = impSIMDPopStack(); + retNode = gtNewSimdReverseNode(retType, op1, simdBaseType, simdSize); + break; + } + case NI_Vector64_op_ExclusiveOr: case NI_Vector128_op_ExclusiveOr: { diff --git a/src/coreclr/jit/hwintrinsiclistarm64.h b/src/coreclr/jit/hwintrinsiclistarm64.h index b127b61920d4b7..7d261d855a00e8 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64.h +++ b/src/coreclr/jit/hwintrinsiclistarm64.h @@ -33,6 +33,10 @@ HARDWARE_INTRINSIC(Vector64, AsUInt16, HARDWARE_INTRINSIC(Vector64, AsUInt32, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector64, AsUInt64, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector64, Ceiling, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector64, ConcatLowerLower, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector64, ConcatLowerUpper, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector64, ConcatUpperLower, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector64, ConcatUpperUpper, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, ConditionalSelect, 8, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, ConvertToDouble, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector64, ConvertToInt32, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) @@ -45,6 +49,8 @@ HARDWARE_INTRINSIC(Vector64, ConvertToUInt32Native, HARDWARE_INTRINSIC(Vector64, ConvertToUInt64, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector64, ConvertToUInt64Native, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector64, Create, 8, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) +HARDWARE_INTRINSIC(Vector64, CreateAlternatingSequence, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector64, CreateGeometricSequence, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, CreateScalar, 8, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector64, CreateScalarUnsafe, 8, 1, {INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_invalid, INS_invalid, INS_fmov, INS_invalid}, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) HARDWARE_INTRINSIC(Vector64, CreateSequence, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) @@ -96,6 +102,7 @@ HARDWARE_INTRINSIC(Vector64, MinNumber, HARDWARE_INTRINSIC(Vector64, MultiplyAddEstimate, 8, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, Narrow, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, NarrowWithSaturation, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector64, Reverse, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, Round, 8, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, ShiftLeft, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, Shuffle, 8, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_CanBenefitFromConstantProp) @@ -111,9 +118,13 @@ HARDWARE_INTRINSIC(Vector64, ToScalar, HARDWARE_INTRINSIC(Vector64, ToVector128, 8, 1, {INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov}, HW_Category_SIMD, HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Vector64, ToVector128Unsafe, 8, 1, {INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov, INS_mov}, HW_Category_SIMD, HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Vector64, Truncate, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector64, UnzipEven, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector64, UnzipOdd, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, WidenLower, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector64, WidenUpper, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector64, WithElement, 8, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(Vector64, ZipLower, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector64, ZipUpper, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, get_AllBitsSet, 8, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, get_E, 8, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, get_Epsilon, 8, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) @@ -125,6 +136,7 @@ HARDWARE_INTRINSIC(Vector64, get_NegativeZero, HARDWARE_INTRINSIC(Vector64, get_One, 8, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, get_Pi, 8, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, get_PositiveInfinity, 8, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector64, get_SignSequence, 8, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, get_Tau, 8, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, get_Zero, 8, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector64, op_Addition, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) @@ -173,6 +185,10 @@ HARDWARE_INTRINSIC(Vector128, AsVector2, HARDWARE_INTRINSIC(Vector128, AsVector3, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_mov, INS_invalid}, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector128, AsVector4, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, Ceiling, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, ConcatLowerLower, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, ConcatLowerUpper, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, ConcatUpperLower, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, ConcatUpperUpper, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, ConditionalSelect, 16, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, ConvertToDouble, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, ConvertToInt32, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) @@ -185,6 +201,8 @@ HARDWARE_INTRINSIC(Vector128, ConvertToUInt32Native, HARDWARE_INTRINSIC(Vector128, ConvertToUInt64, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, ConvertToUInt64Native, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, Create, 16, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) +HARDWARE_INTRINSIC(Vector128, CreateAlternatingSequence, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, CreateGeometricSequence, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, CreateScalar, 16, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, CreateScalarUnsafe, 16, 1, {INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_ins, INS_fmov, INS_fmov}, HW_Category_SIMD, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment) HARDWARE_INTRINSIC(Vector128, CreateSequence, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) @@ -238,6 +256,7 @@ HARDWARE_INTRINSIC(Vector128, MinNumber, HARDWARE_INTRINSIC(Vector128, MultiplyAddEstimate, 16, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, Narrow, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, NarrowWithSaturation, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, Reverse, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, Round, 16, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, ShiftLeft, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, Shuffle, 16, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_CanBenefitFromConstantProp) @@ -251,11 +270,15 @@ HARDWARE_INTRINSIC(Vector128, SubtractSaturate, HARDWARE_INTRINSIC(Vector128, Sum, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, ToScalar, 16, 1, {INS_smov, INS_umov, INS_smov, INS_umov, INS_smov, INS_umov, INS_umov, INS_umov, INS_dup, INS_dup}, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SIMDScalar|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Vector128, Truncate, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, UnzipEven, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, UnzipOdd, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, WidenLower, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, WidenUpper, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, WithElement, 16, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Vector128, WithLower, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, WithUpper, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) +HARDWARE_INTRINSIC(Vector128, ZipLower, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, ZipUpper, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_AllBitsSet, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_E, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_Epsilon, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) @@ -267,6 +290,7 @@ HARDWARE_INTRINSIC(Vector128, get_NegativeZero, HARDWARE_INTRINSIC(Vector128, get_One, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_Pi, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_PositiveInfinity, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, get_SignSequence, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_Tau, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_Zero, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, op_Addition, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Helper, HW_Flag_InvalidNodeId) diff --git a/src/coreclr/jit/hwintrinsiclistxarch.h b/src/coreclr/jit/hwintrinsiclistxarch.h index de2322872c60fc..bcbc4a0b0927f1 100644 --- a/src/coreclr/jit/hwintrinsiclistxarch.h +++ b/src/coreclr/jit/hwintrinsiclistxarch.h @@ -51,6 +51,10 @@ HARDWARE_INTRINSIC(Vector128, AsVector2, HARDWARE_INTRINSIC(Vector128, AsVector3, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movups, INS_invalid}, -1, 1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Vector128, AsVector4, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, Ceiling, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, ConcatLowerLower, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, ConcatLowerUpper, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, ConcatUpperLower, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, ConcatUpperUpper, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, ConditionalSelect, 16, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector128, ConvertToDouble, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, ConvertToInt32, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) @@ -63,6 +67,8 @@ HARDWARE_INTRINSIC(Vector128, ConvertToUInt32Native, HARDWARE_INTRINSIC(Vector128, ConvertToUInt64, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, ConvertToUInt64Native, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, Create, 16, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) +HARDWARE_INTRINSIC(Vector128, CreateAlternatingSequence, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, CreateGeometricSequence, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, CreateScalar, 16, 1, {INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd}, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics) HARDWARE_INTRINSIC(Vector128, CreateScalarUnsafe, 16, 1, {INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd}, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_NoRMWSemantics) HARDWARE_INTRINSIC(Vector128, CreateSequence, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) @@ -114,6 +120,7 @@ HARDWARE_INTRINSIC(Vector128, MinNumber, HARDWARE_INTRINSIC(Vector128, MultiplyAddEstimate, 16, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, Narrow, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, NarrowWithSaturation, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector128, Reverse, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, Round, 16, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, ShiftLeft, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, Shuffle, 16, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_CanBenefitFromConstantProp) @@ -130,9 +137,13 @@ HARDWARE_INTRINSIC(Vector128, ToVector256, HARDWARE_INTRINSIC(Vector128, ToVector256Unsafe, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movdqu32, INS_movups, INS_movupd}, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_NormalizeSmallTypeToInt) HARDWARE_INTRINSIC(Vector128, ToVector512, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd}, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoRMWSemantics|HW_Flag_NormalizeSmallTypeToInt) HARDWARE_INTRINSIC(Vector128, Truncate, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, UnzipEven, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, UnzipOdd, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, WidenLower, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, WidenUpper, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector128, WithElement, 16, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector128, ZipLower, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, ZipUpper, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_AllBitsSet, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_E, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_Epsilon, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) @@ -144,6 +155,7 @@ HARDWARE_INTRINSIC(Vector128, get_NegativeZero, HARDWARE_INTRINSIC(Vector128, get_One, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_Pi, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_PositiveInfinity, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector128, get_SignSequence, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_Tau, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, get_Zero, 16, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector128, op_Addition, 16, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) @@ -188,6 +200,10 @@ HARDWARE_INTRINSIC(Vector256, AsUInt64, HARDWARE_INTRINSIC(Vector256, AsVector, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, AsVector256, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, Ceiling, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector256, ConcatLowerLower, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector256, ConcatLowerUpper, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector256, ConcatUpperLower, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector256, ConcatUpperUpper, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, ConditionalSelect, 32, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, ConvertToDouble, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector256, ConvertToInt32, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) @@ -200,6 +216,8 @@ HARDWARE_INTRINSIC(Vector256, ConvertToUInt32Native, HARDWARE_INTRINSIC(Vector256, ConvertToUInt64, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector256, ConvertToUInt64Native, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector256, Create, 32, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector256, CreateAlternatingSequence, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector256, CreateGeometricSequence, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, CreateScalar, 32, 1, {INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd}, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, CreateScalarUnsafe, 32, 1, {INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd}, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, CreateSequence, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_AvxOnlyCompatible) @@ -253,6 +271,7 @@ HARDWARE_INTRINSIC(Vector256, MinNumber, HARDWARE_INTRINSIC(Vector256, MultiplyAddEstimate, 32, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, Narrow, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, NarrowWithSaturation, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector256, Reverse, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, Round, 32, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, ShiftLeft, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, Shuffle, 32, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_CanBenefitFromConstantProp) @@ -268,11 +287,15 @@ HARDWARE_INTRINSIC(Vector256, ToScalar, HARDWARE_INTRINSIC(Vector256, ToVector512, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd}, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NormalizeSmallTypeToInt) HARDWARE_INTRINSIC(Vector256, ToVector512Unsafe, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_movdqu32, INS_movdqu32, INS_vmovdqu64, INS_vmovdqu64, INS_movups, INS_movupd}, -1, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg|HW_Flag_NormalizeSmallTypeToInt) HARDWARE_INTRINSIC(Vector256, Truncate, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector256, UnzipEven, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector256, UnzipOdd, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, WidenLower, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector256, WidenUpper, 32, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector256, WithElement, 32, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_BaseTypeFromFirstArg|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, WithLower, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) HARDWARE_INTRINSIC(Vector256, WithUpper, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen|HW_Flag_AvxOnlyCompatible) +HARDWARE_INTRINSIC(Vector256, ZipLower, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector256, ZipUpper, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, get_AllBitsSet, 32, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, get_E, 32, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, get_Epsilon, 32, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) @@ -284,6 +307,7 @@ HARDWARE_INTRINSIC(Vector256, get_NegativeZero, HARDWARE_INTRINSIC(Vector256, get_One, 32, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, get_Pi, 32, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, get_PositiveInfinity, 32, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector256, get_SignSequence, 32, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, get_Tau, 32, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, get_Zero, 32, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector256, op_Addition, 32, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) @@ -328,6 +352,10 @@ HARDWARE_INTRINSIC(Vector512, AsUInt64, HARDWARE_INTRINSIC(Vector512, AsVector, 64, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector512, AsVector512, 64, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector512, Ceiling, 64, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector512, ConcatLowerLower, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector512, ConcatLowerUpper, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector512, ConcatUpperLower, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector512, ConcatUpperUpper, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, ConditionalSelect, 64, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector512, ConvertToDouble, 64, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector512, ConvertToInt32, 64, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) @@ -340,6 +368,8 @@ HARDWARE_INTRINSIC(Vector512, ConvertToUInt32Native, HARDWARE_INTRINSIC(Vector512, ConvertToUInt64, 64, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector512, ConvertToUInt64Native, 64, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector512, Create, 64, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) +HARDWARE_INTRINSIC(Vector512, CreateAlternatingSequence, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector512, CreateGeometricSequence, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, CreateScalar, 64, 1, {INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd}, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Vector512, CreateScalarUnsafe, 64, 1, {INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd}, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Vector512, CreateSequence, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) @@ -394,6 +424,7 @@ HARDWARE_INTRINSIC(Vector512, MinNumber, HARDWARE_INTRINSIC(Vector512, MultiplyAddEstimate, 64, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, Narrow, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, NarrowWithSaturation, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Vector512, Reverse, 64, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, Round, 64, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, ShiftLeft, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, Shuffle, 64, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_CanBenefitFromConstantProp) @@ -407,11 +438,15 @@ HARDWARE_INTRINSIC(Vector512, SubtractSaturate, HARDWARE_INTRINSIC(Vector512, Sum, 64, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector512, ToScalar, 64, 1, {INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd32, INS_movd64, INS_movd64, INS_movss, INS_movsd_simd}, -1, -1, HW_Category_SIMDScalar, HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector512, Truncate, 64, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector512, UnzipEven, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector512, UnzipOdd, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, WidenLower, 64, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector512, WidenUpper, 64, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector512, WithElement, 64, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoContainment|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Vector512, WithLower, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) HARDWARE_INTRINSIC(Vector512, WithUpper, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_SpecialImport|HW_Flag_NoCodeGen) +HARDWARE_INTRINSIC(Vector512, ZipLower, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector512, ZipUpper, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, get_AllBitsSet, 64, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, get_E, 64, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, get_Epsilon, 64, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) @@ -423,6 +458,7 @@ HARDWARE_INTRINSIC(Vector512, get_NegativeZero, HARDWARE_INTRINSIC(Vector512, get_One, 64, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, get_Pi, 64, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, get_PositiveInfinity, 64, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) +HARDWARE_INTRINSIC(Vector512, get_SignSequence, 64, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, get_Tau, 64, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, get_Zero, 64, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) HARDWARE_INTRINSIC(Vector512, op_Addition, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId) diff --git a/src/coreclr/jit/hwintrinsicxarch.cpp b/src/coreclr/jit/hwintrinsicxarch.cpp index ef52964d430eac..cdf8dcda42186f 100644 --- a/src/coreclr/jit/hwintrinsicxarch.cpp +++ b/src/coreclr/jit/hwintrinsicxarch.cpp @@ -2258,6 +2258,64 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, break; } + case NI_Vector128_CreateGeometricSequence: + case NI_Vector256_CreateGeometricSequence: + case NI_Vector512_CreateGeometricSequence: + { + assert(sig->numArgs == 2); + + if (!impStackTop(0).val->OperIsConst()) + { + break; + } + + if (varTypeIsFloating(simdBaseType) && !impStackTop(1).val->OperIsConst() && + (getSIMDVectorLength(simdSize, simdBaseType) > 2)) + { + // Floating-point multiplication is not associative; use the fallback to preserve recurrence order. + break; + } + + if (!impStackTop(1).val->OperIsConst() && (simdSize == 32) && varTypeIsIntegral(simdBaseType) && + !compOpportunisticallyDependsOn(InstructionSet_AVX2)) + { + // We can't deal with TYP_SIMD32 for integral types if the compiler doesn't support AVX2 + break; + } + + impSpillSideEffect(true, stackState.esStackDepth - + 2 DEBUGARG("Spilling op1 side effects for vector CreateGeometricSequence")); + + op2 = impPopStack().val; + op1 = impPopStack().val; + + retNode = gtNewSimdCreateGeometricSequenceNode(retType, op1, op2, simdBaseType, simdSize); + break; + } + + case NI_Vector128_CreateAlternatingSequence: + case NI_Vector256_CreateAlternatingSequence: + case NI_Vector512_CreateAlternatingSequence: + { + assert(sig->numArgs == 2); + + if ((!impStackTop(1).val->OperIsConst() || !impStackTop(0).val->OperIsConst()) && (simdSize == 32) && + varTypeIsIntegral(simdBaseType) && !compOpportunisticallyDependsOn(InstructionSet_AVX2)) + { + // We can't deal with TYP_SIMD32 for integral types if the compiler doesn't support AVX2 + break; + } + + impSpillSideEffect(true, stackState.esStackDepth - + 2 DEBUGARG("Spilling op1 side effects for vector CreateAlternatingSequence")); + + op2 = impPopStack().val; + op1 = impPopStack().val; + + retNode = gtNewSimdCreateAlternatingSequenceNode(retType, op1, op2, simdBaseType, simdSize); + break; + } + case NI_Vector128_op_Division: case NI_Vector256_op_Division: case NI_Vector512_op_Division: @@ -2545,6 +2603,21 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, break; } + case NI_Vector128_get_SignSequence: + case NI_Vector256_get_SignSequence: + case NI_Vector512_get_SignSequence: + { + assert(sig->numArgs == 0); + + var_types scalarType = genActualType(simdBaseType); + GenTree* one = gtNewOneConNode(scalarType); + GenTree* negativeOne = varTypeIsFloating(simdBaseType) ? gtNewDconNode(-1.0, simdBaseType) + : gtNewAllBitsSetConNode(scalarType); + + retNode = gtNewSimdCreateAlternatingSequenceNode(retType, one, negativeOne, simdBaseType, simdSize); + break; + } + case NI_Vector128_get_NaN: case NI_Vector256_get_NaN: case NI_Vector512_get_NaN: @@ -4167,6 +4240,135 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, break; } + case NI_Vector128_ConcatLowerLower: + case NI_Vector256_ConcatLowerLower: + case NI_Vector512_ConcatLowerLower: + case NI_Vector128_ConcatLowerUpper: + case NI_Vector256_ConcatLowerUpper: + case NI_Vector512_ConcatLowerUpper: + case NI_Vector128_ConcatUpperLower: + case NI_Vector256_ConcatUpperLower: + case NI_Vector512_ConcatUpperLower: + case NI_Vector128_ConcatUpperUpper: + case NI_Vector256_ConcatUpperUpper: + case NI_Vector512_ConcatUpperUpper: + { + assert(sig->numArgs == 2); + + if (simdSize == 16) + { + bool supportsX86BaseShuffle = + (simdBaseType == TYP_INT) || (simdBaseType == TYP_UINT) || (simdBaseType == TYP_FLOAT); + + if (!supportsX86BaseShuffle && !compOpportunisticallyDependsOn(InstructionSet_AVX2)) + { + break; + } + } + + op2 = impSIMDPopStack(); + op1 = impSIMDPopStack(); + + bool leftUpper = + (intrinsic == NI_Vector128_ConcatUpperLower) || (intrinsic == NI_Vector256_ConcatUpperLower) || + (intrinsic == NI_Vector512_ConcatUpperLower) || (intrinsic == NI_Vector128_ConcatUpperUpper) || + (intrinsic == NI_Vector256_ConcatUpperUpper) || (intrinsic == NI_Vector512_ConcatUpperUpper); + bool rightUpper = + (intrinsic == NI_Vector128_ConcatLowerUpper) || (intrinsic == NI_Vector256_ConcatLowerUpper) || + (intrinsic == NI_Vector512_ConcatLowerUpper) || (intrinsic == NI_Vector128_ConcatUpperUpper) || + (intrinsic == NI_Vector256_ConcatUpperUpper) || (intrinsic == NI_Vector512_ConcatUpperUpper); + + retNode = gtNewSimdConcatNode(retType, op1, op2, simdBaseType, simdSize, leftUpper, rightUpper); + break; + } + + case NI_Vector128_ZipLower: + case NI_Vector256_ZipLower: + case NI_Vector512_ZipLower: + case NI_Vector128_ZipUpper: + case NI_Vector256_ZipUpper: + case NI_Vector512_ZipUpper: + { + assert(sig->numArgs == 2); + + if ((simdSize == 32) && varTypeIsIntegral(simdBaseType) && + !compOpportunisticallyDependsOn(InstructionSet_AVX2)) + { + break; + } + + op2 = impSIMDPopStack(); + op1 = impSIMDPopStack(); + + bool upper = (intrinsic == NI_Vector128_ZipUpper) || (intrinsic == NI_Vector256_ZipUpper) || + (intrinsic == NI_Vector512_ZipUpper); + retNode = gtNewSimdZipNode(retType, op1, op2, simdBaseType, simdSize, upper); + break; + } + + case NI_Vector128_UnzipEven: + case NI_Vector256_UnzipEven: + case NI_Vector512_UnzipEven: + case NI_Vector128_UnzipOdd: + case NI_Vector256_UnzipOdd: + case NI_Vector512_UnzipOdd: + { + assert(sig->numArgs == 2); + + if (simdSize == 16) + { + bool supportsX86BaseShuffle = + (simdBaseType == TYP_INT) || (simdBaseType == TYP_UINT) || (simdBaseType == TYP_FLOAT); + + if (!supportsX86BaseShuffle && !compOpportunisticallyDependsOn(InstructionSet_AVX2)) + { + break; + } + } + else if (simdSize > 16) + { + if (!compOpportunisticallyDependsOn(varTypeIsFloating(simdBaseType) ? InstructionSet_AVX + : InstructionSet_AVX2)) + { + break; + } + } + + op2 = impSIMDPopStack(); + op1 = impSIMDPopStack(); + + bool odd = (intrinsic == NI_Vector128_UnzipOdd) || (intrinsic == NI_Vector256_UnzipOdd) || + (intrinsic == NI_Vector512_UnzipOdd); + retNode = gtNewSimdUnzipNode(retType, op1, op2, simdBaseType, simdSize, odd); + break; + } + + case NI_Vector128_Reverse: + case NI_Vector256_Reverse: + case NI_Vector512_Reverse: + { + assert(sig->numArgs == 1); + + if (simdSize == 32) + { + if (!compOpportunisticallyDependsOn(varTypeIsFloating(simdBaseType) ? InstructionSet_AVX + : InstructionSet_AVX2)) + { + break; + } + } + + if ((simdSize == 64) && varTypeIsByte(simdBaseType) && + !compOpportunisticallyDependsOn(InstructionSet_AVX512v2)) + { + break; + } + + op1 = impSIMDPopStack(); + retNode = gtNewSimdReverseNode(retType, op1, simdBaseType, simdSize); + break; + } + case NI_Vector128_op_ExclusiveOr: case NI_Vector256_op_ExclusiveOr: case NI_Vector512_op_ExclusiveOr: diff --git a/src/libraries/System.Numerics.Vectors/ref/System.Numerics.Vectors.cs b/src/libraries/System.Numerics.Vectors/ref/System.Numerics.Vectors.cs index 630ba00644a847..6a68e7de235454 100644 --- a/src/libraries/System.Numerics.Vectors/ref/System.Numerics.Vectors.cs +++ b/src/libraries/System.Numerics.Vectors/ref/System.Numerics.Vectors.cs @@ -312,6 +312,21 @@ public static partial class Vector public static System.Numerics.Vector CreateScalar(T value) { throw null; } public static System.Numerics.Vector CreateScalarUnsafe(T value) { throw null; } public static System.Numerics.Vector CreateSequence(T start, T step) { throw null; } + public static System.Numerics.Vector CreateGeometricSequence(T initial, [System.Diagnostics.CodeAnalysis.ConstantExpected] T multiplier) { throw null; } + public static System.Numerics.Vector CreateAlternatingSequence(T even, T odd) { throw null; } + public static System.Numerics.Vector CreateHarmonicSequence(T start, T step) { throw null; } + public static System.Numerics.Vector CreateCauchySequence(T start, T step) { throw null; } + public static System.Numerics.Vector ZipLower(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector ZipUpper(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static (System.Numerics.Vector Lower, System.Numerics.Vector Upper) Zip(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector UnzipEven(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector UnzipOdd(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static (System.Numerics.Vector Even, System.Numerics.Vector Odd) Unzip(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector ConcatLowerLower(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector ConcatUpperLower(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector ConcatUpperUpper(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector ConcatLowerUpper(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Reverse(System.Numerics.Vector vector) { throw null; } public static System.Numerics.Vector DegreesToRadians(System.Numerics.Vector degrees) { throw null; } public static System.Numerics.Vector DegreesToRadians(System.Numerics.Vector degrees) { throw null; } public static System.Numerics.Vector Divide(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } @@ -1131,6 +1146,7 @@ public readonly void CopyTo(System.Span destination) { } public static System.Numerics.Vector AllBitsSet { get { throw null; } } public static int Count { get { throw null; } } public static System.Numerics.Vector Indices { get { throw null; } } + public static System.Numerics.Vector SignSequence { get { throw null; } } public static bool IsSupported { get { throw null; } } public T this[int index] { get { throw null; } } public static System.Numerics.Vector One { get { throw null; } } diff --git a/src/libraries/System.Numerics.Vectors/tests/GenericVectorTests.cs b/src/libraries/System.Numerics.Vectors/tests/GenericVectorTests.cs index f18f9ff9278419..ea809b1851c167 100644 --- a/src/libraries/System.Numerics.Vectors/tests/GenericVectorTests.cs +++ b/src/libraries/System.Numerics.Vectors/tests/GenericVectorTests.cs @@ -4531,6 +4531,152 @@ private static void TestCreateSequence(T start, T step) } } + [Fact] + public void CreateGeometricSequenceInt32Test() + { + Vector sequence = Vector.CreateGeometricSequence(1, 2); + int expected = 1; + + for (int index = 0; index < Vector.Count; index++) + { + Assert.Equal(expected, sequence.GetElement(index)); + expected *= 2; + } + } + + [Fact] + public void CreateAlternatingSequenceInt32Test() + { + Vector sequence = Vector.CreateAlternatingSequence(5, -5); + + for (int index = 0; index < Vector.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 5 : -5, sequence.GetElement(index)); + } + } + + [Fact] + public void CreateHarmonicSequenceDoubleTest() + { + Vector sequence = Vector.CreateHarmonicSequence(1.0, 1.0); + double expected = 1.0; + + for (int index = 0; index < Vector.Count; index++) + { + AssertExtensions.Equal(1.0 / expected, sequence.GetElement(index), 1e-15); + expected += 1.0; + } + } + + [Fact] + public void CreateCauchySequenceDoubleTest() + { + Vector sequence = Vector.CreateCauchySequence(1.0, 1.0); + double expected = 1.0; + + for (int index = 0; index < Vector.Count; index++) + { + AssertExtensions.Equal(Math.Sqrt(expected), sequence.GetElement(index), 1e-15); + expected += 1.0; + } + } + + [Fact] + public void SignSequenceInt32Test() + { + Vector sequence = Vector.SignSequence; + + for (int index = 0; index < Vector.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1 : -1, sequence.GetElement(index)); + } + } + + [Fact] + public void SignSequenceUInt32Test() + { + Vector sequence = Vector.SignSequence; + + for (int index = 0; index < Vector.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1u : uint.MaxValue, sequence.GetElement(index)); + } + } + + [Fact] + public void SignSequenceSingleTest() + { + Vector sequence = Vector.SignSequence; + + for (int index = 0; index < Vector.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1.0f : -1.0f, sequence.GetElement(index)); + } + } + + [Fact] + public void SignSequenceDoubleTest() + { + Vector sequence = Vector.SignSequence; + + for (int index = 0; index < Vector.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1.0 : -1.0, sequence.GetElement(index)); + } + } + + [Fact] + public void LaneOperationsInt32Test() + { + Vector left = Vector.CreateSequence(0, 1); + Vector right = Vector.CreateSequence(100, 1); + int count = Vector.Count; + int lowerCount = (count + 1) / 2; + int upperStart = count - lowerCount; + + AssertVectorEqual(CreateVector(index => ((index & 1) == 0) ? left.GetElement(index / 2) : right.GetElement(index / 2)), Vector.ZipLower(left, right)); + AssertVectorEqual(CreateVector(index => ((index & 1) == 0) ? left.GetElement(upperStart + (index / 2)) : right.GetElement(upperStart + (index / 2))), Vector.ZipUpper(left, right)); + + (Vector lower, Vector upper) = Vector.Zip(left, right); + AssertVectorEqual(Vector.ZipLower(left, right), lower); + AssertVectorEqual(Vector.ZipUpper(left, right), upper); + + AssertVectorEqual(left, Vector.UnzipEven(lower, upper)); + AssertVectorEqual(right, Vector.UnzipOdd(lower, upper)); + + (Vector even, Vector odd) = Vector.Unzip(lower, upper); + AssertVectorEqual(left, even); + AssertVectorEqual(right, odd); + + AssertVectorEqual(CreateVector(index => (index < lowerCount) ? left.GetElement(index) : right.GetElement(index - lowerCount)), Vector.ConcatLowerLower(left, right)); + AssertVectorEqual(CreateVector(index => (index < lowerCount) ? left.GetElement(upperStart + index) : right.GetElement(index - lowerCount)), Vector.ConcatUpperLower(left, right)); + AssertVectorEqual(CreateVector(index => (index < lowerCount) ? left.GetElement(upperStart + index) : right.GetElement(upperStart + index - lowerCount)), Vector.ConcatUpperUpper(left, right)); + AssertVectorEqual(CreateVector(index => (index < lowerCount) ? left.GetElement(index) : right.GetElement(upperStart + index - lowerCount)), Vector.ConcatLowerUpper(left, right)); + + AssertVectorEqual(CreateVector(index => left.GetElement(count - 1 - index)), Vector.Reverse(left)); + } + + private static Vector CreateVector(Func elementSelector) + { + int[] values = new int[Vector.Count]; + + for (int index = 0; index < values.Length; index++) + { + values[index] = elementSelector(index); + } + + return new Vector(values); + } + + private static void AssertVectorEqual(Vector expected, Vector actual) + where T : struct + { + for (int index = 0; index < Vector.Count; index++) + { + Assert.Equal(expected.GetElement(index), actual.GetElement(index)); + } + } + [Theory] [MemberData(nameof(GenericMathTestMemberData.CosDouble), MemberType = typeof(GenericMathTestMemberData))] public void CosDoubleTest(double value, double expectedResult, double variance) diff --git a/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector.cs b/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector.cs index bb14e36a8be3d5..6dd60a9a58b994 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector.cs @@ -879,6 +879,245 @@ public static Vector CreateScalarUnsafe(T value) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector CreateSequence(T start, T step) => (Vector.Indices * step) + Create(start); + /// Creates a new instance where the elements begin at a specified value and are multiplied by another specified value. + /// The type of the elements in the vector. + /// The value that element 0 will be initialized to. + /// The value that indicates how each element should be scaled from the previous. + /// A new instance with the first element initialized to and each subsequent element initialized to the value of the previous element multiplied by . + /// The type of and () is not supported. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector CreateGeometricSequence(T initial, [ConstantExpected] T multiplier) + { + int count = Vector.Count; + Unsafe.SkipInit(out Vector result); + + T value = initial; + + for (int index = 0; index < count; index++) + { + result.SetElementUnsafe(index, value); + value = Scalar.Multiply(value, multiplier); + } + + return result; + } + + /// Creates a new instance whose elements alternate between two specified values. + /// The type of the elements in the vector. + /// The value assigned to even-indexed elements. + /// The value assigned to odd-indexed elements. + /// A new instance whose even-indexed elements are initialized to and odd-indexed elements are initialized to . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector CreateAlternatingSequence(T even, T odd) + { + int count = Vector.Count; + Unsafe.SkipInit(out Vector result); + + for (int index = 0; index < count; index++) + { + result.SetElementUnsafe(index, ((index & 1) == 0) ? even : odd); + } + + return result; + } + + /// Creates a new instance whose elements are the reciprocal of an arithmetic sequence. + /// The type of the elements in the vector. + /// The value that element 0 of the arithmetic sequence will be initialized to. + /// The value that indicates how far apart each element of the arithmetic sequence should be from the previous. + /// A new instance whose elements are initialized to one divided by the corresponding element of the arithmetic sequence. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector CreateHarmonicSequence(T start, T step) => Vector.One / CreateSequence(start, step); + + /// Creates a new instance whose elements are the square root of an arithmetic sequence. + /// The type of the elements in the vector. + /// The value that element 0 of the arithmetic sequence will be initialized to. + /// The value that indicates how far apart each element of the arithmetic sequence should be from the previous. + /// A new instance whose elements are initialized to the square root of the corresponding element of the arithmetic sequence. + /// The type of and () is not supported. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector CreateCauchySequence(T start, T step) => SquareRoot(CreateSequence(start, step)); + + /// Creates a new vector by concatenating the lower halves of two vectors. + /// The type of the elements in the vector. + /// The vector that provides the lower half of the result. + /// The vector that provides the upper half of the result. + /// A new vector whose lower half comes from the lower half of and whose upper half comes from the lower half of . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector ConcatLowerLower(Vector left, Vector right) => ConcatHalves(left, right, leftUpper: false, rightUpper: false); + + /// Creates a new vector by concatenating the upper half of one vector and the lower half of another vector. + /// The type of the elements in the vector. + /// The vector that provides the lower half of the result. + /// The vector that provides the upper half of the result. + /// A new vector whose lower half comes from the upper half of and whose upper half comes from the lower half of . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector ConcatUpperLower(Vector left, Vector right) => ConcatHalves(left, right, leftUpper: true, rightUpper: false); + + /// Creates a new vector by concatenating the upper halves of two vectors. + /// The type of the elements in the vector. + /// The vector that provides the lower half of the result. + /// The vector that provides the upper half of the result. + /// A new vector whose lower half comes from the upper half of and whose upper half comes from the upper half of . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector ConcatUpperUpper(Vector left, Vector right) => ConcatHalves(left, right, leftUpper: true, rightUpper: true); + + /// Creates a new vector by concatenating the lower half of one vector and the upper half of another vector. + /// The type of the elements in the vector. + /// The vector that provides the lower half of the result. + /// The vector that provides the upper half of the result. + /// A new vector whose lower half comes from the lower half of and whose upper half comes from the upper half of . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector ConcatLowerUpper(Vector left, Vector right) => ConcatHalves(left, right, leftUpper: false, rightUpper: true); + + /// Interleaves the lower halves of two vectors. + /// The type of the elements in the vector. + /// The vector that provides the even-indexed elements. + /// The vector that provides the odd-indexed elements. + /// A new vector containing interleaved elements from the lower halves of and . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector ZipLower(Vector left, Vector right) => Zip(left, right, upper: false); + + /// Interleaves the upper halves of two vectors. + /// The type of the elements in the vector. + /// The vector that provides the even-indexed elements. + /// The vector that provides the odd-indexed elements. + /// A new vector containing interleaved elements from the upper halves of and . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector ZipUpper(Vector left, Vector right) => Zip(left, right, upper: true); + + /// Interleaves two vectors into their lower and upper halves. + /// The type of the elements in the vector. + /// The vector that provides the even-indexed elements. + /// The vector that provides the odd-indexed elements. + /// A pair of vectors containing interleaved elements from the lower and upper halves of and . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static (Vector Lower, Vector Upper) Zip(Vector left, Vector right) => (ZipLower(left, right), ZipUpper(left, right)); + + private static Vector Zip(Vector left, Vector right, bool upper) + { + int count = Vector.Count; + int lowerCount = (count + 1) / 2; + int start = upper ? count - lowerCount : 0; + + Unsafe.SkipInit(out Vector result); + + for (int index = 0; index < count; index++) + { + int elementIndex = start + (index / 2); + T value = ((index & 1) == 0) + ? left.GetElementUnsafe(elementIndex) + : right.GetElementUnsafe(elementIndex); + + result.SetElementUnsafe(index, value); + } + + return result; + } + + /// De-interleaves the even-indexed elements from two vectors. + /// The type of the elements in the vector. + /// The vector that provides the lower half of the result. + /// The vector that provides the upper half of the result. + /// A new vector containing the even-indexed elements from followed by the even-indexed elements from . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector UnzipEven(Vector left, Vector right) => Unzip(left, right, odd: false); + + /// De-interleaves the odd-indexed elements from two vectors. + /// The type of the elements in the vector. + /// The vector that provides the lower half of the result. + /// The vector that provides the upper half of the result. + /// A new vector containing the odd-indexed elements from followed by the odd-indexed elements from . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector UnzipOdd(Vector left, Vector right) => Unzip(left, right, odd: true); + + /// De-interleaves two vectors into their even-indexed and odd-indexed elements. + /// The type of the elements in the vector. + /// The vector that provides the lower half of each result. + /// The vector that provides the upper half of each result. + /// A pair of vectors containing the even-indexed and odd-indexed elements from and . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static (Vector Even, Vector Odd) Unzip(Vector left, Vector right) => (UnzipEven(left, right), UnzipOdd(left, right)); + + private static Vector Unzip(Vector left, Vector right, bool odd) + { + int count = Vector.Count; + int start = odd ? 1 : 0; + int lowerCount = (count - start + 1) / 2; + + if (lowerCount == 0) + { + return Vector.Zero; + } + + Unsafe.SkipInit(out Vector result); + + for (int index = 0; index < count; index++) + { + T value = (index < lowerCount) + ? left.GetElementUnsafe(start + (index * 2)) + : right.GetElementUnsafe(start + ((index - lowerCount) * 2)); + + result.SetElementUnsafe(index, value); + } + + return result; + } + + /// Creates a new vector with the elements of a specified vector in reverse order. + /// The type of the elements in the vector. + /// The vector whose elements will be reversed. + /// A new vector containing the elements of in reverse order. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector Reverse(Vector vector) + { + int count = Vector.Count; + Unsafe.SkipInit(out Vector result); + + for (int index = 0; index < count; index++) + { + result.SetElementUnsafe(index, vector.GetElementUnsafe(count - 1 - index)); + } + + return result; + } + + private static Vector ConcatHalves(Vector left, Vector right, bool leftUpper, bool rightUpper) + { + int count = Vector.Count; + int lowerCount = (count + 1) / 2; + int leftStart = leftUpper ? count - lowerCount : 0; + int rightStart = rightUpper ? count - lowerCount : 0; + + Unsafe.SkipInit(out Vector result); + + for (int index = 0; index < count; index++) + { + T value = (index < lowerCount) + ? left.GetElementUnsafe(leftStart + index) + : right.GetElementUnsafe(rightStart + index - lowerCount); + + result.SetElementUnsafe(index, value); + } + + return result; + } + internal static Vector DegreesToRadians(Vector degrees) where T : ITrigonometricFunctions { @@ -2969,6 +3208,7 @@ public static (Vector Sin, Vector Cos) SinCos(Vector vector /// The vector whose square root is to be computed. /// The type of the elements in the vector. /// A vector whose elements are the square root of the corresponding elements in . + /// The type of () is not supported. [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector SquareRoot(Vector value) diff --git a/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector_1.cs b/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector_1.cs index 0dd762c866e312..6744f5886ea099 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector_1.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector_1.cs @@ -164,6 +164,18 @@ public static Vector Indices } } + /// + /// Gets a new with elements that alternate between one and negative one, starting with one; + /// for unsigned element types, the negative-one value is represented as all bits set. + /// + /// The type of the vector () is not supported. + public static Vector SignSequence + { + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => Vector.CreateAlternatingSequence(Scalar.One, Scalar.Subtract(default!, Scalar.One)); + } + /// Gets true if is supported; otherwise, false. /// true if is supported; otherwise, false. public static bool IsSupported diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs index 1cebc6f55a6b81..cbac312e44db73 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs @@ -1578,6 +1578,202 @@ public static Vector128 CreateScalarUnsafe(T value) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector128 CreateSequence(T start, T step) => (Vector128.Indices * step) + Create(start); + /// Creates a new instance where the elements begin at a specified value and are multiplied by another specified value. + /// The type of the elements in the vector. + /// The value that element 0 will be initialized to. + /// The value that indicates how each element should be scaled from the previous. + /// A new instance with the first element initialized to and each subsequent element initialized to the value of the previous element multiplied by . + /// The type of and () is not supported. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 CreateGeometricSequence(T initial, [ConstantExpected] T multiplier) + { + int count = Vector128.Count; + Unsafe.SkipInit(out Vector128 result); + + T value = initial; + + for (int index = 0; index < count; index++) + { + result.SetElementUnsafe(index, value); + value = Scalar.Multiply(value, multiplier); + } + + return result; + } + + /// Creates a new instance whose elements alternate between two specified values. + /// The type of the elements in the vector. + /// The value assigned to even-indexed elements. + /// The value assigned to odd-indexed elements. + /// A new instance whose even-indexed elements are initialized to and odd-indexed elements are initialized to . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 CreateAlternatingSequence(T even, T odd) + { + int count = Vector128.Count; + Unsafe.SkipInit(out Vector128 result); + + for (int index = 0; index < count; index++) + { + result.SetElementUnsafe(index, ((index & 1) == 0) ? even : odd); + } + + return result; + } + + /// Creates a new instance whose elements are the reciprocal of an arithmetic sequence. + /// The type of the elements in the vector. + /// The value that element 0 of the arithmetic sequence will be initialized to. + /// The value that indicates how far apart each element of the arithmetic sequence should be from the previous. + /// A new instance whose elements are initialized to one divided by the corresponding element of the arithmetic sequence. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 CreateHarmonicSequence(T start, T step) => Vector128.One / CreateSequence(start, step); + + /// Creates a new instance whose elements are the square root of an arithmetic sequence. + /// The type of the elements in the vector. + /// The value that element 0 of the arithmetic sequence will be initialized to. + /// The value that indicates how far apart each element of the arithmetic sequence should be from the previous. + /// A new instance whose elements are initialized to the square root of the corresponding element of the arithmetic sequence. + /// The type of and () is not supported. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 CreateCauchySequence(T start, T step) => Sqrt(CreateSequence(start, step)); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 ConcatLowerLower(Vector128 left, Vector128 right) => ConcatHalves(left, right, leftUpper: false, rightUpper: false); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 ConcatUpperLower(Vector128 left, Vector128 right) => ConcatHalves(left, right, leftUpper: true, rightUpper: false); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 ConcatUpperUpper(Vector128 left, Vector128 right) => ConcatHalves(left, right, leftUpper: true, rightUpper: true); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 ConcatLowerUpper(Vector128 left, Vector128 right) => ConcatHalves(left, right, leftUpper: false, rightUpper: true); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 ZipLower(Vector128 left, Vector128 right) => Zip(left, right, upper: false); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 ZipUpper(Vector128 left, Vector128 right) => Zip(left, right, upper: true); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static (Vector128 Lower, Vector128 Upper) Zip(Vector128 left, Vector128 right) => (ZipLower(left, right), ZipUpper(left, right)); + + private static Vector128 Zip(Vector128 left, Vector128 right, bool upper) + { + int count = Vector128.Count; + int lowerCount = (count + 1) / 2; + int start = upper ? count - lowerCount : 0; + + Unsafe.SkipInit(out Vector128 result); + + for (int index = 0; index < count; index++) + { + int elementIndex = start + (index / 2); + T value = ((index & 1) == 0) + ? left.GetElementUnsafe(elementIndex) + : right.GetElementUnsafe(elementIndex); + + result.SetElementUnsafe(index, value); + } + + return result; + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 UnzipEven(Vector128 left, Vector128 right) => Unzip(left, right, odd: false); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 UnzipOdd(Vector128 left, Vector128 right) => Unzip(left, right, odd: true); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static (Vector128 Even, Vector128 Odd) Unzip(Vector128 left, Vector128 right) => (UnzipEven(left, right), UnzipOdd(left, right)); + + private static Vector128 Unzip(Vector128 left, Vector128 right, bool odd) + { + int count = Vector128.Count; + int start = odd ? 1 : 0; + int lowerCount = (count - start + 1) / 2; + + if (lowerCount == 0) + { + return Vector128.Zero; + } + + Unsafe.SkipInit(out Vector128 result); + + for (int index = 0; index < count; index++) + { + T value = (index < lowerCount) + ? left.GetElementUnsafe(start + (index * 2)) + : right.GetElementUnsafe(start + ((index - lowerCount) * 2)); + + result.SetElementUnsafe(index, value); + } + + return result; + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Reverse(Vector128 vector) + { + int count = Vector128.Count; + Unsafe.SkipInit(out Vector128 result); + + for (int index = 0; index < count; index++) + { + result.SetElementUnsafe(index, vector.GetElementUnsafe(count - 1 - index)); + } + + return result; + } + + private static Vector128 ConcatHalves(Vector128 left, Vector128 right, bool leftUpper, bool rightUpper) + { + int count = Vector128.Count; + int lowerCount = (count + 1) / 2; + int leftStart = leftUpper ? count - lowerCount : 0; + int rightStart = rightUpper ? count - lowerCount : 0; + + Unsafe.SkipInit(out Vector128 result); + + for (int index = 0; index < count; index++) + { + T value = (index < lowerCount) + ? left.GetElementUnsafe(leftStart + index) + : right.GetElementUnsafe(rightStart + index - lowerCount); + + result.SetElementUnsafe(index, value); + } + + return result; + } + /// [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128_1.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128_1.cs index e00dbbe7258076..b1a16747d3ee81 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128_1.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128_1.cs @@ -74,6 +74,18 @@ public static Vector128 Indices } } + /// + /// Gets a new with elements that alternate between one and negative one, starting with one; + /// for unsigned element types, the negative-one value is represented as all bits set. + /// + /// The type of the vector () is not supported. + public static Vector128 SignSequence + { + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => Vector128.CreateAlternatingSequence(Scalar.One, Scalar.Subtract(default!, Scalar.One)); + } + /// Gets true if is supported; otherwise, false. /// true if is supported; otherwise, false. public static bool IsSupported diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256.cs index cd446c7646ac03..fbbf02c28f731b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256.cs @@ -1658,6 +1658,188 @@ public static Vector256 CreateScalarUnsafe(T value) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector256 CreateSequence(T start, T step) => (Vector256.Indices * step) + Create(start); + /// Creates a new instance where the elements begin at a specified value and are multiplied by another specified value. + /// The type of the elements in the vector. + /// The value that element 0 will be initialized to. + /// The value that indicates how each element should be scaled from the previous. + /// A new instance with the first element initialized to and each subsequent element initialized to the value of the previous element multiplied by . + /// The type of and () is not supported. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 CreateGeometricSequence(T initial, [ConstantExpected] T multiplier) + { + T upperInitial = initial; + + for (int index = 0; index < Vector128.Count; index++) + { + upperInitial = Scalar.Multiply(upperInitial, multiplier); + } + + return Create( + Vector128.CreateGeometricSequence(initial, multiplier), + Vector128.CreateGeometricSequence(upperInitial, multiplier) + ); + } + + /// Creates a new instance whose elements alternate between two specified values. + /// The type of the elements in the vector. + /// The value assigned to even-indexed elements. + /// The value assigned to odd-indexed elements. + /// A new instance whose even-indexed elements are initialized to and odd-indexed elements are initialized to . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 CreateAlternatingSequence(T even, T odd) + { + Vector128 sequence = Vector128.CreateAlternatingSequence(even, odd); + return Create(sequence, sequence); + } + + /// Creates a new instance whose elements are the reciprocal of an arithmetic sequence. + /// The type of the elements in the vector. + /// The value that element 0 of the arithmetic sequence will be initialized to. + /// The value that indicates how far apart each element of the arithmetic sequence should be from the previous. + /// A new instance whose elements are initialized to one divided by the corresponding element of the arithmetic sequence. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 CreateHarmonicSequence(T start, T step) + { + if (IsHardwareAccelerated) + { + return Vector256.One / CreateSequence(start, step); + } + + T upperStart = Scalar.Add(start, Scalar.Multiply(Scalar.Convert(Vector128.Count), step)); + + return Create( + Vector128.CreateHarmonicSequence(start, step), + Vector128.CreateHarmonicSequence(upperStart, step) + ); + } + + /// Creates a new instance whose elements are the square root of an arithmetic sequence. + /// The type of the elements in the vector. + /// The value that element 0 of the arithmetic sequence will be initialized to. + /// The value that indicates how far apart each element of the arithmetic sequence should be from the previous. + /// A new instance whose elements are initialized to the square root of the corresponding element of the arithmetic sequence. + /// The type of and () is not supported. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 CreateCauchySequence(T start, T step) + { + if (IsHardwareAccelerated) + { + return Sqrt(CreateSequence(start, step)); + } + + T upperStart = Scalar.Add(start, Scalar.Multiply(Scalar.Convert(Vector128.Count), step)); + + return Create( + Vector128.CreateCauchySequence(start, step), + Vector128.CreateCauchySequence(upperStart, step) + ); + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 ConcatLowerLower(Vector256 left, Vector256 right) => Create(left._lower, right._lower); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 ConcatUpperLower(Vector256 left, Vector256 right) => Create(left._upper, right._lower); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 ConcatUpperUpper(Vector256 left, Vector256 right) => Create(left._upper, right._upper); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 ConcatLowerUpper(Vector256 left, Vector256 right) => Create(left._lower, right._upper); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 ZipLower(Vector256 left, Vector256 right) => Create( + Vector128.ZipLower(left._lower, right._lower), + Vector128.ZipUpper(left._lower, right._lower) + ); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 ZipUpper(Vector256 left, Vector256 right) => Create( + Vector128.ZipLower(left._upper, right._upper), + Vector128.ZipUpper(left._upper, right._upper) + ); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static (Vector256 Lower, Vector256 Upper) Zip(Vector256 left, Vector256 right) + { + if (Avx2.IsSupported && ((typeof(T) == typeof(int)) || (typeof(T) == typeof(uint)))) + { + Vector256 lower = Avx2.UnpackLow(left.AsInt32(), right.AsInt32()); + Vector256 upper = Avx2.UnpackHigh(left.AsInt32(), right.AsInt32()); + + return ( + Avx2.Permute2x128(lower, upper, 0x20).As(), + Avx2.Permute2x128(lower, upper, 0x31).As() + ); + } + + return (ZipLower(left, right), ZipUpper(left, right)); + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 UnzipEven(Vector256 left, Vector256 right) => Create( + Vector128.UnzipEven(left._lower, left._upper), + Vector128.UnzipEven(right._lower, right._upper) + ); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 UnzipOdd(Vector256 left, Vector256 right) => Create( + Vector128.UnzipOdd(left._lower, left._upper), + Vector128.UnzipOdd(right._lower, right._upper) + ); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static (Vector256 Even, Vector256 Odd) Unzip(Vector256 left, Vector256 right) + { + if (Avx2.IsSupported && ((typeof(T) == typeof(int)) || (typeof(T) == typeof(uint)))) + { + Vector256 leftUnzip = Avx2.Shuffle(left.AsInt32(), 0xD8); + leftUnzip = Avx2.Permute4x64(leftUnzip.AsInt64(), 0xD8).AsInt32(); + + Vector256 rightUnzip = Avx2.Shuffle(right.AsInt32(), 0xD8); + rightUnzip = Avx2.Permute4x64(rightUnzip.AsInt64(), 0xD8).AsInt32(); + + return ( + Avx2.Permute2x128(leftUnzip, rightUnzip, 0x20).As(), + Avx2.Permute2x128(leftUnzip, rightUnzip, 0x31).As() + ); + } + + return (UnzipEven(left, right), UnzipOdd(left, right)); + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Reverse(Vector256 vector) => Create( + Vector128.Reverse(vector._upper), + Vector128.Reverse(vector._lower) + ); + /// [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256_1.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256_1.cs index 0418b0d01a79bc..1b0b371ab1c710 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256_1.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256_1.cs @@ -73,6 +73,18 @@ public static Vector256 Indices } } + /// + /// Gets a new with elements that alternate between one and negative one, starting with one; + /// for unsigned element types, the negative-one value is represented as all bits set. + /// + /// The type of the vector () is not supported. + public static Vector256 SignSequence + { + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => Vector256.CreateAlternatingSequence(Scalar.One, Scalar.Subtract(default!, Scalar.One)); + } + /// Gets true if is supported; otherwise, false. /// true if is supported; otherwise, false. public static bool IsSupported diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512.cs index 6e71109c3ab9c5..5b6bd969079906 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512.cs @@ -6,6 +6,7 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; namespace System.Runtime.Intrinsics { @@ -1690,6 +1691,210 @@ public static Vector512 CreateScalarUnsafe(T value) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector512 CreateSequence(T start, T step) => (Vector512.Indices * step) + Create(start); + /// Creates a new instance where the elements begin at a specified value and are multiplied by another specified value. + /// The type of the elements in the vector. + /// The value that element 0 will be initialized to. + /// The value that indicates how each element should be scaled from the previous. + /// A new instance with the first element initialized to and each subsequent element initialized to the value of the previous element multiplied by . + /// The type of and () is not supported. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 CreateGeometricSequence(T initial, [ConstantExpected] T multiplier) + { + T upperInitial = initial; + + for (int index = 0; index < Vector256.Count; index++) + { + upperInitial = Scalar.Multiply(upperInitial, multiplier); + } + + return Create( + Vector256.CreateGeometricSequence(initial, multiplier), + Vector256.CreateGeometricSequence(upperInitial, multiplier) + ); + } + + /// Creates a new instance whose elements alternate between two specified values. + /// The type of the elements in the vector. + /// The value assigned to even-indexed elements. + /// The value assigned to odd-indexed elements. + /// A new instance whose even-indexed elements are initialized to and odd-indexed elements are initialized to . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 CreateAlternatingSequence(T even, T odd) + { + Vector256 sequence = Vector256.CreateAlternatingSequence(even, odd); + return Create(sequence, sequence); + } + + /// Creates a new instance whose elements are the reciprocal of an arithmetic sequence. + /// The type of the elements in the vector. + /// The value that element 0 of the arithmetic sequence will be initialized to. + /// The value that indicates how far apart each element of the arithmetic sequence should be from the previous. + /// A new instance whose elements are initialized to one divided by the corresponding element of the arithmetic sequence. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 CreateHarmonicSequence(T start, T step) + { + if (IsHardwareAccelerated) + { + return Vector512.One / CreateSequence(start, step); + } + + T upperStart = Scalar.Add(start, Scalar.Multiply(Scalar.Convert(Vector256.Count), step)); + + return Create( + Vector256.CreateHarmonicSequence(start, step), + Vector256.CreateHarmonicSequence(upperStart, step) + ); + } + + /// Creates a new instance whose elements are the square root of an arithmetic sequence. + /// The type of the elements in the vector. + /// The value that element 0 of the arithmetic sequence will be initialized to. + /// The value that indicates how far apart each element of the arithmetic sequence should be from the previous. + /// A new instance whose elements are initialized to the square root of the corresponding element of the arithmetic sequence. + /// The type of and () is not supported. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 CreateCauchySequence(T start, T step) + { + if (IsHardwareAccelerated) + { + return Sqrt(CreateSequence(start, step)); + } + + T upperStart = Scalar.Add(start, Scalar.Multiply(Scalar.Convert(Vector256.Count), step)); + + return Create( + Vector256.CreateCauchySequence(start, step), + Vector256.CreateCauchySequence(upperStart, step) + ); + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 ConcatLowerLower(Vector512 left, Vector512 right) => Create(left._lower, right._lower); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 ConcatUpperLower(Vector512 left, Vector512 right) => Create(left._upper, right._lower); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 ConcatUpperUpper(Vector512 left, Vector512 right) => Create(left._upper, right._upper); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 ConcatLowerUpper(Vector512 left, Vector512 right) => Create(left._lower, right._upper); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 ZipLower(Vector512 left, Vector512 right) => Create( + Vector256.ZipLower(left._lower, right._lower), + Vector256.ZipUpper(left._lower, right._lower) + ); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 ZipUpper(Vector512 left, Vector512 right) => Create( + Vector256.ZipLower(left._upper, right._upper), + Vector256.ZipUpper(left._upper, right._upper) + ); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static (Vector512 Lower, Vector512 Upper) Zip(Vector512 left, Vector512 right) + { + if (Avx512F.IsSupported && ((typeof(T) == typeof(int)) || (typeof(T) == typeof(uint)))) + { + Vector512 lower = Avx512F.UnpackLow(left.AsInt32(), right.AsInt32()); + Vector512 upper = Avx512F.UnpackHigh(left.AsInt32(), right.AsInt32()); + + Vector512 lowerResult = Avx512F.Shuffle4x128(lower, upper, 0x44); + lowerResult = Avx512F.Shuffle4x128(lowerResult, lowerResult, 0xD8); + + Vector512 upperResult = Avx512F.Shuffle4x128(lower, upper, 0xEE); + upperResult = Avx512F.Shuffle4x128(upperResult, upperResult, 0xD8); + + return (lowerResult.As(), upperResult.As()); + } + + if (IsHardwareAccelerated) + { + return (ZipLower(left, right), ZipUpper(left, right)); + } + + (Vector256 lower0, Vector256 upper0) = Vector256.Zip(left._lower, right._lower); + (Vector256 lower1, Vector256 upper1) = Vector256.Zip(left._upper, right._upper); + + return (Create(lower0, upper0), Create(lower1, upper1)); + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 UnzipEven(Vector512 left, Vector512 right) => Create( + Vector256.UnzipEven(left._lower, left._upper), + Vector256.UnzipEven(right._lower, right._upper) + ); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 UnzipOdd(Vector512 left, Vector512 right) => Create( + Vector256.UnzipOdd(left._lower, left._upper), + Vector256.UnzipOdd(right._lower, right._upper) + ); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static (Vector512 Even, Vector512 Odd) Unzip(Vector512 left, Vector512 right) + { + if (Avx512F.IsSupported && ((typeof(T) == typeof(int)) || (typeof(T) == typeof(uint)))) + { + Vector512 evenIndices = Vector512.Create( + 0, 2, 4, 6, 8, 10, 12, 14, + 16, 18, 20, 22, 24, 26, 28, 30 + ); + Vector512 oddIndices = Vector512.Create( + 1, 3, 5, 7, 9, 11, 13, 15, + 17, 19, 21, 23, 25, 27, 29, 31 + ); + + Vector512 even = Avx512F.PermuteVar16x32x2(left.AsInt32(), evenIndices, right.AsInt32()); + Vector512 odd = Avx512F.PermuteVar16x32x2(left.AsInt32(), oddIndices, right.AsInt32()); + + return (even.As(), odd.As()); + } + + if (IsHardwareAccelerated) + { + return (UnzipEven(left, right), UnzipOdd(left, right)); + } + + (Vector256 even0, Vector256 odd0) = Vector256.Unzip(left._lower, left._upper); + (Vector256 even1, Vector256 odd1) = Vector256.Unzip(right._lower, right._upper); + + return (Create(even0, even1), Create(odd0, odd1)); + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Reverse(Vector512 vector) => Create( + Vector256.Reverse(vector._upper), + Vector256.Reverse(vector._lower) + ); + /// [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512_1.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512_1.cs index 3841113a21ec11..cb0b30cc0d8906 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512_1.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512_1.cs @@ -73,6 +73,18 @@ public static Vector512 Indices } } + /// + /// Gets a new with elements that alternate between one and negative one, starting with one; + /// for unsigned element types, the negative-one value is represented as all bits set. + /// + /// The type of the vector () is not supported. + public static Vector512 SignSequence + { + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => Vector512.CreateAlternatingSequence(Scalar.One, Scalar.Subtract(default!, Scalar.One)); + } + /// Gets true if is supported; otherwise, false. /// true if is supported; otherwise, false. public static bool IsSupported diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64.cs index 8655d9778f0529..d25feac61ba53e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64.cs @@ -1386,6 +1386,202 @@ public static Vector64 CreateScalarUnsafe(T value) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector64 CreateSequence(T start, T step) => (Vector64.Indices * step) + Create(start); + /// Creates a new instance where the elements begin at a specified value and are multiplied by another specified value. + /// The type of the elements in the vector. + /// The value that element 0 will be initialized to. + /// The value that indicates how each element should be scaled from the previous. + /// A new instance with the first element initialized to and each subsequent element initialized to the value of the previous element multiplied by . + /// The type of and () is not supported. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 CreateGeometricSequence(T initial, [ConstantExpected] T multiplier) + { + int count = Vector64.Count; + Unsafe.SkipInit(out Vector64 result); + + T value = initial; + + for (int index = 0; index < count; index++) + { + result.SetElementUnsafe(index, value); + value = Scalar.Multiply(value, multiplier); + } + + return result; + } + + /// Creates a new instance whose elements alternate between two specified values. + /// The type of the elements in the vector. + /// The value assigned to even-indexed elements. + /// The value assigned to odd-indexed elements. + /// A new instance whose even-indexed elements are initialized to and odd-indexed elements are initialized to . + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 CreateAlternatingSequence(T even, T odd) + { + int count = Vector64.Count; + Unsafe.SkipInit(out Vector64 result); + + for (int index = 0; index < count; index++) + { + result.SetElementUnsafe(index, ((index & 1) == 0) ? even : odd); + } + + return result; + } + + /// Creates a new instance whose elements are the reciprocal of an arithmetic sequence. + /// The type of the elements in the vector. + /// The value that element 0 of the arithmetic sequence will be initialized to. + /// The value that indicates how far apart each element of the arithmetic sequence should be from the previous. + /// A new instance whose elements are initialized to one divided by the corresponding element of the arithmetic sequence. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 CreateHarmonicSequence(T start, T step) => Vector64.One / CreateSequence(start, step); + + /// Creates a new instance whose elements are the square root of an arithmetic sequence. + /// The type of the elements in the vector. + /// The value that element 0 of the arithmetic sequence will be initialized to. + /// The value that indicates how far apart each element of the arithmetic sequence should be from the previous. + /// A new instance whose elements are initialized to the square root of the corresponding element of the arithmetic sequence. + /// The type of and () is not supported. + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 CreateCauchySequence(T start, T step) => Sqrt(CreateSequence(start, step)); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 ConcatLowerLower(Vector64 left, Vector64 right) => ConcatHalves(left, right, leftUpper: false, rightUpper: false); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 ConcatUpperLower(Vector64 left, Vector64 right) => ConcatHalves(left, right, leftUpper: true, rightUpper: false); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 ConcatUpperUpper(Vector64 left, Vector64 right) => ConcatHalves(left, right, leftUpper: true, rightUpper: true); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 ConcatLowerUpper(Vector64 left, Vector64 right) => ConcatHalves(left, right, leftUpper: false, rightUpper: true); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 ZipLower(Vector64 left, Vector64 right) => Zip(left, right, upper: false); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 ZipUpper(Vector64 left, Vector64 right) => Zip(left, right, upper: true); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static (Vector64 Lower, Vector64 Upper) Zip(Vector64 left, Vector64 right) => (ZipLower(left, right), ZipUpper(left, right)); + + private static Vector64 Zip(Vector64 left, Vector64 right, bool upper) + { + int count = Vector64.Count; + int lowerCount = (count + 1) / 2; + int start = upper ? count - lowerCount : 0; + + Unsafe.SkipInit(out Vector64 result); + + for (int index = 0; index < count; index++) + { + int elementIndex = start + (index / 2); + T value = ((index & 1) == 0) + ? left.GetElementUnsafe(elementIndex) + : right.GetElementUnsafe(elementIndex); + + result.SetElementUnsafe(index, value); + } + + return result; + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 UnzipEven(Vector64 left, Vector64 right) => Unzip(left, right, odd: false); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 UnzipOdd(Vector64 left, Vector64 right) => Unzip(left, right, odd: true); + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static (Vector64 Even, Vector64 Odd) Unzip(Vector64 left, Vector64 right) => (UnzipEven(left, right), UnzipOdd(left, right)); + + private static Vector64 Unzip(Vector64 left, Vector64 right, bool odd) + { + int count = Vector64.Count; + int start = odd ? 1 : 0; + int lowerCount = (count - start + 1) / 2; + + if (lowerCount == 0) + { + return Vector64.Zero; + } + + Unsafe.SkipInit(out Vector64 result); + + for (int index = 0; index < count; index++) + { + T value = (index < lowerCount) + ? left.GetElementUnsafe(start + (index * 2)) + : right.GetElementUnsafe(start + ((index - lowerCount) * 2)); + + result.SetElementUnsafe(index, value); + } + + return result; + } + + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector64 Reverse(Vector64 vector) + { + int count = Vector64.Count; + Unsafe.SkipInit(out Vector64 result); + + for (int index = 0; index < count; index++) + { + result.SetElementUnsafe(index, vector.GetElementUnsafe(count - 1 - index)); + } + + return result; + } + + private static Vector64 ConcatHalves(Vector64 left, Vector64 right, bool leftUpper, bool rightUpper) + { + int count = Vector64.Count; + int lowerCount = (count + 1) / 2; + int leftStart = leftUpper ? count - lowerCount : 0; + int rightStart = rightUpper ? count - lowerCount : 0; + + Unsafe.SkipInit(out Vector64 result); + + for (int index = 0; index < count; index++) + { + T value = (index < lowerCount) + ? left.GetElementUnsafe(leftStart + index) + : right.GetElementUnsafe(rightStart + index - lowerCount); + + result.SetElementUnsafe(index, value); + } + + return result; + } + internal static Vector64 DegreesToRadians(Vector64 degrees) where T : ITrigonometricFunctions { diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64_1.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64_1.cs index df7a7dafddd0f2..9df3a302d09f3c 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64_1.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64_1.cs @@ -73,6 +73,18 @@ public static Vector64 Indices } } + /// + /// Gets a new with elements that alternate between one and negative one, starting with one; + /// for unsigned element types, the negative-one value is represented as all bits set. + /// + /// The type of the vector () is not supported. + public static Vector64 SignSequence + { + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => Vector64.CreateAlternatingSequence(Scalar.One, Scalar.Subtract(default!, Scalar.One)); + } + /// Gets true if is supported; otherwise, false. /// true if is supported; otherwise, false. public static bool IsSupported diff --git a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs index de7788351ca6da..3bdc8526df2a43 100644 --- a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs +++ b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs @@ -171,6 +171,21 @@ public static void CopyTo(this System.Runtime.Intrinsics.Vector128 vector, public static System.Runtime.Intrinsics.Vector128 CreateScalarUnsafe(ulong value) { throw null; } public static System.Runtime.Intrinsics.Vector128 CreateScalarUnsafe(T value) { throw null; } public static System.Runtime.Intrinsics.Vector128 CreateSequence(T start, T step) { throw null; } + public static System.Runtime.Intrinsics.Vector128 CreateGeometricSequence(T initial, [System.Diagnostics.CodeAnalysis.ConstantExpected] T multiplier) { throw null; } + public static System.Runtime.Intrinsics.Vector128 CreateAlternatingSequence(T even, T odd) { throw null; } + public static System.Runtime.Intrinsics.Vector128 CreateHarmonicSequence(T start, T step) { throw null; } + public static System.Runtime.Intrinsics.Vector128 CreateCauchySequence(T start, T step) { throw null; } + public static System.Runtime.Intrinsics.Vector128 ZipLower(System.Runtime.Intrinsics.Vector128 left, System.Runtime.Intrinsics.Vector128 right) { throw null; } + public static System.Runtime.Intrinsics.Vector128 ZipUpper(System.Runtime.Intrinsics.Vector128 left, System.Runtime.Intrinsics.Vector128 right) { throw null; } + public static (System.Runtime.Intrinsics.Vector128 Lower, System.Runtime.Intrinsics.Vector128 Upper) Zip(System.Runtime.Intrinsics.Vector128 left, System.Runtime.Intrinsics.Vector128 right) { throw null; } + public static System.Runtime.Intrinsics.Vector128 UnzipEven(System.Runtime.Intrinsics.Vector128 left, System.Runtime.Intrinsics.Vector128 right) { throw null; } + public static System.Runtime.Intrinsics.Vector128 UnzipOdd(System.Runtime.Intrinsics.Vector128 left, System.Runtime.Intrinsics.Vector128 right) { throw null; } + public static (System.Runtime.Intrinsics.Vector128 Even, System.Runtime.Intrinsics.Vector128 Odd) Unzip(System.Runtime.Intrinsics.Vector128 left, System.Runtime.Intrinsics.Vector128 right) { throw null; } + public static System.Runtime.Intrinsics.Vector128 ConcatLowerLower(System.Runtime.Intrinsics.Vector128 left, System.Runtime.Intrinsics.Vector128 right) { throw null; } + public static System.Runtime.Intrinsics.Vector128 ConcatUpperLower(System.Runtime.Intrinsics.Vector128 left, System.Runtime.Intrinsics.Vector128 right) { throw null; } + public static System.Runtime.Intrinsics.Vector128 ConcatUpperUpper(System.Runtime.Intrinsics.Vector128 left, System.Runtime.Intrinsics.Vector128 right) { throw null; } + public static System.Runtime.Intrinsics.Vector128 ConcatLowerUpper(System.Runtime.Intrinsics.Vector128 left, System.Runtime.Intrinsics.Vector128 right) { throw null; } + public static System.Runtime.Intrinsics.Vector128 Reverse(System.Runtime.Intrinsics.Vector128 vector) { throw null; } public static System.Runtime.Intrinsics.Vector128 Create(System.ReadOnlySpan values) { throw null; } public static System.Runtime.Intrinsics.Vector128 Create(System.Runtime.Intrinsics.Vector64 value) { throw null; } public static System.Runtime.Intrinsics.Vector128 Create(System.Runtime.Intrinsics.Vector64 lower, System.Runtime.Intrinsics.Vector64 upper) { throw null; } @@ -444,6 +459,7 @@ public static void CopyTo(this System.Runtime.Intrinsics.Vector128 vector, public static System.Runtime.Intrinsics.Vector128 AllBitsSet { get { throw null; } } public static int Count { get { throw null; } } public static System.Runtime.Intrinsics.Vector128 Indices { get { throw null; } } + public static System.Runtime.Intrinsics.Vector128 SignSequence { get { throw null; } } public static bool IsSupported { get { throw null; } } public static System.Runtime.Intrinsics.Vector128 One { get { throw null; } } public static System.Runtime.Intrinsics.Vector128 Zero { get { throw null; } } @@ -622,6 +638,21 @@ public static void CopyTo(this System.Runtime.Intrinsics.Vector256 vector, public static System.Runtime.Intrinsics.Vector256 CreateScalarUnsafe(ulong value) { throw null; } public static System.Runtime.Intrinsics.Vector256 CreateScalarUnsafe(T value) { throw null; } public static System.Runtime.Intrinsics.Vector256 CreateSequence(T start, T step) { throw null; } + public static System.Runtime.Intrinsics.Vector256 CreateGeometricSequence(T initial, [System.Diagnostics.CodeAnalysis.ConstantExpected] T multiplier) { throw null; } + public static System.Runtime.Intrinsics.Vector256 CreateAlternatingSequence(T even, T odd) { throw null; } + public static System.Runtime.Intrinsics.Vector256 CreateHarmonicSequence(T start, T step) { throw null; } + public static System.Runtime.Intrinsics.Vector256 CreateCauchySequence(T start, T step) { throw null; } + public static System.Runtime.Intrinsics.Vector256 ZipLower(System.Runtime.Intrinsics.Vector256 left, System.Runtime.Intrinsics.Vector256 right) { throw null; } + public static System.Runtime.Intrinsics.Vector256 ZipUpper(System.Runtime.Intrinsics.Vector256 left, System.Runtime.Intrinsics.Vector256 right) { throw null; } + public static (System.Runtime.Intrinsics.Vector256 Lower, System.Runtime.Intrinsics.Vector256 Upper) Zip(System.Runtime.Intrinsics.Vector256 left, System.Runtime.Intrinsics.Vector256 right) { throw null; } + public static System.Runtime.Intrinsics.Vector256 UnzipEven(System.Runtime.Intrinsics.Vector256 left, System.Runtime.Intrinsics.Vector256 right) { throw null; } + public static System.Runtime.Intrinsics.Vector256 UnzipOdd(System.Runtime.Intrinsics.Vector256 left, System.Runtime.Intrinsics.Vector256 right) { throw null; } + public static (System.Runtime.Intrinsics.Vector256 Even, System.Runtime.Intrinsics.Vector256 Odd) Unzip(System.Runtime.Intrinsics.Vector256 left, System.Runtime.Intrinsics.Vector256 right) { throw null; } + public static System.Runtime.Intrinsics.Vector256 ConcatLowerLower(System.Runtime.Intrinsics.Vector256 left, System.Runtime.Intrinsics.Vector256 right) { throw null; } + public static System.Runtime.Intrinsics.Vector256 ConcatUpperLower(System.Runtime.Intrinsics.Vector256 left, System.Runtime.Intrinsics.Vector256 right) { throw null; } + public static System.Runtime.Intrinsics.Vector256 ConcatUpperUpper(System.Runtime.Intrinsics.Vector256 left, System.Runtime.Intrinsics.Vector256 right) { throw null; } + public static System.Runtime.Intrinsics.Vector256 ConcatLowerUpper(System.Runtime.Intrinsics.Vector256 left, System.Runtime.Intrinsics.Vector256 right) { throw null; } + public static System.Runtime.Intrinsics.Vector256 Reverse(System.Runtime.Intrinsics.Vector256 vector) { throw null; } public static System.Runtime.Intrinsics.Vector256 Create(System.ReadOnlySpan values) { throw null; } public static System.Runtime.Intrinsics.Vector256 Create(System.Runtime.Intrinsics.Vector64 value) { throw null; } public static System.Runtime.Intrinsics.Vector256 Create(System.Runtime.Intrinsics.Vector128 value) { throw null; } @@ -896,6 +927,7 @@ public static void CopyTo(this System.Runtime.Intrinsics.Vector256 vector, public static System.Runtime.Intrinsics.Vector256 AllBitsSet { get { throw null; } } public static int Count { get { throw null; } } public static System.Runtime.Intrinsics.Vector256 Indices { get { throw null; } } + public static System.Runtime.Intrinsics.Vector256 SignSequence { get { throw null; } } public static bool IsSupported { get { throw null; } } public static System.Runtime.Intrinsics.Vector256 One { get { throw null; } } public static System.Runtime.Intrinsics.Vector256 Zero { get { throw null; } } @@ -1074,6 +1106,21 @@ public static void CopyTo(this System.Runtime.Intrinsics.Vector512 vector, public static System.Runtime.Intrinsics.Vector512 CreateScalarUnsafe(ulong value) { throw null; } public static System.Runtime.Intrinsics.Vector512 CreateScalarUnsafe(T value) { throw null; } public static System.Runtime.Intrinsics.Vector512 CreateSequence(T start, T step) { throw null; } + public static System.Runtime.Intrinsics.Vector512 CreateGeometricSequence(T initial, [System.Diagnostics.CodeAnalysis.ConstantExpected] T multiplier) { throw null; } + public static System.Runtime.Intrinsics.Vector512 CreateAlternatingSequence(T even, T odd) { throw null; } + public static System.Runtime.Intrinsics.Vector512 CreateHarmonicSequence(T start, T step) { throw null; } + public static System.Runtime.Intrinsics.Vector512 CreateCauchySequence(T start, T step) { throw null; } + public static System.Runtime.Intrinsics.Vector512 ZipLower(System.Runtime.Intrinsics.Vector512 left, System.Runtime.Intrinsics.Vector512 right) { throw null; } + public static System.Runtime.Intrinsics.Vector512 ZipUpper(System.Runtime.Intrinsics.Vector512 left, System.Runtime.Intrinsics.Vector512 right) { throw null; } + public static (System.Runtime.Intrinsics.Vector512 Lower, System.Runtime.Intrinsics.Vector512 Upper) Zip(System.Runtime.Intrinsics.Vector512 left, System.Runtime.Intrinsics.Vector512 right) { throw null; } + public static System.Runtime.Intrinsics.Vector512 UnzipEven(System.Runtime.Intrinsics.Vector512 left, System.Runtime.Intrinsics.Vector512 right) { throw null; } + public static System.Runtime.Intrinsics.Vector512 UnzipOdd(System.Runtime.Intrinsics.Vector512 left, System.Runtime.Intrinsics.Vector512 right) { throw null; } + public static (System.Runtime.Intrinsics.Vector512 Even, System.Runtime.Intrinsics.Vector512 Odd) Unzip(System.Runtime.Intrinsics.Vector512 left, System.Runtime.Intrinsics.Vector512 right) { throw null; } + public static System.Runtime.Intrinsics.Vector512 ConcatLowerLower(System.Runtime.Intrinsics.Vector512 left, System.Runtime.Intrinsics.Vector512 right) { throw null; } + public static System.Runtime.Intrinsics.Vector512 ConcatUpperLower(System.Runtime.Intrinsics.Vector512 left, System.Runtime.Intrinsics.Vector512 right) { throw null; } + public static System.Runtime.Intrinsics.Vector512 ConcatUpperUpper(System.Runtime.Intrinsics.Vector512 left, System.Runtime.Intrinsics.Vector512 right) { throw null; } + public static System.Runtime.Intrinsics.Vector512 ConcatLowerUpper(System.Runtime.Intrinsics.Vector512 left, System.Runtime.Intrinsics.Vector512 right) { throw null; } + public static System.Runtime.Intrinsics.Vector512 Reverse(System.Runtime.Intrinsics.Vector512 vector) { throw null; } public static System.Runtime.Intrinsics.Vector512 Create(System.ReadOnlySpan values) { throw null; } public static System.Runtime.Intrinsics.Vector512 Create(System.Runtime.Intrinsics.Vector64 value) { throw null; } public static System.Runtime.Intrinsics.Vector512 Create(System.Runtime.Intrinsics.Vector128 value) { throw null; } @@ -1347,6 +1394,7 @@ public static void CopyTo(this System.Runtime.Intrinsics.Vector512 vector, public static System.Runtime.Intrinsics.Vector512 AllBitsSet { get { throw null; } } public static int Count { get { throw null; } } public static System.Runtime.Intrinsics.Vector512 Indices { get { throw null; } } + public static System.Runtime.Intrinsics.Vector512 SignSequence { get { throw null; } } public static bool IsSupported { get { throw null; } } public static System.Runtime.Intrinsics.Vector512 One { get { throw null; } } public static System.Runtime.Intrinsics.Vector512 Zero { get { throw null; } } @@ -1502,6 +1550,21 @@ public static void CopyTo(this System.Runtime.Intrinsics.Vector64 vector, public static System.Runtime.Intrinsics.Vector64 CreateScalarUnsafe(ulong value) { throw null; } public static System.Runtime.Intrinsics.Vector64 CreateScalarUnsafe(T value) { throw null; } public static System.Runtime.Intrinsics.Vector64 CreateSequence(T start, T step) { throw null; } + public static System.Runtime.Intrinsics.Vector64 CreateGeometricSequence(T initial, [System.Diagnostics.CodeAnalysis.ConstantExpected] T multiplier) { throw null; } + public static System.Runtime.Intrinsics.Vector64 CreateAlternatingSequence(T even, T odd) { throw null; } + public static System.Runtime.Intrinsics.Vector64 CreateHarmonicSequence(T start, T step) { throw null; } + public static System.Runtime.Intrinsics.Vector64 CreateCauchySequence(T start, T step) { throw null; } + public static System.Runtime.Intrinsics.Vector64 ZipLower(System.Runtime.Intrinsics.Vector64 left, System.Runtime.Intrinsics.Vector64 right) { throw null; } + public static System.Runtime.Intrinsics.Vector64 ZipUpper(System.Runtime.Intrinsics.Vector64 left, System.Runtime.Intrinsics.Vector64 right) { throw null; } + public static (System.Runtime.Intrinsics.Vector64 Lower, System.Runtime.Intrinsics.Vector64 Upper) Zip(System.Runtime.Intrinsics.Vector64 left, System.Runtime.Intrinsics.Vector64 right) { throw null; } + public static System.Runtime.Intrinsics.Vector64 UnzipEven(System.Runtime.Intrinsics.Vector64 left, System.Runtime.Intrinsics.Vector64 right) { throw null; } + public static System.Runtime.Intrinsics.Vector64 UnzipOdd(System.Runtime.Intrinsics.Vector64 left, System.Runtime.Intrinsics.Vector64 right) { throw null; } + public static (System.Runtime.Intrinsics.Vector64 Even, System.Runtime.Intrinsics.Vector64 Odd) Unzip(System.Runtime.Intrinsics.Vector64 left, System.Runtime.Intrinsics.Vector64 right) { throw null; } + public static System.Runtime.Intrinsics.Vector64 ConcatLowerLower(System.Runtime.Intrinsics.Vector64 left, System.Runtime.Intrinsics.Vector64 right) { throw null; } + public static System.Runtime.Intrinsics.Vector64 ConcatUpperLower(System.Runtime.Intrinsics.Vector64 left, System.Runtime.Intrinsics.Vector64 right) { throw null; } + public static System.Runtime.Intrinsics.Vector64 ConcatUpperUpper(System.Runtime.Intrinsics.Vector64 left, System.Runtime.Intrinsics.Vector64 right) { throw null; } + public static System.Runtime.Intrinsics.Vector64 ConcatLowerUpper(System.Runtime.Intrinsics.Vector64 left, System.Runtime.Intrinsics.Vector64 right) { throw null; } + public static System.Runtime.Intrinsics.Vector64 Reverse(System.Runtime.Intrinsics.Vector64 vector) { throw null; } public static System.Runtime.Intrinsics.Vector64 Create(System.ReadOnlySpan values) { throw null; } public static System.Runtime.Intrinsics.Vector64 Create(T value) { throw null; } public static System.Runtime.Intrinsics.Vector64 Create(T[] values) { throw null; } @@ -1761,6 +1824,7 @@ public static void CopyTo(this System.Runtime.Intrinsics.Vector64 vector, public static System.Runtime.Intrinsics.Vector64 AllBitsSet { get { throw null; } } public static int Count { get { throw null; } } public static System.Runtime.Intrinsics.Vector64 Indices { get { throw null; } } + public static System.Runtime.Intrinsics.Vector64 SignSequence { get { throw null; } } public static bool IsSupported { get { throw null; } } public static System.Runtime.Intrinsics.Vector64 One { get { throw null; } } public static System.Runtime.Intrinsics.Vector64 Zero { get { throw null; } } diff --git a/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector128Tests.cs b/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector128Tests.cs index 6246c1231c32d2..ac20fe9f0adf7f 100644 --- a/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector128Tests.cs +++ b/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector128Tests.cs @@ -5305,6 +5305,185 @@ private static void TestCreateSequence(T start, T step) } } + [Fact] + public void CreateGeometricSequenceInt32Test() + { + Vector128 sequence = Vector128.CreateGeometricSequence(1, 2); + int expected = 1; + + for (int index = 0; index < Vector128.Count; index++) + { + Assert.Equal(expected, sequence.GetElement(index)); + expected *= 2; + } + } + + [Fact] + public void CreateGeometricSequenceSingleNonConstantInitialTest() + { + const float multiplier = 1.0064822f; + float initial = GetNonConstant(1.0059024f); + Vector128 sequence = Vector128.CreateGeometricSequence(initial, multiplier); + float expected = initial; + + for (int index = 0; index < Vector128.Count; index++) + { + Assert.Equal(expected, sequence.GetElement(index)); + expected *= multiplier; + } + } + + [Fact] + public void CreateGeometricSequenceDoubleNonConstantInitialTest() + { + const double multiplier = 1e-50; + double initial = GetNonConstant(1e-154); + Vector128 sequence = Vector128.CreateGeometricSequence(initial, multiplier); + double expected = initial; + + for (int index = 0; index < Vector128.Count; index++) + { + Assert.Equal(expected, sequence.GetElement(index)); + expected *= multiplier; + } + } + + [Fact] + public void CreateAlternatingSequenceInt32Test() + { + Vector128 sequence = Vector128.CreateAlternatingSequence(5, -5); + + for (int index = 0; index < Vector128.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 5 : -5, sequence.GetElement(index)); + } + } + + [Fact] + public void CreateHarmonicSequenceDoubleTest() + { + Vector128 sequence = Vector128.CreateHarmonicSequence(1.0, 1.0); + double expected = 1.0; + + for (int index = 0; index < Vector128.Count; index++) + { + AssertExtensions.Equal(1.0 / expected, sequence.GetElement(index), 1e-15); + expected += 1.0; + } + } + + [Fact] + public void CreateCauchySequenceDoubleTest() + { + Vector128 sequence = Vector128.CreateCauchySequence(1.0, 1.0); + double expected = 1.0; + + for (int index = 0; index < Vector128.Count; index++) + { + AssertExtensions.Equal(Math.Sqrt(expected), sequence.GetElement(index), 1e-15); + expected += 1.0; + } + } + + [Fact] + public void SignSequenceInt32Test() + { + Vector128 sequence = Vector128.SignSequence; + + for (int index = 0; index < Vector128.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1 : -1, sequence.GetElement(index)); + } + } + + [Fact] + public void SignSequenceUInt32Test() + { + Vector128 sequence = Vector128.SignSequence; + + for (int index = 0; index < Vector128.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1u : uint.MaxValue, sequence.GetElement(index)); + } + } + + [Fact] + public void SignSequenceSingleTest() + { + Vector128 sequence = Vector128.SignSequence; + + for (int index = 0; index < Vector128.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1.0f : -1.0f, sequence.GetElement(index)); + } + } + + [Fact] + public void SignSequenceDoubleTest() + { + Vector128 sequence = Vector128.SignSequence; + + for (int index = 0; index < Vector128.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1.0 : -1.0, sequence.GetElement(index)); + } + } + + [Fact] + public void LaneOperationsInt32Test() + { + Vector128 left = Vector128.CreateSequence(0, 1); + Vector128 right = Vector128.CreateSequence(100, 1); + int count = Vector128.Count; + int lowerCount = (count + 1) / 2; + int upperStart = count - lowerCount; + + AssertVectorEqual(CreateVector128(index => ((index & 1) == 0) ? left.GetElement(index / 2) : right.GetElement(index / 2)), Vector128.ZipLower(left, right)); + AssertVectorEqual(CreateVector128(index => ((index & 1) == 0) ? left.GetElement(upperStart + (index / 2)) : right.GetElement(upperStart + (index / 2))), Vector128.ZipUpper(left, right)); + + (Vector128 lower, Vector128 upper) = Vector128.Zip(left, right); + AssertVectorEqual(Vector128.ZipLower(left, right), lower); + AssertVectorEqual(Vector128.ZipUpper(left, right), upper); + + AssertVectorEqual(left, Vector128.UnzipEven(lower, upper)); + AssertVectorEqual(right, Vector128.UnzipOdd(lower, upper)); + + (Vector128 even, Vector128 odd) = Vector128.Unzip(lower, upper); + AssertVectorEqual(left, even); + AssertVectorEqual(right, odd); + + AssertVectorEqual(CreateVector128(index => (index < lowerCount) ? left.GetElement(index) : right.GetElement(index - lowerCount)), Vector128.ConcatLowerLower(left, right)); + AssertVectorEqual(CreateVector128(index => (index < lowerCount) ? left.GetElement(upperStart + index) : right.GetElement(index - lowerCount)), Vector128.ConcatUpperLower(left, right)); + AssertVectorEqual(CreateVector128(index => (index < lowerCount) ? left.GetElement(upperStart + index) : right.GetElement(upperStart + index - lowerCount)), Vector128.ConcatUpperUpper(left, right)); + AssertVectorEqual(CreateVector128(index => (index < lowerCount) ? left.GetElement(index) : right.GetElement(upperStart + index - lowerCount)), Vector128.ConcatLowerUpper(left, right)); + + AssertVectorEqual(CreateVector128(index => left.GetElement(count - 1 - index)), Vector128.Reverse(left)); + } + + private static Vector128 CreateVector128(Func elementSelector) + { + int[] values = new int[Vector128.Count]; + + for (int index = 0; index < values.Length; index++) + { + values[index] = elementSelector(index); + } + + return Vector128.Create(values); + } + + private static void AssertVectorEqual(Vector128 expected, Vector128 actual) + where T : struct + { + for (int index = 0; index < Vector128.Count; index++) + { + Assert.Equal(expected.GetElement(index), actual.GetElement(index)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static T GetNonConstant(T value) => value; + [Theory] [MemberData(nameof(GenericMathTestMemberData.AsinDouble), MemberType = typeof(GenericMathTestMemberData))] public void AsinDoubleTest(double value, double expectedResult, double variance) diff --git a/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector256Tests.cs b/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector256Tests.cs index 728a87900f314a..33649743726a39 100644 --- a/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector256Tests.cs +++ b/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector256Tests.cs @@ -6481,6 +6481,185 @@ private static void TestCreateSequence(T start, T step) } } + [Fact] + public void CreateGeometricSequenceInt32Test() + { + Vector256 sequence = Vector256.CreateGeometricSequence(1, 2); + int expected = 1; + + for (int index = 0; index < Vector256.Count; index++) + { + Assert.Equal(expected, sequence.GetElement(index)); + expected *= 2; + } + } + + [Fact] + public void CreateGeometricSequenceSingleNonConstantInitialTest() + { + const float multiplier = 1.0064822f; + float initial = GetNonConstant(1.0059024f); + Vector256 sequence = Vector256.CreateGeometricSequence(initial, multiplier); + float expected = initial; + + for (int index = 0; index < Vector256.Count; index++) + { + Assert.Equal(expected, sequence.GetElement(index)); + expected *= multiplier; + } + } + + [Fact] + public void CreateGeometricSequenceDoubleNonConstantInitialTest() + { + const double multiplier = 1e-50; + double initial = GetNonConstant(1e-154); + Vector256 sequence = Vector256.CreateGeometricSequence(initial, multiplier); + double expected = initial; + + for (int index = 0; index < Vector256.Count; index++) + { + Assert.Equal(expected, sequence.GetElement(index)); + expected *= multiplier; + } + } + + [Fact] + public void CreateAlternatingSequenceInt32Test() + { + Vector256 sequence = Vector256.CreateAlternatingSequence(5, -5); + + for (int index = 0; index < Vector256.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 5 : -5, sequence.GetElement(index)); + } + } + + [Fact] + public void CreateHarmonicSequenceDoubleTest() + { + Vector256 sequence = Vector256.CreateHarmonicSequence(1.0, 1.0); + double expected = 1.0; + + for (int index = 0; index < Vector256.Count; index++) + { + AssertExtensions.Equal(1.0 / expected, sequence.GetElement(index), 1e-15); + expected += 1.0; + } + } + + [Fact] + public void CreateCauchySequenceDoubleTest() + { + Vector256 sequence = Vector256.CreateCauchySequence(1.0, 1.0); + double expected = 1.0; + + for (int index = 0; index < Vector256.Count; index++) + { + AssertExtensions.Equal(Math.Sqrt(expected), sequence.GetElement(index), 1e-15); + expected += 1.0; + } + } + + [Fact] + public void SignSequenceInt32Test() + { + Vector256 sequence = Vector256.SignSequence; + + for (int index = 0; index < Vector256.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1 : -1, sequence.GetElement(index)); + } + } + + [Fact] + public void SignSequenceUInt32Test() + { + Vector256 sequence = Vector256.SignSequence; + + for (int index = 0; index < Vector256.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1u : uint.MaxValue, sequence.GetElement(index)); + } + } + + [Fact] + public void SignSequenceSingleTest() + { + Vector256 sequence = Vector256.SignSequence; + + for (int index = 0; index < Vector256.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1.0f : -1.0f, sequence.GetElement(index)); + } + } + + [Fact] + public void SignSequenceDoubleTest() + { + Vector256 sequence = Vector256.SignSequence; + + for (int index = 0; index < Vector256.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1.0 : -1.0, sequence.GetElement(index)); + } + } + + [Fact] + public void LaneOperationsInt32Test() + { + Vector256 left = Vector256.CreateSequence(0, 1); + Vector256 right = Vector256.CreateSequence(100, 1); + int count = Vector256.Count; + int lowerCount = (count + 1) / 2; + int upperStart = count - lowerCount; + + AssertVectorEqual(CreateVector256(index => ((index & 1) == 0) ? left.GetElement(index / 2) : right.GetElement(index / 2)), Vector256.ZipLower(left, right)); + AssertVectorEqual(CreateVector256(index => ((index & 1) == 0) ? left.GetElement(upperStart + (index / 2)) : right.GetElement(upperStart + (index / 2))), Vector256.ZipUpper(left, right)); + + (Vector256 lower, Vector256 upper) = Vector256.Zip(left, right); + AssertVectorEqual(Vector256.ZipLower(left, right), lower); + AssertVectorEqual(Vector256.ZipUpper(left, right), upper); + + AssertVectorEqual(left, Vector256.UnzipEven(lower, upper)); + AssertVectorEqual(right, Vector256.UnzipOdd(lower, upper)); + + (Vector256 even, Vector256 odd) = Vector256.Unzip(lower, upper); + AssertVectorEqual(left, even); + AssertVectorEqual(right, odd); + + AssertVectorEqual(CreateVector256(index => (index < lowerCount) ? left.GetElement(index) : right.GetElement(index - lowerCount)), Vector256.ConcatLowerLower(left, right)); + AssertVectorEqual(CreateVector256(index => (index < lowerCount) ? left.GetElement(upperStart + index) : right.GetElement(index - lowerCount)), Vector256.ConcatUpperLower(left, right)); + AssertVectorEqual(CreateVector256(index => (index < lowerCount) ? left.GetElement(upperStart + index) : right.GetElement(upperStart + index - lowerCount)), Vector256.ConcatUpperUpper(left, right)); + AssertVectorEqual(CreateVector256(index => (index < lowerCount) ? left.GetElement(index) : right.GetElement(upperStart + index - lowerCount)), Vector256.ConcatLowerUpper(left, right)); + + AssertVectorEqual(CreateVector256(index => left.GetElement(count - 1 - index)), Vector256.Reverse(left)); + } + + private static Vector256 CreateVector256(Func elementSelector) + { + int[] values = new int[Vector256.Count]; + + for (int index = 0; index < values.Length; index++) + { + values[index] = elementSelector(index); + } + + return Vector256.Create(values); + } + + private static void AssertVectorEqual(Vector256 expected, Vector256 actual) + where T : struct + { + for (int index = 0; index < Vector256.Count; index++) + { + Assert.Equal(expected.GetElement(index), actual.GetElement(index)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static T GetNonConstant(T value) => value; + [Theory] [MemberData(nameof(GenericMathTestMemberData.AsinDouble), MemberType = typeof(GenericMathTestMemberData))] public void AsinDoubleTest(double value, double expectedResult, double variance) diff --git a/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector512Tests.cs b/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector512Tests.cs index d3c430020b5225..d020b66f60c2c1 100644 --- a/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector512Tests.cs +++ b/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector512Tests.cs @@ -6264,6 +6264,185 @@ private static void TestCreateSequence(T start, T step) } } + [Fact] + public void CreateGeometricSequenceInt32Test() + { + Vector512 sequence = Vector512.CreateGeometricSequence(1, 2); + int expected = 1; + + for (int index = 0; index < Vector512.Count; index++) + { + Assert.Equal(expected, sequence.GetElement(index)); + expected *= 2; + } + } + + [Fact] + public void CreateGeometricSequenceSingleNonConstantInitialTest() + { + const float multiplier = 1.0064822f; + float initial = GetNonConstant(1.0059024f); + Vector512 sequence = Vector512.CreateGeometricSequence(initial, multiplier); + float expected = initial; + + for (int index = 0; index < Vector512.Count; index++) + { + Assert.Equal(expected, sequence.GetElement(index)); + expected *= multiplier; + } + } + + [Fact] + public void CreateGeometricSequenceDoubleNonConstantInitialTest() + { + const double multiplier = 1e-50; + double initial = GetNonConstant(1e-154); + Vector512 sequence = Vector512.CreateGeometricSequence(initial, multiplier); + double expected = initial; + + for (int index = 0; index < Vector512.Count; index++) + { + Assert.Equal(expected, sequence.GetElement(index)); + expected *= multiplier; + } + } + + [Fact] + public void CreateAlternatingSequenceInt32Test() + { + Vector512 sequence = Vector512.CreateAlternatingSequence(5, -5); + + for (int index = 0; index < Vector512.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 5 : -5, sequence.GetElement(index)); + } + } + + [Fact] + public void CreateHarmonicSequenceDoubleTest() + { + Vector512 sequence = Vector512.CreateHarmonicSequence(1.0, 1.0); + double expected = 1.0; + + for (int index = 0; index < Vector512.Count; index++) + { + AssertExtensions.Equal(1.0 / expected, sequence.GetElement(index), 1e-15); + expected += 1.0; + } + } + + [Fact] + public void CreateCauchySequenceDoubleTest() + { + Vector512 sequence = Vector512.CreateCauchySequence(1.0, 1.0); + double expected = 1.0; + + for (int index = 0; index < Vector512.Count; index++) + { + AssertExtensions.Equal(Math.Sqrt(expected), sequence.GetElement(index), 1e-15); + expected += 1.0; + } + } + + [Fact] + public void SignSequenceInt32Test() + { + Vector512 sequence = Vector512.SignSequence; + + for (int index = 0; index < Vector512.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1 : -1, sequence.GetElement(index)); + } + } + + [Fact] + public void SignSequenceUInt32Test() + { + Vector512 sequence = Vector512.SignSequence; + + for (int index = 0; index < Vector512.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1u : uint.MaxValue, sequence.GetElement(index)); + } + } + + [Fact] + public void SignSequenceSingleTest() + { + Vector512 sequence = Vector512.SignSequence; + + for (int index = 0; index < Vector512.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1.0f : -1.0f, sequence.GetElement(index)); + } + } + + [Fact] + public void SignSequenceDoubleTest() + { + Vector512 sequence = Vector512.SignSequence; + + for (int index = 0; index < Vector512.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1.0 : -1.0, sequence.GetElement(index)); + } + } + + [Fact] + public void LaneOperationsInt32Test() + { + Vector512 left = Vector512.CreateSequence(0, 1); + Vector512 right = Vector512.CreateSequence(100, 1); + int count = Vector512.Count; + int lowerCount = (count + 1) / 2; + int upperStart = count - lowerCount; + + AssertVectorEqual(CreateVector512(index => ((index & 1) == 0) ? left.GetElement(index / 2) : right.GetElement(index / 2)), Vector512.ZipLower(left, right)); + AssertVectorEqual(CreateVector512(index => ((index & 1) == 0) ? left.GetElement(upperStart + (index / 2)) : right.GetElement(upperStart + (index / 2))), Vector512.ZipUpper(left, right)); + + (Vector512 lower, Vector512 upper) = Vector512.Zip(left, right); + AssertVectorEqual(Vector512.ZipLower(left, right), lower); + AssertVectorEqual(Vector512.ZipUpper(left, right), upper); + + AssertVectorEqual(left, Vector512.UnzipEven(lower, upper)); + AssertVectorEqual(right, Vector512.UnzipOdd(lower, upper)); + + (Vector512 even, Vector512 odd) = Vector512.Unzip(lower, upper); + AssertVectorEqual(left, even); + AssertVectorEqual(right, odd); + + AssertVectorEqual(CreateVector512(index => (index < lowerCount) ? left.GetElement(index) : right.GetElement(index - lowerCount)), Vector512.ConcatLowerLower(left, right)); + AssertVectorEqual(CreateVector512(index => (index < lowerCount) ? left.GetElement(upperStart + index) : right.GetElement(index - lowerCount)), Vector512.ConcatUpperLower(left, right)); + AssertVectorEqual(CreateVector512(index => (index < lowerCount) ? left.GetElement(upperStart + index) : right.GetElement(upperStart + index - lowerCount)), Vector512.ConcatUpperUpper(left, right)); + AssertVectorEqual(CreateVector512(index => (index < lowerCount) ? left.GetElement(index) : right.GetElement(upperStart + index - lowerCount)), Vector512.ConcatLowerUpper(left, right)); + + AssertVectorEqual(CreateVector512(index => left.GetElement(count - 1 - index)), Vector512.Reverse(left)); + } + + private static Vector512 CreateVector512(Func elementSelector) + { + int[] values = new int[Vector512.Count]; + + for (int index = 0; index < values.Length; index++) + { + values[index] = elementSelector(index); + } + + return Vector512.Create(values); + } + + private static void AssertVectorEqual(Vector512 expected, Vector512 actual) + where T : struct + { + for (int index = 0; index < Vector512.Count; index++) + { + Assert.Equal(expected.GetElement(index), actual.GetElement(index)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static T GetNonConstant(T value) => value; + [Theory] [MemberData(nameof(GenericMathTestMemberData.AsinDouble), MemberType = typeof(GenericMathTestMemberData))] public void AsinDoubleTest(double value, double expectedResult, double variance) diff --git a/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector64Tests.cs b/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector64Tests.cs index be91c3325549fb..f5f7160bbc082d 100644 --- a/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector64Tests.cs +++ b/src/libraries/System.Runtime.Intrinsics/tests/Vectors/Vector64Tests.cs @@ -4579,6 +4579,204 @@ private static void TestCreateSequence(T start, T step) } } + [Fact] + public void CreateGeometricSequenceInt32Test() + { + Vector64 sequence = Vector64.CreateGeometricSequence(1, 2); + int expected = 1; + + for (int index = 0; index < Vector64.Count; index++) + { + Assert.Equal(expected, sequence.GetElement(index)); + expected *= 2; + } + } + + [Fact] + public void CreateAlternatingSequenceInt32Test() + { + Vector64 sequence = Vector64.CreateAlternatingSequence(5, -5); + + for (int index = 0; index < Vector64.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 5 : -5, sequence.GetElement(index)); + } + } + + [Fact] + public void CreateHarmonicSequenceDoubleTest() + { + Vector64 sequence = Vector64.CreateHarmonicSequence(1.0, 1.0); + double expected = 1.0; + + for (int index = 0; index < Vector64.Count; index++) + { + AssertExtensions.Equal(1.0 / expected, sequence.GetElement(index), 1e-15); + expected += 1.0; + } + } + + [Fact] + public void CreateCauchySequenceDoubleTest() + { + Vector64 sequence = Vector64.CreateCauchySequence(1.0, 1.0); + double expected = 1.0; + + for (int index = 0; index < Vector64.Count; index++) + { + AssertExtensions.Equal(Math.Sqrt(expected), sequence.GetElement(index), 1e-15); + expected += 1.0; + } + } + + [Fact] + public void SignSequenceInt32Test() + { + Vector64 sequence = Vector64.SignSequence; + + for (int index = 0; index < Vector64.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1 : -1, sequence.GetElement(index)); + } + } + + [Fact] + public void SignSequenceUInt32Test() + { + Vector64 sequence = Vector64.SignSequence; + + for (int index = 0; index < Vector64.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1u : uint.MaxValue, sequence.GetElement(index)); + } + } + + [Fact] + public void SignSequenceSingleTest() + { + Vector64 sequence = Vector64.SignSequence; + + for (int index = 0; index < Vector64.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1.0f : -1.0f, sequence.GetElement(index)); + } + } + + [Fact] + public void SignSequenceDoubleTest() + { + Vector64 sequence = Vector64.SignSequence; + + for (int index = 0; index < Vector64.Count; index++) + { + Assert.Equal(((index & 1) == 0) ? 1.0 : -1.0, sequence.GetElement(index)); + } + } + + [Fact] + public void LaneOperationsInt32Test() + { + Vector64 left = Vector64.CreateSequence(0, 1); + Vector64 right = Vector64.CreateSequence(100, 1); + int count = Vector64.Count; + int lowerCount = (count + 1) / 2; + int upperStart = count - lowerCount; + + AssertVectorEqual(CreateVector64(index => ((index & 1) == 0) ? left.GetElement(index / 2) : right.GetElement(index / 2)), Vector64.ZipLower(left, right)); + AssertVectorEqual(CreateVector64(index => ((index & 1) == 0) ? left.GetElement(upperStart + (index / 2)) : right.GetElement(upperStart + (index / 2))), Vector64.ZipUpper(left, right)); + + (Vector64 lower, Vector64 upper) = Vector64.Zip(left, right); + AssertVectorEqual(Vector64.ZipLower(left, right), lower); + AssertVectorEqual(Vector64.ZipUpper(left, right), upper); + + AssertVectorEqual(left, Vector64.UnzipEven(lower, upper)); + AssertVectorEqual(right, Vector64.UnzipOdd(lower, upper)); + + (Vector64 even, Vector64 odd) = Vector64.Unzip(lower, upper); + AssertVectorEqual(left, even); + AssertVectorEqual(right, odd); + + AssertVectorEqual(CreateVector64(index => (index < lowerCount) ? left.GetElement(index) : right.GetElement(index - lowerCount)), Vector64.ConcatLowerLower(left, right)); + AssertVectorEqual(CreateVector64(index => (index < lowerCount) ? left.GetElement(upperStart + index) : right.GetElement(index - lowerCount)), Vector64.ConcatUpperLower(left, right)); + AssertVectorEqual(CreateVector64(index => (index < lowerCount) ? left.GetElement(upperStart + index) : right.GetElement(upperStart + index - lowerCount)), Vector64.ConcatUpperUpper(left, right)); + AssertVectorEqual(CreateVector64(index => (index < lowerCount) ? left.GetElement(index) : right.GetElement(upperStart + index - lowerCount)), Vector64.ConcatLowerUpper(left, right)); + + AssertVectorEqual(CreateVector64(index => left.GetElement(count - 1 - index)), Vector64.Reverse(left)); + } + + [Fact] + public void LaneOperationsInt64CountOneTest() + { + Vector64 left = Vector64.Create(10L); + Vector64 right = Vector64.Create(20L); + + AssertVectorEqual(left, Vector64.ZipLower(left, right)); + AssertVectorEqual(left, Vector64.ZipUpper(left, right)); + + (Vector64 lower, Vector64 upper) = Vector64.Zip(left, right); + AssertVectorEqual(left, lower); + AssertVectorEqual(left, upper); + + AssertVectorEqual(left, Vector64.UnzipEven(left, right)); + AssertVectorEqual(Vector64.Zero, Vector64.UnzipOdd(left, right)); + + (Vector64 even, Vector64 odd) = Vector64.Unzip(left, right); + AssertVectorEqual(left, even); + AssertVectorEqual(Vector64.Zero, odd); + + AssertVectorEqual(left, Vector64.ConcatLowerLower(left, right)); + AssertVectorEqual(left, Vector64.ConcatUpperLower(left, right)); + AssertVectorEqual(left, Vector64.ConcatUpperUpper(left, right)); + AssertVectorEqual(left, Vector64.ConcatLowerUpper(left, right)); + } + + [Fact] + public void LaneOperationsDoubleCountOneTest() + { + Vector64 left = Vector64.Create(10.0); + Vector64 right = Vector64.Create(20.0); + + AssertVectorEqual(left, Vector64.ZipLower(left, right)); + AssertVectorEqual(left, Vector64.ZipUpper(left, right)); + + (Vector64 lower, Vector64 upper) = Vector64.Zip(left, right); + AssertVectorEqual(left, lower); + AssertVectorEqual(left, upper); + + AssertVectorEqual(left, Vector64.UnzipEven(left, right)); + AssertVectorEqual(Vector64.Zero, Vector64.UnzipOdd(left, right)); + + (Vector64 even, Vector64 odd) = Vector64.Unzip(left, right); + AssertVectorEqual(left, even); + AssertVectorEqual(Vector64.Zero, odd); + + AssertVectorEqual(left, Vector64.ConcatLowerLower(left, right)); + AssertVectorEqual(left, Vector64.ConcatUpperLower(left, right)); + AssertVectorEqual(left, Vector64.ConcatUpperUpper(left, right)); + AssertVectorEqual(left, Vector64.ConcatLowerUpper(left, right)); + } + + private static Vector64 CreateVector64(Func elementSelector) + { + int[] values = new int[Vector64.Count]; + + for (int index = 0; index < values.Length; index++) + { + values[index] = elementSelector(index); + } + + return Vector64.Create(values); + } + + private static void AssertVectorEqual(Vector64 expected, Vector64 actual) + where T : struct + { + for (int index = 0; index < Vector64.Count; index++) + { + Assert.Equal(expected.GetElement(index), actual.GetElement(index)); + } + } + [Theory] [MemberData(nameof(GenericMathTestMemberData.AsinDouble), MemberType = typeof(GenericMathTestMemberData))] public void AsinDoubleTest(double value, double expectedResult, double variance)