Skip to content

[release/9.0] JIT ARM64-SVE: Backport "Avoid combining conditional select for reduction instrinsics" (#107207) #107722

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jeffschwMSFT merged 3 commits into from
Sep 12, 2024
Merged

[release/9.0] JIT ARM64-SVE: Backport "Avoid combining conditional select for reduction instrinsics" (#107207) #107722

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7a19ecd
Backport #107207
SwapnilGaikwad Sep 3, 2024
ec77372
Merge branch 'release/9.0' into backport-107207
amanasifkhalid Sep 12, 2024
c8c08ba
Merge branch 'release/9.0' into backport-107207
jeffschwMSFT Sep 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions src/coreclr/jit/hwintrinsic.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,9 @@ enum HWIntrinsicFlag : unsigned int
// (HW_Flag_BaseTypeFrom{First, Second}Arg must also be set to denote the position of the ValueTuple)
HW_Flag_BaseTypeFromValueTupleArg = 0x10000000,

// The intrinsic is a reduce operation.
HW_Flag_ReduceOperation = 0x20000000,

#endif // TARGET_XARCH

// The intrinsic is a FusedMultiplyAdd intrinsic
Expand Down Expand Up @@ -1011,6 +1014,12 @@ struct HWIntrinsicInfo
return (flags & HW_Flag_BaseTypeFromValueTupleArg) != 0;
}

static bool IsReduceOperation(NamedIntrinsic id)
{
const HWIntrinsicFlag flags = lookupFlags(id);
return (flags & HW_Flag_ReduceOperation) != 0;
}

static NamedIntrinsic GetScalarInputVariant(NamedIntrinsic id)
{
assert(HasScalarInputVariant(id));
Expand Down
18 changes: 9 additions & 9 deletions src/coreclr/jit/hwintrinsiclistarm64sve.h
View file
Open in desktop

Large diffs are not rendered by default.

5 changes: 3 additions & 2 deletions src/coreclr/jit/lowerarmarch.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4056,9 +4056,10 @@ GenTree* Lowering::LowerHWIntrinsicCndSel(GenTreeHWIntrinsic* cndSelNode)
NamedIntrinsic nestedOp2Id = nestedOp2->AsHWIntrinsic()->GetHWIntrinsicId();

// If the nested op uses Pg/Z, then inactive lanes will result in zeros, so can only transform if
// op3 is all zeros.
// op3 is all zeros. Such a Csel operation is absorbed into the instruction when emitted. Skip this optimisation
// when the nestedOp is a reduce operation.

if (nestedOp1->IsMaskAllBitsSet() &&
if (nestedOp1->IsMaskAllBitsSet() && !HWIntrinsicInfo::IsReduceOperation(nestedOp2Id) &&
(!HWIntrinsicInfo::IsZeroingMaskedOperation(nestedOp2Id) || op3->IsVectorZero()))
{
GenTree* nestedOp2 = nestedCndSel->Op(2);
Expand Down
63 changes: 62 additions & 1 deletion src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,67 @@
}
}";

const string VecReduceUnOpTest_VectorValidationLogicForCndSel = @"
{
var hasFailed = (mask[0] != 0 ? {ValidateReduceOpResult}: (falseVal[0] != result[0]));

if (hasFailed)
{
succeeded = false;
}
else
{
for (var i = 1; i < RetElementCount; i++)
{
var iterResult = (mask[i] != 0) ? 0 : falseVal[i];
if (mask[i] != 0)
{
// Pick the trueValue
if (iterResult != result[i])
{
succeeded = false;
break;
}
}
else
{
// For false, the values are merged with destination, and we do not know
// those contents would be, so skip verification for them.
}
}
}
}";

const string VecReduceUnOpTest_VectorValidationLogicForCndSel_FalseValue = @"
{
var hasFailed = (mask[0] != 0) ? (trueVal[0] != result[0]): {ValidateReduceOpResult};
if (hasFailed)
{
succeeded = false;
}
else
{
for (var i = 1; i < RetElementCount; i++)
{
var iterResult = (mask[i] != 0) ? trueVal[i] : 0;
if (mask[i] != 0)
{
// Pick the trueValue
if (iterResult != result[i])
{
succeeded = false;
break;
}
}
else
{
// For false, the values are merged with destination, and we do not know
// those contents would be, so skip verification for them.
}
}
}
}";

const string VecReduceOpTest_ValidationLogic = @"if ({ValidateReduceOpResult})
{
succeeded = false;
Expand Down Expand Up @@ -293,7 +354,7 @@
("_SveImmTernOpFirstArgTestTemplate.template", "SveVecImmTernOpFirstArgTest.template", new Dictionary<string, string> { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleTernVecOpTest_ValidationLogicForCndSel, ["TemplateValidationLogicForCndSel_FalseValue"] = SimpleTernVecOpTest_ValidationLogicForCndSel_FalseValue }),
("_SveScalarTernOpTestTemplate.template", "SveScalarTernOpTest.template", new Dictionary<string, string> { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleScalarOpTest_ValidationLogic }),
("_SveImm2UnaryOpTestTemplate.template", "SveVecImm2UnOpTest.template", new Dictionary<string, string> { ["TemplateName"] = "Imm", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic }),
("_SveMinimalUnaryOpTestTemplate.template", "SveVecReduceUnOpTest.template", new Dictionary<string, string> { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = VecReduceOpTest_ValidationLogic }),
("_SveMinimalUnaryOpTestTemplate.template", "SveVecReduceUnOpTest.template", new Dictionary<string, string> { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = VecReduceOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = VecReduceUnOpTest_VectorValidationLogicForCndSel, ["TemplateValidationLogicForCndSel_FalseValue"] = VecReduceUnOpTest_VectorValidationLogicForCndSel_FalseValue }),
("_SveMasklessUnaryOpTestTemplate.template", "SveMasklessSimpleVecOpTest.template", new Dictionary<string, string> { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic }),
("_SveVecAndScalarOpTest.template", "SveVecAndScalarOpTest.template", new Dictionary<string, string> { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_VectorValidationLogic }),
("_SveMasklessBinaryOpTestTemplate.template", "SveMasklessVecBinOpTest.template", new Dictionary<string, string> { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic }),
Expand Down
129 changes: 129 additions & 0 deletions src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveMinimalUnaryOpTestTemplate.template
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,12 @@ namespace JIT.HardwareIntrinsics.Arm

// Validates passing an instance member of a struct works
test.RunStructFldScenario();

// Validates executing the test inside conditional, with op3 as falseValue
test.ConditionalSelect_FalseOp();

// Validates executing the test inside conditional, with op3 as zero
test.ConditionalSelect_ZeroOp();
}
else
{
Expand Down Expand Up @@ -139,18 +145,25 @@ namespace JIT.HardwareIntrinsics.Arm
private static readonly int Op1ElementCount = Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>() / sizeof({Op1BaseType});
private static readonly int RetElementCount = Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>() / sizeof({RetBaseType});

private static {RetBaseType}[] _maskData = new {RetBaseType}[RetElementCount];
private static {Op1BaseType}[] _data1 = new {Op1BaseType}[Op1ElementCount];

private {RetVectorType}<{RetBaseType}> _mask;
private {Op1VectorType}<{Op1BaseType}> _fld1;
private {RetVectorType}<{RetBaseType}> _falseFld;

private DataTable _dataTable;

public {TemplateName}UnaryOpTest__{TestName}()
{
Succeeded = true;

for (var i = 0; i < RetElementCount; i++) { _maskData[i] = ({RetBaseType})(TestLibrary.Generator.Get{RetBaseType}() % 2); }
Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetVectorType}<{RetBaseType}>, byte>(ref _mask), ref Unsafe.As<{RetBaseType}, byte>(ref _maskData[0]), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>());

for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; }
Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>());
Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetVectorType}<{RetBaseType}>, byte>(ref _falseFld), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>());

for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; }
_dataTable = new DataTable(_data1, new {RetBaseType}[RetElementCount], LargestVectorSize);
Expand Down Expand Up @@ -239,6 +252,66 @@ namespace JIT.HardwareIntrinsics.Arm
test.RunStructFldScenario(this);
}

public void ConditionalSelect_FalseOp()
{
TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_mask - operation in trueValue");
ConditionalSelectScenario_TrueValue(_mask, _fld1, _falseFld);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_zero - operation in trueValue");
ConditionalSelectScenario_TrueValue({RetVectorType}<{RetBaseType}>.Zero, _fld1, _falseFld);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_all - operation in trueValue");
ConditionalSelectScenario_TrueValue({RetVectorType}<{RetBaseType}>.AllBitsSet, _fld1, _falseFld);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_mask - operation in falseValue");
ConditionalSelectScenario_FalseValue(_mask, _fld1, _falseFld);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_zero - operation in falseValue");
ConditionalSelectScenario_FalseValue({RetVectorType}<{RetBaseType}>.Zero, _fld1, _falseFld);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_all - operation in falseValue");
ConditionalSelectScenario_FalseValue({RetVectorType}<{RetBaseType}>.AllBitsSet, _fld1, _falseFld);
}

public void ConditionalSelect_ZeroOp()
{
TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_mask - operation in trueValue");
ConditionalSelectScenario_TrueValue(_mask, _fld1, {RetVectorType}<{RetBaseType}>.Zero);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero - operation in trueValue");
ConditionalSelectScenario_TrueValue({RetVectorType}<{RetBaseType}>.Zero, _fld1, {RetVectorType}<{RetBaseType}>.Zero);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all - operation in trueValue");
ConditionalSelectScenario_TrueValue({RetVectorType}<{RetBaseType}>.AllBitsSet, _fld1, {RetVectorType}<{RetBaseType}>.Zero);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_mask - operation in falseValue");
ConditionalSelectScenario_FalseValue(_mask, _fld1, {RetVectorType}<{RetBaseType}>.Zero);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero - operation in falseValue");
ConditionalSelectScenario_FalseValue({RetVectorType}<{RetBaseType}>.Zero, _fld1, {RetVectorType}<{RetBaseType}>.Zero);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all - operation in falseValue");
ConditionalSelectScenario_FalseValue({RetVectorType}<{RetBaseType}>.AllBitsSet, _fld1, {RetVectorType}<{RetBaseType}>.Zero);
}

[method: MethodImpl(MethodImplOptions.AggressiveInlining)]
private void ConditionalSelectScenario_TrueValue({RetVectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {RetVectorType}<{RetBaseType}> falseOp)
{
var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1), falseOp);

Unsafe.Write(_dataTable.outArrayPtr, result);
ValidateConditionalSelectResult_TrueValue(mask, op1, falseOp, _dataTable.outArrayPtr);
}

[method: MethodImpl(MethodImplOptions.AggressiveInlining)]
private void ConditionalSelectScenario_FalseValue({RetVectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {RetVectorType}<{RetBaseType}> trueOp)
{
var result = Sve.ConditionalSelect(mask, trueOp, {Isa}.{Method}(op1));

Unsafe.Write(_dataTable.outArrayPtr, result);
ValidateConditionalSelectResult_FalseValue(mask, op1, trueOp, _dataTable.outArrayPtr);
}

public void RunUnsupportedScenario()
{
TestLibrary.TestFramework.BeginScenario(nameof(RunUnsupportedScenario));
Expand All @@ -260,6 +333,62 @@ namespace JIT.HardwareIntrinsics.Arm
}
}

private void ValidateConditionalSelectResult_TrueValue({RetVectorType}<{RetBaseType}> maskOp, {Op1VectorType}<{Op1BaseType}> leftOp, {RetVectorType}<{RetBaseType}> falseOp, void* output, [CallerMemberName] string method = "")
{
{RetBaseType}[] mask = new {RetBaseType}[RetElementCount];
{Op1BaseType}[] firstOp = new {Op1BaseType}[Op1ElementCount];
{RetBaseType}[] falseVal = new {RetBaseType}[RetElementCount];
{RetBaseType}[] result = new {RetBaseType}[RetElementCount];

Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref mask[0]), maskOp);
Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref firstOp[0]), leftOp);
Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref falseVal[0]), falseOp);
Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref result[0]), ref Unsafe.AsRef<byte>(output), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>());

bool succeeded = true;
{TemplateValidationLogicForCndSel}

if (!succeeded)
{
TestLibrary.TestFramework.LogInformation($"{nameof(Sve)}.{nameof({Isa}.{Method})}<{RetBaseType}>({RetVectorType}<{RetBaseType}>, {RetVectorType}<{RetBaseType}>): {method} failed:");
TestLibrary.TestFramework.LogInformation($" mask: ({string.Join(", ", mask)})");
TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})");
TestLibrary.TestFramework.LogInformation($" falseOp: ({string.Join(", ", falseVal)})");
TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})");
TestLibrary.TestFramework.LogInformation(string.Empty);

Succeeded = false;
}
}

private void ValidateConditionalSelectResult_FalseValue({RetVectorType}<{RetBaseType}> maskOp, {Op1VectorType}<{Op1BaseType}> leftOp, {RetVectorType}<{RetBaseType}> trueOp, void* output, [CallerMemberName] string method = "")
{
{RetBaseType}[] mask = new {RetBaseType}[RetElementCount];
{Op1BaseType}[] firstOp = new {Op1BaseType}[Op1ElementCount];
{RetBaseType}[] trueVal = new {RetBaseType}[RetElementCount];
{RetBaseType}[] result = new {RetBaseType}[RetElementCount];

Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref mask[0]), maskOp);
Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref firstOp[0]), leftOp);
Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref trueVal[0]), trueOp);
Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref result[0]), ref Unsafe.AsRef<byte>(output), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>());

bool succeeded = true;
{TemplateValidationLogicForCndSel_FalseValue}

if (!succeeded)
{
TestLibrary.TestFramework.LogInformation($"{nameof(Sve)}.{nameof({Isa}.{Method})}<{RetBaseType}>({RetVectorType}<{RetBaseType}>, {RetVectorType}<{RetBaseType}>): {method} failed:");
TestLibrary.TestFramework.LogInformation($" mask: ({string.Join(", ", mask)})");
TestLibrary.TestFramework.LogInformation($"firstOp: ({string.Join(", ", firstOp)})");
TestLibrary.TestFramework.LogInformation($" trueOp: ({string.Join(", ", trueVal)})");
TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})");
TestLibrary.TestFramework.LogInformation(string.Empty);

Succeeded = false;
}
}

private void ValidateResult({Op1VectorType}<{Op1BaseType}> op1, void* result, [CallerMemberName] string method = "")
{
{Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount];
Expand Down
57 changes: 57 additions & 0 deletions src/tests/JIT/Regression/JitBlue/Runtime_106868/Runtime_106868.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using System.Runtime.CompilerServices;
using Xunit;

// Generated by Fuzzlyn v2.3 on 2024-08-23 10:17:54
// Run on Arm64 Windows
// Seed: 14752078066107523191-vectort,vector64,vector128,armsve
// Reduced from 52.7 KiB to 0.7 KiB in 00:00:54
// Hits JIT assert in Release:
// Assertion failed '!"Got unexpected instruction format after MOVPRFX"' in 'Program:Main(Fuzzlyn.ExecutionServer.IRuntime)' during 'Emit code' (IL size 54; hash 0xade6b36b; FullOpts)
//
// File: C:\dev\dotnet\runtime2\src\coreclr\jit\emitarm64sve.cpp Line: 18623
//

using System;
using System.Numerics;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.Arm;

public struct S2
{
public Vector<int> F2;
}

public class Runtime_106868
{
public static S2 s_1;

public static int M4()
{
Vector<long> vr17 = default(Vector<long>);
var vr11 = Vector.Create<long>(0);
var vr8 = Sve.SubtractSaturate(vr17, vr11);
return 1;
}

[Fact]
public static void TestEntryPoint()
{
if (Sve.IsSupported)
{
var vr12 = Vector.Create<int>(0);
var vr13 = Vector.Create<int>(0);
var vr14 = M4();
var vr15 = Vector128.CreateScalar(vr14).AsVector();
var vr16 = Sve.AndAcross(vr13);
s_1.F2 = Sve.ConditionalSelect(vr12, vr16, vr15);
Consume(s_1.F2);
}
}

[MethodImpl(MethodImplOptions.NoInlining)]
private static void Consume(Vector<int> v)
{
}
}
9 changes: 9 additions & 0 deletions src/tests/JIT/Regression/JitBlue/Runtime_106868/Runtime_106868.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<Optimize>True</Optimize>
<NoWarn>$(NoWarn),SYSLIB5003</NoWarn>
</PropertyGroup>
<ItemGroup>
<Compile Include="$(MSBuildProjectName).cs" />
</ItemGroup>
</Project>
Loading