From 75738530d846c93228977ec769471d9c6b9d9ead Mon Sep 17 00:00:00 2001 From: Yeting Kuo Date: Mon, 1 Jul 2024 21:08:26 -0700 Subject: [PATCH 1/2] [RISCV] Recommit "Expand vp.stride.load to splat of a scalar load." This is recommit of #98140. It should be based on #98205 which changes the feature of hardware zero stride optimization. It's a similar patch as a214c521f8763b36dd400b89017f74ad5ae4b6c7 for vp.stride.load. Some targets prefer pattern (vmv.v.x (load)) instead of vlse with zero stride. --- llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp | 48 +++++++++++++++++++ .../RISCV/rvv/fixed-vectors-strided-vpload.ll | 26 ++++++++++ llvm/test/CodeGen/RISCV/rvv/strided-vpload.ll | 24 ++++++++++ 3 files changed, 98 insertions(+) diff --git a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp index 6e0f429c34b2f..efdebdfb796c1 100644 --- a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp +++ b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp @@ -18,9 +18,11 @@ #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/IR/Dominators.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/InstVisitor.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicsRISCV.h" #include "llvm/IR/PatternMatch.h" #include "llvm/InitializePasses.h" #include "llvm/Pass.h" @@ -35,6 +37,7 @@ namespace { class RISCVCodeGenPrepare : public FunctionPass, public InstVisitor { const DataLayout *DL; + const DominatorTree *DT; const RISCVSubtarget *ST; public: @@ -48,12 +51,14 @@ class RISCVCodeGenPrepare : public FunctionPass, void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesCFG(); + AU.addRequired(); AU.addRequired(); } bool visitInstruction(Instruction &I) { return false; } bool visitAnd(BinaryOperator &BO); bool visitIntrinsicInst(IntrinsicInst &I); + bool expandVPStrideLoad(IntrinsicInst &I); }; } // end anonymous namespace @@ -128,6 +133,9 @@ bool RISCVCodeGenPrepare::visitAnd(BinaryOperator &BO) { // Which eliminates the scalar -> vector -> scalar crossing during instruction // selection. bool RISCVCodeGenPrepare::visitIntrinsicInst(IntrinsicInst &I) { + if (expandVPStrideLoad(I)) + return true; + if (I.getIntrinsicID() != Intrinsic::vector_reduce_fadd) return false; @@ -155,6 +163,45 @@ bool RISCVCodeGenPrepare::visitIntrinsicInst(IntrinsicInst &I) { return true; } +bool RISCVCodeGenPrepare::expandVPStrideLoad(IntrinsicInst &II) { + Value *BasePtr, *VL; + + using namespace PatternMatch; + if (!match(&II, m_Intrinsic( + m_Value(BasePtr), m_Zero(), m_AllOnes(), m_Value(VL)))) + return false; + + if (!isKnownNonZero(VL, {*DL, DT, nullptr, &II})) + return false; + + auto *VTy = cast(II.getType()); + + IRBuilder<> Builder(&II); + + // Extend VL from i32 to XLen if needed. + if (ST->is64Bit()) + VL = Builder.CreateZExt(VL, Builder.getInt64Ty()); + + Type *STy = VTy->getElementType(); + Value *Val = Builder.CreateLoad(STy, BasePtr); + const auto &TLI = *ST->getTargetLowering(); + Value *Res; + + // TODO: Also support fixed/illegal vector types to splat with evl = vl. + if (isa(VTy) && TLI.isTypeLegal(EVT::getEVT(VTy))) { + unsigned VMVOp = STy->isFloatingPointTy() ? Intrinsic::riscv_vfmv_v_f + : Intrinsic::riscv_vmv_v_x; + Res = Builder.CreateIntrinsic(VMVOp, {VTy, VL->getType()}, + {PoisonValue::get(VTy), Val, VL}); + } else { + Res = Builder.CreateVectorSplat(VTy->getElementCount(), Val); + } + + II.replaceAllUsesWith(Res); + II.eraseFromParent(); + return true; +} + bool RISCVCodeGenPrepare::runOnFunction(Function &F) { if (skipFunction(F)) return false; @@ -164,6 +211,7 @@ bool RISCVCodeGenPrepare::runOnFunction(Function &F) { ST = &TM.getSubtarget(F); DL = &F.getDataLayout(); + DT = &getAnalysis().getDomTree(); bool MadeChange = false; for (auto &BB : F) diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-vpload.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-vpload.ll index 5e64e9fbc1a2f..df5730e22cbe7 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-vpload.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-vpload.ll @@ -626,3 +626,29 @@ define <33 x double> @strided_load_v33f64(ptr %ptr, i64 %stride, <33 x i1> %mask } declare <33 x double> @llvm.experimental.vp.strided.load.v33f64.p0.i64(ptr, i64, <33 x i1>, i32) + +; TODO: Use accurate evl. +; Test unmasked integer zero strided +define <4 x i8> @zero_strided_unmasked_vpload_4i8_i8(ptr %ptr) { +; CHECK-LABEL: zero_strided_unmasked_vpload_4i8_i8: +; CHECK: # %bb.0: +; CHECK-NEXT: lbu a0, 0(a0) +; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, ma +; CHECK-NEXT: vmv.v.x v8, a0 +; CHECK-NEXT: ret + %load = call <4 x i8> @llvm.experimental.vp.strided.load.4i8.p0.i8(ptr %ptr, i8 0, <4 x i1> splat (i1 true), i32 3) + ret <4 x i8> %load +} + +; TODO: Use accurate evl. +; Test unmasked float zero strided +define <4 x half> @zero_strided_unmasked_vpload_4f16(ptr %ptr) { +; CHECK-LABEL: zero_strided_unmasked_vpload_4f16: +; CHECK: # %bb.0: +; CHECK-NEXT: flh fa5, 0(a0) +; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma +; CHECK-NEXT: vfmv.v.f v8, fa5 +; CHECK-NEXT: ret + %load = call <4 x half> @llvm.experimental.vp.strided.load.4f16.p0.i32(ptr %ptr, i32 0, <4 x i1> splat (i1 true), i32 3) + ret <4 x half> %load +} diff --git a/llvm/test/CodeGen/RISCV/rvv/strided-vpload.ll b/llvm/test/CodeGen/RISCV/rvv/strided-vpload.ll index 4d3bced0bcb50..33e16a3c09410 100644 --- a/llvm/test/CodeGen/RISCV/rvv/strided-vpload.ll +++ b/llvm/test/CodeGen/RISCV/rvv/strided-vpload.ll @@ -780,3 +780,27 @@ define @strided_load_nxv17f64(ptr %ptr, i64 %stride, @llvm.experimental.vp.strided.load.nxv17f64.p0.i64(ptr, i64, , i32) declare @llvm.experimental.vector.extract.nxv1f64( %vec, i64 %idx) declare @llvm.experimental.vector.extract.nxv16f64( %vec, i64 %idx) + +; Test unmasked integer zero strided +define @zero_strided_unmasked_vpload_nxv1i8_i8(ptr %ptr) { +; CHECK-LABEL: zero_strided_unmasked_vpload_nxv1i8_i8: +; CHECK: # %bb.0: +; CHECK-NEXT: lbu a0, 0(a0) +; CHECK-NEXT: vsetivli zero, 4, e8, mf8, ta, ma +; CHECK-NEXT: vmv.v.x v8, a0 +; CHECK-NEXT: ret + %load = call @llvm.experimental.vp.strided.load.nxv1i8.p0.i8(ptr %ptr, i8 0, splat (i1 true), i32 4) + ret %load +} + +; Test unmasked float zero strided +define @zero_strided_unmasked_vpload_nxv1f16(ptr %ptr) { +; CHECK-LABEL: zero_strided_unmasked_vpload_nxv1f16: +; CHECK: # %bb.0: +; CHECK-NEXT: flh fa5, 0(a0) +; CHECK-NEXT: vsetivli zero, 4, e16, mf4, ta, ma +; CHECK-NEXT: vfmv.v.f v8, fa5 +; CHECK-NEXT: ret + %load = call @llvm.experimental.vp.strided.load.nxv1f16.p0.i32(ptr %ptr, i32 0, splat (i1 true), i32 4) + ret %load +} From b36d06aea666f5f9e45be5ec25f0991199929722 Mon Sep 17 00:00:00 2001 From: Yeting Kuo Date: Sun, 14 Jul 2024 23:00:17 -0700 Subject: [PATCH 2/2] Add hasOptimizedZeroStrideLoad check back and test for optimized-zero-stride-load. --- llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp | 3 ++ .../RISCV/rvv/fixed-vectors-strided-vpload.ll | 46 +++++++++++------ llvm/test/CodeGen/RISCV/rvv/strided-vpload.ll | 50 +++++++++++++------ 3 files changed, 69 insertions(+), 30 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp index efdebdfb796c1..35c46157c2eb9 100644 --- a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp +++ b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp @@ -164,6 +164,9 @@ bool RISCVCodeGenPrepare::visitIntrinsicInst(IntrinsicInst &I) { } bool RISCVCodeGenPrepare::expandVPStrideLoad(IntrinsicInst &II) { + if (ST->hasOptimizedZeroStrideLoad()) + return false; + Value *BasePtr, *VL; using namespace PatternMatch; diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-vpload.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-vpload.ll index df5730e22cbe7..41c7d1f5fd64c 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-vpload.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-vpload.ll @@ -1,10 +1,16 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+v,+zvfh,+optimized-zero-stride-load \ +; RUN: -verify-machineinstrs < %s \ +; RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-RV32,CHECK-OPT +; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+v,+zvfh,+optimized-zero-stride-load \ +; RUN: -verify-machineinstrs < %s \ +; RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-RV64,CHECK-OPT ; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+v,+zvfh \ ; RUN: -verify-machineinstrs < %s \ -; RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-RV32 +; RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-RV32,CHECK-NO-OPT ; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+v,+zvfh \ ; RUN: -verify-machineinstrs < %s \ -; RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-RV64 +; RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-RV64,CHECK-NO-OPT declare <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i8(ptr, i8, <2 x i1>, i32) @@ -630,12 +636,18 @@ declare <33 x double> @llvm.experimental.vp.strided.load.v33f64.p0.i64(ptr, i64, ; TODO: Use accurate evl. ; Test unmasked integer zero strided define <4 x i8> @zero_strided_unmasked_vpload_4i8_i8(ptr %ptr) { -; CHECK-LABEL: zero_strided_unmasked_vpload_4i8_i8: -; CHECK: # %bb.0: -; CHECK-NEXT: lbu a0, 0(a0) -; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, ma -; CHECK-NEXT: vmv.v.x v8, a0 -; CHECK-NEXT: ret +; CHECK-OPT-LABEL: zero_strided_unmasked_vpload_4i8_i8: +; CHECK-OPT: # %bb.0: +; CHECK-OPT-NEXT: vsetivli zero, 3, e8, mf4, ta, ma +; CHECK-OPT-NEXT: vlse8.v v8, (a0), zero +; CHECK-OPT-NEXT: ret +; +; CHECK-NO-OPT-LABEL: zero_strided_unmasked_vpload_4i8_i8: +; CHECK-NO-OPT: # %bb.0: +; CHECK-NO-OPT-NEXT: lbu a0, 0(a0) +; CHECK-NO-OPT-NEXT: vsetivli zero, 4, e8, mf4, ta, ma +; CHECK-NO-OPT-NEXT: vmv.v.x v8, a0 +; CHECK-NO-OPT-NEXT: ret %load = call <4 x i8> @llvm.experimental.vp.strided.load.4i8.p0.i8(ptr %ptr, i8 0, <4 x i1> splat (i1 true), i32 3) ret <4 x i8> %load } @@ -643,12 +655,18 @@ define <4 x i8> @zero_strided_unmasked_vpload_4i8_i8(ptr %ptr) { ; TODO: Use accurate evl. ; Test unmasked float zero strided define <4 x half> @zero_strided_unmasked_vpload_4f16(ptr %ptr) { -; CHECK-LABEL: zero_strided_unmasked_vpload_4f16: -; CHECK: # %bb.0: -; CHECK-NEXT: flh fa5, 0(a0) -; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma -; CHECK-NEXT: vfmv.v.f v8, fa5 -; CHECK-NEXT: ret +; CHECK-OPT-LABEL: zero_strided_unmasked_vpload_4f16: +; CHECK-OPT: # %bb.0: +; CHECK-OPT-NEXT: vsetivli zero, 3, e16, mf2, ta, ma +; CHECK-OPT-NEXT: vlse16.v v8, (a0), zero +; CHECK-OPT-NEXT: ret +; +; CHECK-NO-OPT-LABEL: zero_strided_unmasked_vpload_4f16: +; CHECK-NO-OPT: # %bb.0: +; CHECK-NO-OPT-NEXT: flh fa5, 0(a0) +; CHECK-NO-OPT-NEXT: vsetivli zero, 4, e16, mf2, ta, ma +; CHECK-NO-OPT-NEXT: vfmv.v.f v8, fa5 +; CHECK-NO-OPT-NEXT: ret %load = call <4 x half> @llvm.experimental.vp.strided.load.4f16.p0.i32(ptr %ptr, i32 0, <4 x i1> splat (i1 true), i32 3) ret <4 x half> %load } diff --git a/llvm/test/CodeGen/RISCV/rvv/strided-vpload.ll b/llvm/test/CodeGen/RISCV/rvv/strided-vpload.ll index 33e16a3c09410..6b8ded4914226 100644 --- a/llvm/test/CodeGen/RISCV/rvv/strided-vpload.ll +++ b/llvm/test/CodeGen/RISCV/rvv/strided-vpload.ll @@ -1,10 +1,16 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+v,+zvfh,+optimized-zero-stride-load \ +; RUN: -verify-machineinstrs < %s \ +; RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-RV32,CHECK-OPT +; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+v,+zvfh,+optimized-zero-stride-load \ +; RUN: -verify-machineinstrs < %s \ +; RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-RV64,CHECK-OPT ; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+v,+zvfh \ -; RUN: -verify-machineinstrs < %s | FileCheck %s \ -; RUN: -check-prefixes=CHECK,CHECK-RV32 +; RUN: -verify-machineinstrs < %s \ +; RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-RV32,CHECK-NO-OPT ; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+v,+zvfh \ -; RUN: -verify-machineinstrs < %s | FileCheck %s \ -; RUN: -check-prefixes=CHECK,CHECK-RV64 +; RUN: -verify-machineinstrs < %s \ +; RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-RV64,CHECK-NO-OPT declare @llvm.experimental.vp.strided.load.nxv1i8.p0.i8(ptr, i8, , i32) @@ -783,24 +789,36 @@ declare @llvm.experimental.vector.extract.nxv16f64( @zero_strided_unmasked_vpload_nxv1i8_i8(ptr %ptr) { -; CHECK-LABEL: zero_strided_unmasked_vpload_nxv1i8_i8: -; CHECK: # %bb.0: -; CHECK-NEXT: lbu a0, 0(a0) -; CHECK-NEXT: vsetivli zero, 4, e8, mf8, ta, ma -; CHECK-NEXT: vmv.v.x v8, a0 -; CHECK-NEXT: ret +; CHECK-OPT-LABEL: zero_strided_unmasked_vpload_nxv1i8_i8: +; CHECK-OPT: # %bb.0: +; CHECK-OPT-NEXT: vsetivli zero, 4, e8, mf8, ta, ma +; CHECK-OPT-NEXT: vlse8.v v8, (a0), zero +; CHECK-OPT-NEXT: ret +; +; CHECK-NO-OPT-LABEL: zero_strided_unmasked_vpload_nxv1i8_i8: +; CHECK-NO-OPT: # %bb.0: +; CHECK-NO-OPT-NEXT: lbu a0, 0(a0) +; CHECK-NO-OPT-NEXT: vsetivli zero, 4, e8, mf8, ta, ma +; CHECK-NO-OPT-NEXT: vmv.v.x v8, a0 +; CHECK-NO-OPT-NEXT: ret %load = call @llvm.experimental.vp.strided.load.nxv1i8.p0.i8(ptr %ptr, i8 0, splat (i1 true), i32 4) ret %load } ; Test unmasked float zero strided define @zero_strided_unmasked_vpload_nxv1f16(ptr %ptr) { -; CHECK-LABEL: zero_strided_unmasked_vpload_nxv1f16: -; CHECK: # %bb.0: -; CHECK-NEXT: flh fa5, 0(a0) -; CHECK-NEXT: vsetivli zero, 4, e16, mf4, ta, ma -; CHECK-NEXT: vfmv.v.f v8, fa5 -; CHECK-NEXT: ret +; CHECK-OPT-LABEL: zero_strided_unmasked_vpload_nxv1f16: +; CHECK-OPT: # %bb.0: +; CHECK-OPT-NEXT: vsetivli zero, 4, e16, mf4, ta, ma +; CHECK-OPT-NEXT: vlse16.v v8, (a0), zero +; CHECK-OPT-NEXT: ret +; +; CHECK-NO-OPT-LABEL: zero_strided_unmasked_vpload_nxv1f16: +; CHECK-NO-OPT: # %bb.0: +; CHECK-NO-OPT-NEXT: flh fa5, 0(a0) +; CHECK-NO-OPT-NEXT: vsetivli zero, 4, e16, mf4, ta, ma +; CHECK-NO-OPT-NEXT: vfmv.v.f v8, fa5 +; CHECK-NO-OPT-NEXT: ret %load = call @llvm.experimental.vp.strided.load.nxv1f16.p0.i32(ptr %ptr, i32 0, splat (i1 true), i32 4) ret %load }