diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td index 573342846b4cf2..83ef0af20242ef 100644 --- a/llvm/include/llvm/Target/TargetSelectionDAG.td +++ b/llvm/include/llvm/Target/TargetSelectionDAG.td @@ -2141,6 +2141,12 @@ def atomic_load_nonext_128 : let MemoryVT = i128; } +def atomic_load_nonext_v4i32 : + PatFrag<(ops node:$ptr), (atomic_load_nonext node:$ptr)> { + let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic? + let MemoryVT = v4i32; +} + def atomic_load_zext_8 : PatFrag<(ops node:$ptr), (atomic_load_zext node:$ptr)> { let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic? diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp index 11f30f62109395..909029ccf14584 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -168,6 +168,12 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM, setOperationAction(ISD::ATOMIC_LOAD, MVT::bf16, Promote); AddPromotedToType(ISD::ATOMIC_LOAD, MVT::bf16, MVT::i16); + setOperationAction(ISD::ATOMIC_LOAD, MVT::v2f32, Promote); + AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v2f32, MVT::i64); + + setOperationAction(ISD::ATOMIC_LOAD, MVT::v4f32, Promote); + AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v4f32, MVT::v4i32); + setOperationAction(ISD::ATOMIC_STORE, MVT::f32, Promote); AddPromotedToType(ISD::ATOMIC_STORE, MVT::f32, MVT::i32); @@ -636,7 +642,7 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM, ISD::FABS, ISD::AssertZext, ISD::AssertSext, ISD::INTRINSIC_WO_CHAIN}); - setMaxAtomicSizeInBitsSupported(64); + setMaxAtomicSizeInBitsSupported(128); setMaxDivRemBitWidthSupported(64); setMaxLargeFPConvertBitWidthSupported(64); } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td index 9b6443a99ac559..9385f86b2a33ed 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td @@ -513,6 +513,10 @@ def atomic_load_nonext_128_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_12 let IsAtomic = 1; } +def atomic_load_nonext_v4i32_#as : PatFrag<(ops node:$ptr), (atomic_load_nonext_v4i32 node:$ptr)> { + let IsAtomic = 1; +} + def atomic_load_zext_8_#as : PatFrag<(ops node:$ptr), (atomic_load_zext_8 node:$ptr)> { let IsAtomic = 1; } diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td index c0fb73df9c7640..824ccdcc70fa2d 100644 --- a/llvm/lib/Target/AMDGPU/FLATInstructions.td +++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td @@ -2101,9 +2101,12 @@ let OtherPredicates = [HasFlatAddressSpace], True16Predicate = UseRealTrue16Inst } defm : FlatLoadPats ; +defm : FlatLoadPats ; defm : FlatLoadPats ; +defm : FlatLoadPats ; defm : FlatLoadPats ; defm : FlatLoadPats ; +defm : FlatLoadPats ; defm : FlatStorePats ; defm : FlatStorePats ; @@ -2281,9 +2284,12 @@ defm : GlobalFLATStorePats ; // the memory legalizer will set the cache bits and insert the // appropriate waits. defm : GlobalFLATLoadPats ; +defm : GlobalFLATLoadPats ; defm : GlobalFLATLoadPats ; defm : GlobalFLATLoadPats ; +defm : GlobalFLATLoadPats ; defm : GlobalFLATLoadPats ; +defm : GlobalFLATLoadPats ; defm : GlobalFLATStorePats ; defm : GlobalFLATStorePats ; diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 6750dfcbaac624..525a530b21ca73 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -703,6 +703,9 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, setOperationAction(ISD::LOAD, MVT::v2f16, Promote); AddPromotedToType(ISD::LOAD, MVT::v2f16, MVT::i32); + setOperationAction(ISD::ATOMIC_LOAD, MVT::v2f16, Promote); + AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v2f16, MVT::i32); + setOperationAction(ISD::AND, MVT::v2i16, Promote); AddPromotedToType(ISD::AND, MVT::v2i16, MVT::i32); setOperationAction(ISD::OR, MVT::v2i16, Promote); @@ -717,6 +720,12 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, setOperationAction(ISD::LOAD, MVT::v4bf16, Promote); AddPromotedToType(ISD::LOAD, MVT::v4bf16, MVT::v2i32); + setOperationAction(ISD::ATOMIC_LOAD, MVT::v4f16, Promote); + AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v4f16, MVT::i64); + + setOperationAction(ISD::ATOMIC_LOAD, MVT::v8f16, Promote); + AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v8f16, MVT::v4i32); + setOperationAction(ISD::STORE, MVT::v4i16, Promote); AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::v2i32); setOperationAction(ISD::STORE, MVT::v4f16, Promote); @@ -19689,6 +19698,27 @@ getPrivateAtomicExpansionKind(const GCNSubtarget &STI) { : TargetLowering::AtomicExpansionKind::NotAtomic; } +/// Returns true if \p Ty is a vector float type that AMDGPU supports for +/// atomic load. These types are promoted to an equivalently sized integer +/// type before instruction selection, so the backend needs to be told +/// explicitly which vector types are allowed through the atomic expand pass. +/// TODO: Support atomic store for these types. +static bool isSupportedAtomicVectorType(Type *Ty) { + auto *VT = dyn_cast(Ty); + if (!VT) + return false; + + Type *EltTy = VT->getElementType(); + unsigned NumElts = VT->getNumElements(); + // Support v2f32, v4f32. + if (EltTy->isFloatTy()) + return NumElts == 2 || NumElts == 4; + // Support v2f16, v4f16, v8f16. + if (EltTy->isHalfTy()) + return NumElts == 2 || NumElts == 4 || NumElts == 8; + return false; +} + TargetLowering::AtomicExpansionKind SITargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const { unsigned AS = RMW->getPointerAddressSpace(); @@ -19975,30 +20005,43 @@ SITargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const { TargetLowering::AtomicExpansionKind SITargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const { - return LI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS - ? getPrivateAtomicExpansionKind(*getSubtarget()) - : AtomicExpansionKind::None; + if (LI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS) + return getPrivateAtomicExpansionKind(*getSubtarget()); + + const DataLayout &DL = LI->getDataLayout(); + if (DL.getTypeSizeInBits(LI->getType()) > 64 && + !isSupportedAtomicVectorType(LI->getType())) + return AtomicExpansionKind::CustomExpand; + + return AtomicExpansionKind::None; } TargetLowering::AtomicExpansionKind SITargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const { - return SI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS - ? getPrivateAtomicExpansionKind(*getSubtarget()) - : AtomicExpansionKind::None; + if (SI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS) + return getPrivateAtomicExpansionKind(*getSubtarget()); + + const DataLayout &DL = SI->getDataLayout(); + if (DL.getTypeSizeInBits(SI->getValueOperand()->getType()) > 64) + return AtomicExpansionKind::CustomExpand; + + return AtomicExpansionKind::None; } TargetLowering::AtomicExpansionKind SITargetLowering::shouldExpandAtomicCmpXchgInIR( const AtomicCmpXchgInst *CmpX) const { unsigned AddrSpace = CmpX->getPointerAddressSpace(); + const DataLayout &DL = CmpX->getDataLayout(); + if (DL.getTypeSizeInBits(CmpX->getCompareOperand()->getType()) > 64) + return AtomicExpansionKind::CustomExpand; + if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS) return getPrivateAtomicExpansionKind(*getSubtarget()); if (AddrSpace != AMDGPUAS::FLAT_ADDRESS || !flatInstrMayAccessPrivate(CmpX)) return AtomicExpansionKind::None; - const DataLayout &DL = CmpX->getDataLayout(); - Type *ValTy = CmpX->getNewValOperand()->getType(); // If a 64-bit flat atomic may alias private, we need to avoid using the @@ -20383,6 +20426,14 @@ void SITargetLowering::emitExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) const { if (CI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS) return convertScratchAtomicToFlatAtomic(CI, CI->getPointerOperandIndex()); + const DataLayout &DL = CI->getDataLayout(); + if (DL.getTypeSizeInBits(CI->getCompareOperand()->getType()) > 64) { + CI->getContext().emitError(CI, "unsupported cmpxchg"); + CI->replaceAllUsesWith(PoisonValue::get(CI->getType())); + CI->eraseFromParent(); + return; + } + emitExpandAtomicAddrSpacePredicate(CI); } @@ -20390,6 +20441,14 @@ void SITargetLowering::emitExpandAtomicLoad(LoadInst *LI) const { if (LI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS) return convertScratchAtomicToFlatAtomic(LI, LI->getPointerOperandIndex()); + const DataLayout &DL = LI->getDataLayout(); + if (DL.getTypeSizeInBits(LI->getType()) > 64) { + LI->getContext().emitError(LI, "unsupported atomic load"); + LI->replaceAllUsesWith(PoisonValue::get(LI->getType())); + LI->eraseFromParent(); + return; + } + llvm_unreachable( "Expand Atomic Load only handles SCRATCH -> FLAT conversion"); } @@ -20398,6 +20457,13 @@ void SITargetLowering::emitExpandAtomicStore(StoreInst *SI) const { if (SI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS) return convertScratchAtomicToFlatAtomic(SI, SI->getPointerOperandIndex()); + const DataLayout &DL = SI->getDataLayout(); + if (DL.getTypeSizeInBits(SI->getValueOperand()->getType()) > 64) { + SI->getContext().emitError(SI, "unsupported atomic store"); + SI->eraseFromParent(); + return; + } + llvm_unreachable( "Expand Atomic Store only handles SCRATCH -> FLAT conversion"); } diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir index 5bfb2b2e4d5782..eebf5fac2d9d1a 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir @@ -70,37 +70,37 @@ body: | ; GFX7-LABEL: name: load_atomic_flat_v2s16_seq_cst ; GFX7: liveins: $vgpr0_vgpr1 ; GFX7-NEXT: {{ $}} - ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1 - ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s16>)) - ; GFX7-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>) + ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX7-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>)) + ; GFX7-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]] ; ; GFX9-LABEL: name: load_atomic_flat_v2s16_seq_cst ; GFX9: liveins: $vgpr0_vgpr1 ; GFX9-NEXT: {{ $}} - ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1 - ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s16>)) - ; GFX9-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>) + ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX9-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>)) + ; GFX9-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]] ; ; GFX10-LABEL: name: load_atomic_flat_v2s16_seq_cst ; GFX10: liveins: $vgpr0_vgpr1 ; GFX10-NEXT: {{ $}} - ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1 - ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s16>)) - ; GFX10-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>) + ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX10-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>)) + ; GFX10-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]] ; ; GFX11-LABEL: name: load_atomic_flat_v2s16_seq_cst ; GFX11: liveins: $vgpr0_vgpr1 ; GFX11-NEXT: {{ $}} - ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1 - ; GFX11-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s16>)) - ; GFX11-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>) + ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX11-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>)) + ; GFX11-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]] ; ; GFX12-LABEL: name: load_atomic_flat_v2s16_seq_cst ; GFX12: liveins: $vgpr0_vgpr1 ; GFX12-NEXT: {{ $}} - ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1 - ; GFX12-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<2 x s16>)) - ; GFX12-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>) + ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX12-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>)) + ; GFX12-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]] %0:vgpr(p0) = COPY $vgpr0_vgpr1 %1:vgpr(<2 x s16>) = G_LOAD %0 :: (load seq_cst (<2 x s16>), align 4, addrspace 0) $vgpr0 = COPY %1 @@ -274,37 +274,37 @@ body: | ; GFX7-LABEL: name: load_atomic_flat_v4s16_seq_cst ; GFX7: liveins: $vgpr0_vgpr1 ; GFX7-NEXT: {{ $}} - ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1 - ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<4 x s16>)) - ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>) + ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX7-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>)) + ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]] ; ; GFX9-LABEL: name: load_atomic_flat_v4s16_seq_cst ; GFX9: liveins: $vgpr0_vgpr1 ; GFX9-NEXT: {{ $}} - ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1 - ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<4 x s16>)) - ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>) + ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX9-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>)) + ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]] ; ; GFX10-LABEL: name: load_atomic_flat_v4s16_seq_cst ; GFX10: liveins: $vgpr0_vgpr1 ; GFX10-NEXT: {{ $}} - ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1 - ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<4 x s16>)) - ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>) + ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX10-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>)) + ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]] ; ; GFX11-LABEL: name: load_atomic_flat_v4s16_seq_cst ; GFX11: liveins: $vgpr0_vgpr1 ; GFX11-NEXT: {{ $}} - ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1 - ; GFX11-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<4 x s16>)) - ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>) + ; GFX11-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX11-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>)) + ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]] ; ; GFX12-LABEL: name: load_atomic_flat_v4s16_seq_cst ; GFX12: liveins: $vgpr0_vgpr1 ; GFX12-NEXT: {{ $}} - ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1 - ; GFX12-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p0) :: (load seq_cst (<4 x s16>)) - ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>) + ; GFX12-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX12-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>)) + ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]] %0:vgpr(p0) = COPY $vgpr0_vgpr1 %1:vgpr(<4 x s16>) = G_LOAD %0 :: (load seq_cst (<4 x s16>), align 8, addrspace 0) $vgpr0_vgpr1 = COPY %1 diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir index 474f1308d8e24f..3a79370b750bac 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-global.mir @@ -89,30 +89,30 @@ body: | ; GFX7-LABEL: name: load_atomic_global_v2s16_seq_cst ; GFX7: liveins: $vgpr0_vgpr1 ; GFX7-NEXT: {{ $}} - ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1 - ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s16>), addrspace 1) - ; GFX7-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>) + ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX7-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>), addrspace 1) + ; GFX7-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]] ; ; GFX7-FLAT-LABEL: name: load_atomic_global_v2s16_seq_cst ; GFX7-FLAT: liveins: $vgpr0_vgpr1 ; GFX7-FLAT-NEXT: {{ $}} - ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1 - ; GFX7-FLAT-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s16>), addrspace 1) - ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>) + ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX7-FLAT-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<2 x s16>), addrspace 1) + ; GFX7-FLAT-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]] ; ; GFX9-LABEL: name: load_atomic_global_v2s16_seq_cst ; GFX9: liveins: $vgpr0_vgpr1 ; GFX9-NEXT: {{ $}} - ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1 - ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s16>), addrspace 1) - ; GFX9-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>) + ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX9-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load seq_cst (<2 x s16>), addrspace 1) + ; GFX9-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]] ; ; GFX10-LABEL: name: load_atomic_global_v2s16_seq_cst ; GFX10: liveins: $vgpr0_vgpr1 ; GFX10-NEXT: {{ $}} - ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1 - ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<2 x s16>), addrspace 1) - ; GFX10-NEXT: $vgpr0 = COPY [[LOAD]](<2 x s16>) + ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX10-NEXT: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, implicit $exec :: (load seq_cst (<2 x s16>), addrspace 1) + ; GFX10-NEXT: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]] %0:vgpr(p1) = COPY $vgpr0_vgpr1 %1:vgpr(<2 x s16>) = G_LOAD %0 :: (load seq_cst (<2 x s16>), align 4, addrspace 1) $vgpr0 = COPY %1 @@ -303,30 +303,30 @@ body: | ; GFX7-LABEL: name: load_atomic_global_v4s16_seq_cst ; GFX7: liveins: $vgpr0_vgpr1 ; GFX7-NEXT: {{ $}} - ; GFX7-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1 - ; GFX7-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<4 x s16>), addrspace 1) - ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>) + ; GFX7-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX7-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>), addrspace 1) + ; GFX7-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]] ; ; GFX7-FLAT-LABEL: name: load_atomic_global_v4s16_seq_cst ; GFX7-FLAT: liveins: $vgpr0_vgpr1 ; GFX7-FLAT-NEXT: {{ $}} - ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1 - ; GFX7-FLAT-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<4 x s16>), addrspace 1) - ; GFX7-FLAT-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>) + ; GFX7-FLAT-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX7-FLAT-NEXT: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (<4 x s16>), addrspace 1) + ; GFX7-FLAT-NEXT: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]] ; ; GFX9-LABEL: name: load_atomic_global_v4s16_seq_cst ; GFX9: liveins: $vgpr0_vgpr1 ; GFX9-NEXT: {{ $}} - ; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1 - ; GFX9-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<4 x s16>), addrspace 1) - ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>) + ; GFX9-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX9-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load seq_cst (<4 x s16>), addrspace 1) + ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]] ; ; GFX10-LABEL: name: load_atomic_global_v4s16_seq_cst ; GFX10: liveins: $vgpr0_vgpr1 ; GFX10-NEXT: {{ $}} - ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1 - ; GFX10-NEXT: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load seq_cst (<4 x s16>), addrspace 1) - ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>) + ; GFX10-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1 + ; GFX10-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load seq_cst (<4 x s16>), addrspace 1) + ; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]] %0:vgpr(p1) = COPY $vgpr0_vgpr1 %1:vgpr(<4 x s16>) = G_LOAD %0 :: (load seq_cst (<4 x s16>), align 8, addrspace 1) $vgpr0_vgpr1 = COPY %1 diff --git a/llvm/test/CodeGen/AMDGPU/atomic-load.ll b/llvm/test/CodeGen/AMDGPU/atomic-load.ll new file mode 100644 index 00000000000000..460af0f7602480 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/atomic-load.ll @@ -0,0 +1,180 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 +; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefixes=SDAG %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefixes=GISEL %s + +define amdgpu_cs void @atomic_two_load_monotonic_f32(ptr addrspace(1) %p, ptr addrspace(1) %out) { +; SDAG-LABEL: atomic_two_load_monotonic_f32: +; SDAG: ; %bb.0: +; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV +; SDAG-NEXT: s_wait_loadcnt 0x0 +; SDAG-NEXT: v_add_f32_e32 v0, v0, v1 +; SDAG-NEXT: global_store_b32 v[2:3], v0, off +; SDAG-NEXT: s_endpgm +; +; GISEL-LABEL: atomic_two_load_monotonic_f32: +; GISEL: ; %bb.0: +; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV +; GISEL-NEXT: s_wait_loadcnt 0x0 +; GISEL-NEXT: v_add_f32_e32 v0, v0, v1 +; GISEL-NEXT: global_store_b32 v[2:3], v0, off +; GISEL-NEXT: s_endpgm + %a0 = load atomic <2 x float>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8 + %num1 = extractelement <2 x float> %a0, i32 0 + %num2 = extractelement <2 x float> %a0, i32 1 + %res = fadd float %num1, %num2 + store float %res, ptr addrspace(1) %out, align 4 + ret void +} + +define amdgpu_cs void @atomic_two_load_monotonic_half(ptr addrspace(1) %p, ptr addrspace(1) %out) { +; SDAG-LABEL: atomic_two_load_monotonic_half: +; SDAG: ; %bb.0: +; SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV +; SDAG-NEXT: s_wait_loadcnt 0x0 +; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h +; SDAG-NEXT: global_store_b16 v[2:3], v0, off +; SDAG-NEXT: s_endpgm +; +; GISEL-LABEL: atomic_two_load_monotonic_half: +; GISEL: ; %bb.0: +; GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV +; GISEL-NEXT: s_wait_loadcnt 0x0 +; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0 +; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l +; GISEL-NEXT: global_store_b16 v[2:3], v0, off +; GISEL-NEXT: s_endpgm + %a0 = load atomic <2 x half>, ptr addrspace(1) %p syncscope("agent") monotonic, align 4 + %num1 = extractelement <2 x half> %a0, i32 0 + %num2 = extractelement <2 x half> %a0, i32 1 + %res = fadd half %num1, %num2 + store half %res, ptr addrspace(1) %out, align 4 + ret void +} + +define amdgpu_cs void @atomic_four_load_monotonic_half(ptr addrspace(1) %p, ptr addrspace(1) %out) { +; SDAG-LABEL: atomic_four_load_monotonic_half: +; SDAG: ; %bb.0: +; SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV +; SDAG-NEXT: s_wait_loadcnt 0x0 +; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h +; SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h +; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h +; SDAG-NEXT: global_store_b16 v[2:3], v0, off +; SDAG-NEXT: s_endpgm +; +; GISEL-LABEL: atomic_four_load_monotonic_half: +; GISEL: ; %bb.0: +; GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV +; GISEL-NEXT: s_wait_loadcnt 0x0 +; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0 +; GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1 +; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l +; GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l +; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h +; GISEL-NEXT: global_store_b16 v[2:3], v0, off +; GISEL-NEXT: s_endpgm + %a0 = load atomic <4 x half>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8 + %num1 = extractelement <4 x half> %a0, i32 0 + %num2 = extractelement <4 x half> %a0, i32 1 + %num3 = extractelement <4 x half> %a0, i32 2 + %num4 = extractelement <4 x half> %a0, i32 3 + %add = fadd half %num1, %num2 + %mul = fmul half %num3, %num4 + %res = fadd half %add, %mul + store half %res, ptr addrspace(1) %out, align 4 + ret void +} + +define amdgpu_cs void @atomic_four_load_monotonic_f32(ptr addrspace(1) %p, ptr addrspace(1) %out) { +; SDAG-LABEL: atomic_four_load_monotonic_f32: +; SDAG: ; %bb.0: +; SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off scope:SCOPE_DEV +; SDAG-NEXT: s_wait_loadcnt 0x0 +; SDAG-NEXT: v_dual_add_f32 v0, v4, v5 :: v_dual_mul_f32 v1, v6, v7 +; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; SDAG-NEXT: v_add_f32_e32 v0, v0, v1 +; SDAG-NEXT: global_store_b32 v[2:3], v0, off +; SDAG-NEXT: s_endpgm +; +; GISEL-LABEL: atomic_four_load_monotonic_f32: +; GISEL: ; %bb.0: +; GISEL-NEXT: global_load_b128 v[4:7], v[0:1], off scope:SCOPE_DEV +; GISEL-NEXT: s_wait_loadcnt 0x0 +; GISEL-NEXT: v_dual_add_f32 v0, v4, v5 :: v_dual_mul_f32 v1, v6, v7 +; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GISEL-NEXT: v_add_f32_e32 v0, v0, v1 +; GISEL-NEXT: global_store_b32 v[2:3], v0, off +; GISEL-NEXT: s_endpgm + %a0 = load atomic <4 x float>, ptr addrspace(1) %p syncscope("agent") monotonic, align 16 + %num1 = extractelement <4 x float> %a0, i32 0 + %num2 = extractelement <4 x float> %a0, i32 1 + %num3 = extractelement <4 x float> %a0, i32 2 + %num4 = extractelement <4 x float> %a0, i32 3 + %add = fadd float %num1, %num2 + %mul = fmul float %num3, %num4 + %res = fadd float %add, %mul + store float %res, ptr addrspace(1) %out, align 4 + ret void +} + +define amdgpu_cs void @atomic_eight_load_monotonic_half(ptr addrspace(1) %p, ptr addrspace(1) %out) { +; SDAG-LABEL: atomic_eight_load_monotonic_half: +; SDAG: ; %bb.0: +; SDAG-NEXT: global_load_b128 v[4:7], v[0:1], off scope:SCOPE_DEV +; SDAG-NEXT: s_wait_loadcnt 0x0 +; SDAG-NEXT: v_add_f16_e32 v0.l, v4.l, v4.h +; SDAG-NEXT: v_mul_f16_e32 v0.h, v5.l, v5.h +; SDAG-NEXT: v_add_f16_e32 v1.l, v6.l, v6.h +; SDAG-NEXT: v_mul_f16_e32 v1.h, v7.l, v7.h +; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h +; SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h +; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h +; SDAG-NEXT: global_store_b16 v[2:3], v0, off +; SDAG-NEXT: s_endpgm +; +; GISEL-LABEL: atomic_eight_load_monotonic_half: +; GISEL: ; %bb.0: +; GISEL-NEXT: global_load_b128 v[4:7], v[0:1], off scope:SCOPE_DEV +; GISEL-NEXT: s_wait_loadcnt 0x0 +; GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v4 +; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v5 +; GISEL-NEXT: v_lshrrev_b32_e32 v8, 16, v6 +; GISEL-NEXT: v_lshrrev_b32_e32 v9, 16, v7 +; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GISEL-NEXT: v_add_f16_e32 v0.l, v4.l, v0.l +; GISEL-NEXT: v_mul_f16_e32 v0.h, v5.l, v1.l +; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GISEL-NEXT: v_add_f16_e32 v1.l, v6.l, v8.l +; GISEL-NEXT: v_mul_f16_e32 v1.h, v7.l, v9.l +; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h +; GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h +; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h +; GISEL-NEXT: global_store_b16 v[2:3], v0, off +; GISEL-NEXT: s_endpgm + %a0 = load atomic <8 x half>, ptr addrspace(1) %p syncscope("agent") monotonic, align 16 + %num1 = extractelement <8 x half> %a0, i32 0 + %num2 = extractelement <8 x half> %a0, i32 1 + %num3 = extractelement <8 x half> %a0, i32 2 + %num4 = extractelement <8 x half> %a0, i32 3 + %num5 = extractelement <8 x half> %a0, i32 4 + %num6 = extractelement <8 x half> %a0, i32 5 + %num7 = extractelement <8 x half> %a0, i32 6 + %num8 = extractelement <8 x half> %a0, i32 7 + %add1 = fadd half %num1, %num2 + %add2 = fmul half %num3, %num4 + %add3 = fadd half %num5, %num6 + %add4 = fmul half %num7, %num8 + %add5 = fadd half %add1, %add2 + %add6 = fmul half %add3, %add4 + %res = fadd half %add5, %add6 + store half %res, ptr addrspace(1) %out, align 4 + ret void +}