AArch64SelectionDAGInfo.cpp revision 360784 
1//===-- AArch64SelectionDAGInfo.cpp - AArch64 SelectionDAG Info -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the AArch64SelectionDAGInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AArch64TargetMachine.h"
14using namespace llvm;
15
16#define DEBUG_TYPE "aarch64-selectiondag-info"
17
18SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemset(
19    SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
20    SDValue Size, unsigned Align, bool isVolatile,
21    MachinePointerInfo DstPtrInfo) const {
22  // Check to see if there is a specialized entry-point for memory zeroing.
23  ConstantSDNode *V = dyn_cast<ConstantSDNode>(Src);
24  ConstantSDNode *SizeValue = dyn_cast<ConstantSDNode>(Size);
25  const AArch64Subtarget &STI =
26      DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
27  const char *bzeroName = (V && V->isNullValue())
28      ? DAG.getTargetLoweringInfo().getLibcallName(RTLIB::BZERO) : nullptr;
29  // For small size (< 256), it is not beneficial to use bzero
30  // instead of memset.
31  if (bzeroName && (!SizeValue || SizeValue->getZExtValue() > 256)) {
32    const AArch64TargetLowering &TLI = *STI.getTargetLowering();
33
34    EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout());
35    Type *IntPtrTy = Type::getInt8PtrTy(*DAG.getContext());
36    TargetLowering::ArgListTy Args;
37    TargetLowering::ArgListEntry Entry;
38    Entry.Node = Dst;
39    Entry.Ty = IntPtrTy;
40    Args.push_back(Entry);
41    Entry.Node = Size;
42    Args.push_back(Entry);
43    TargetLowering::CallLoweringInfo CLI(DAG);
44    CLI.setDebugLoc(dl)
45        .setChain(Chain)
46        .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
47                      DAG.getExternalSymbol(bzeroName, IntPtr),
48                      std::move(Args))
49        .setDiscardResult();
50    std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
51    return CallResult.second;
52  }
53  return SDValue();
54}
55bool AArch64SelectionDAGInfo::generateFMAsInMachineCombiner(
56    CodeGenOpt::Level OptLevel) const {
57  return OptLevel >= CodeGenOpt::Aggressive;
58}
59
60static const int kSetTagLoopThreshold = 176;
61
62static SDValue EmitUnrolledSetTag(SelectionDAG &DAG, const SDLoc &dl,
63                                  SDValue Chain, SDValue Ptr, uint64_t ObjSize,
64                                  const MachineMemOperand *BaseMemOperand,
65                                  bool ZeroData) {
66  MachineFunction &MF = DAG.getMachineFunction();
67  unsigned ObjSizeScaled = ObjSize / 16;
68
69  SDValue TagSrc = Ptr;
70  if (Ptr.getOpcode() == ISD::FrameIndex) {
71    int FI = cast<FrameIndexSDNode>(Ptr)->getIndex();
72    Ptr = DAG.getTargetFrameIndex(FI, MVT::i64);
73    // A frame index operand may end up as [SP + offset] => it is fine to use SP
74    // register as the tag source.
75    TagSrc = DAG.getRegister(AArch64::SP, MVT::i64);
76  }
77
78  const unsigned OpCode1 = ZeroData ? AArch64ISD::STZG : AArch64ISD::STG;
79  const unsigned OpCode2 = ZeroData ? AArch64ISD::STZ2G : AArch64ISD::ST2G;
80
81  SmallVector<SDValue, 8> OutChains;
82  unsigned OffsetScaled = 0;
83  while (OffsetScaled < ObjSizeScaled) {
84    if (ObjSizeScaled - OffsetScaled >= 2) {
85      SDValue AddrNode = DAG.getMemBasePlusOffset(Ptr, OffsetScaled * 16, dl);
86      SDValue St = DAG.getMemIntrinsicNode(
87          OpCode2, dl, DAG.getVTList(MVT::Other),
88          {Chain, TagSrc, AddrNode},
89          MVT::v4i64,
90          MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16 * 2));
91      OffsetScaled += 2;
92      OutChains.push_back(St);
93      continue;
94    }
95
96    if (ObjSizeScaled - OffsetScaled > 0) {
97      SDValue AddrNode = DAG.getMemBasePlusOffset(Ptr, OffsetScaled * 16, dl);
98      SDValue St = DAG.getMemIntrinsicNode(
99          OpCode1, dl, DAG.getVTList(MVT::Other),
100          {Chain, TagSrc, AddrNode},
101          MVT::v2i64,
102          MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16));
103      OffsetScaled += 1;
104      OutChains.push_back(St);
105    }
106  }
107
108  SDValue Res = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
109  return Res;
110}
111
112SDValue AArch64SelectionDAGInfo::EmitTargetCodeForSetTag(
113    SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr,
114    SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const {
115  uint64_t ObjSize = cast<ConstantSDNode>(Size)->getZExtValue();
116  assert(ObjSize % 16 == 0);
117
118  MachineFunction &MF = DAG.getMachineFunction();
119  MachineMemOperand *BaseMemOperand = MF.getMachineMemOperand(
120      DstPtrInfo, MachineMemOperand::MOStore, ObjSize, 16);
121
122  bool UseSetTagRangeLoop =
123      kSetTagLoopThreshold >= 0 && (int)ObjSize >= kSetTagLoopThreshold;
124  if (!UseSetTagRangeLoop)
125    return EmitUnrolledSetTag(DAG, dl, Chain, Addr, ObjSize, BaseMemOperand,
126                              ZeroData);
127
128  if (ObjSize % 32 != 0) {
129    SDNode *St1 = DAG.getMachineNode(
130        ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex, dl,
131        {MVT::i64, MVT::Other},
132        {Addr, Addr, DAG.getTargetConstant(1, dl, MVT::i64), Chain});
133    DAG.setNodeMemRefs(cast<MachineSDNode>(St1), {BaseMemOperand});
134    ObjSize -= 16;
135    Addr = SDValue(St1, 0);
136    Chain = SDValue(St1, 1);
137  }
138
139  const EVT ResTys[] = {MVT::i64, MVT::i64, MVT::Other};
140  SDValue Ops[] = {DAG.getConstant(ObjSize, dl, MVT::i64), Addr, Chain};
141  SDNode *St = DAG.getMachineNode(
142      ZeroData ? AArch64::STZGloop : AArch64::STGloop, dl, ResTys, Ops);
143
144  DAG.setNodeMemRefs(cast<MachineSDNode>(St), {BaseMemOperand});
145  return SDValue(St, 2);
146}
147