1//===-- AArch64SelectionDAGInfo.cpp - AArch64 SelectionDAG Info -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the AArch64SelectionDAGInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AArch64TargetMachine.h"
14using namespace llvm;
15
16#define DEBUG_TYPE "aarch64-selectiondag-info"
17
18SDValue AArch64SelectionDAGInfo::EmitMOPS(AArch64ISD::NodeType SDOpcode,
19                                          SelectionDAG &DAG, const SDLoc &DL,
20                                          SDValue Chain, SDValue Dst,
21                                          SDValue SrcOrValue, SDValue Size,
22                                          Align Alignment, bool isVolatile,
23                                          MachinePointerInfo DstPtrInfo,
24                                          MachinePointerInfo SrcPtrInfo) const {
25
26  // Get the constant size of the copy/set.
27  uint64_t ConstSize = 0;
28  if (auto *C = dyn_cast<ConstantSDNode>(Size))
29    ConstSize = C->getZExtValue();
30
31  const bool IsSet = SDOpcode == AArch64ISD::MOPS_MEMSET ||
32                     SDOpcode == AArch64ISD::MOPS_MEMSET_TAGGING;
33
34  const auto MachineOpcode = [&]() {
35    switch (SDOpcode) {
36    case AArch64ISD::MOPS_MEMSET:
37      return AArch64::MOPSMemorySetPseudo;
38    case AArch64ISD::MOPS_MEMSET_TAGGING:
39      return AArch64::MOPSMemorySetTaggingPseudo;
40    case AArch64ISD::MOPS_MEMCOPY:
41      return AArch64::MOPSMemoryCopyPseudo;
42    case AArch64ISD::MOPS_MEMMOVE:
43      return AArch64::MOPSMemoryMovePseudo;
44    default:
45      llvm_unreachable("Unhandled MOPS ISD Opcode");
46    }
47  }();
48
49  MachineFunction &MF = DAG.getMachineFunction();
50
51  auto Vol =
52      isVolatile ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
53  auto DstFlags = MachineMemOperand::MOStore | Vol;
54  auto *DstOp =
55      MF.getMachineMemOperand(DstPtrInfo, DstFlags, ConstSize, Alignment);
56
57  if (IsSet) {
58    // Extend value to i64, if required.
59    if (SrcOrValue.getValueType() != MVT::i64)
60      SrcOrValue = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, SrcOrValue);
61    SDValue Ops[] = {Dst, Size, SrcOrValue, Chain};
62    const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::Other};
63    MachineSDNode *Node = DAG.getMachineNode(MachineOpcode, DL, ResultTys, Ops);
64    DAG.setNodeMemRefs(Node, {DstOp});
65    return SDValue(Node, 2);
66  } else {
67    SDValue Ops[] = {Dst, SrcOrValue, Size, Chain};
68    const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::i64, MVT::Other};
69    MachineSDNode *Node = DAG.getMachineNode(MachineOpcode, DL, ResultTys, Ops);
70
71    auto SrcFlags = MachineMemOperand::MOLoad | Vol;
72    auto *SrcOp =
73        MF.getMachineMemOperand(SrcPtrInfo, SrcFlags, ConstSize, Alignment);
74    DAG.setNodeMemRefs(Node, {DstOp, SrcOp});
75    return SDValue(Node, 3);
76  }
77}
78
79SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemcpy(
80    SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,
81    SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,
82    MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
83  const AArch64Subtarget &STI =
84      DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
85  if (STI.hasMOPS())
86    return EmitMOPS(AArch64ISD::MOPS_MEMCOPY, DAG, DL, Chain, Dst, Src, Size,
87                    Alignment, isVolatile, DstPtrInfo, SrcPtrInfo);
88  return SDValue();
89}
90
91SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemset(
92    SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
93    SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,
94    MachinePointerInfo DstPtrInfo) const {
95  const AArch64Subtarget &STI =
96      DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
97
98  if (STI.hasMOPS()) {
99    return EmitMOPS(AArch64ISD::MOPS_MEMSET, DAG, dl, Chain, Dst, Src, Size,
100                    Alignment, isVolatile, DstPtrInfo, MachinePointerInfo{});
101  }
102  return SDValue();
103}
104
105SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemmove(
106    SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
107    SDValue Size, Align Alignment, bool isVolatile,
108    MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
109  const AArch64Subtarget &STI =
110      DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
111  if (STI.hasMOPS()) {
112    return EmitMOPS(AArch64ISD::MOPS_MEMMOVE, DAG, dl, Chain, Dst, Src, Size,
113                    Alignment, isVolatile, DstPtrInfo, SrcPtrInfo);
114  }
115  return SDValue();
116}
117
118static const int kSetTagLoopThreshold = 176;
119
120static SDValue EmitUnrolledSetTag(SelectionDAG &DAG, const SDLoc &dl,
121                                  SDValue Chain, SDValue Ptr, uint64_t ObjSize,
122                                  const MachineMemOperand *BaseMemOperand,
123                                  bool ZeroData) {
124  MachineFunction &MF = DAG.getMachineFunction();
125  unsigned ObjSizeScaled = ObjSize / 16;
126
127  SDValue TagSrc = Ptr;
128  if (Ptr.getOpcode() == ISD::FrameIndex) {
129    int FI = cast<FrameIndexSDNode>(Ptr)->getIndex();
130    Ptr = DAG.getTargetFrameIndex(FI, MVT::i64);
131    // A frame index operand may end up as [SP + offset] => it is fine to use SP
132    // register as the tag source.
133    TagSrc = DAG.getRegister(AArch64::SP, MVT::i64);
134  }
135
136  const unsigned OpCode1 = ZeroData ? AArch64ISD::STZG : AArch64ISD::STG;
137  const unsigned OpCode2 = ZeroData ? AArch64ISD::STZ2G : AArch64ISD::ST2G;
138
139  SmallVector<SDValue, 8> OutChains;
140  unsigned OffsetScaled = 0;
141  while (OffsetScaled < ObjSizeScaled) {
142    if (ObjSizeScaled - OffsetScaled >= 2) {
143      SDValue AddrNode = DAG.getMemBasePlusOffset(
144          Ptr, TypeSize::getFixed(OffsetScaled * 16), dl);
145      SDValue St = DAG.getMemIntrinsicNode(
146          OpCode2, dl, DAG.getVTList(MVT::Other),
147          {Chain, TagSrc, AddrNode},
148          MVT::v4i64,
149          MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16 * 2));
150      OffsetScaled += 2;
151      OutChains.push_back(St);
152      continue;
153    }
154
155    if (ObjSizeScaled - OffsetScaled > 0) {
156      SDValue AddrNode = DAG.getMemBasePlusOffset(
157          Ptr, TypeSize::getFixed(OffsetScaled * 16), dl);
158      SDValue St = DAG.getMemIntrinsicNode(
159          OpCode1, dl, DAG.getVTList(MVT::Other),
160          {Chain, TagSrc, AddrNode},
161          MVT::v2i64,
162          MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16));
163      OffsetScaled += 1;
164      OutChains.push_back(St);
165    }
166  }
167
168  SDValue Res = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
169  return Res;
170}
171
172SDValue AArch64SelectionDAGInfo::EmitTargetCodeForSetTag(
173    SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr,
174    SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const {
175  uint64_t ObjSize = Size->getAsZExtVal();
176  assert(ObjSize % 16 == 0);
177
178  MachineFunction &MF = DAG.getMachineFunction();
179  MachineMemOperand *BaseMemOperand = MF.getMachineMemOperand(
180      DstPtrInfo, MachineMemOperand::MOStore, ObjSize, Align(16));
181
182  bool UseSetTagRangeLoop =
183      kSetTagLoopThreshold >= 0 && (int)ObjSize >= kSetTagLoopThreshold;
184  if (!UseSetTagRangeLoop)
185    return EmitUnrolledSetTag(DAG, dl, Chain, Addr, ObjSize, BaseMemOperand,
186                              ZeroData);
187
188  const EVT ResTys[] = {MVT::i64, MVT::i64, MVT::Other};
189
190  unsigned Opcode;
191  if (Addr.getOpcode() == ISD::FrameIndex) {
192    int FI = cast<FrameIndexSDNode>(Addr)->getIndex();
193    Addr = DAG.getTargetFrameIndex(FI, MVT::i64);
194    Opcode = ZeroData ? AArch64::STZGloop : AArch64::STGloop;
195  } else {
196    Opcode = ZeroData ? AArch64::STZGloop_wback : AArch64::STGloop_wback;
197  }
198  SDValue Ops[] = {DAG.getTargetConstant(ObjSize, dl, MVT::i64), Addr, Chain};
199  SDNode *St = DAG.getMachineNode(Opcode, dl, ResTys, Ops);
200
201  DAG.setNodeMemRefs(cast<MachineSDNode>(St), {BaseMemOperand});
202  return SDValue(St, 2);
203}
204