1//===-- HexagonISelLowering.cpp - Hexagon DAG Lowering Implementation -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the interfaces that Hexagon uses to lower LLVM code
10// into a selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "HexagonISelLowering.h"
15#include "Hexagon.h"
16#include "HexagonMachineFunctionInfo.h"
17#include "HexagonRegisterInfo.h"
18#include "HexagonSubtarget.h"
19#include "HexagonTargetMachine.h"
20#include "HexagonTargetObjectFile.h"
21#include "llvm/ADT/APInt.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/StringSwitch.h"
25#include "llvm/CodeGen/CallingConvLower.h"
26#include "llvm/CodeGen/MachineFrameInfo.h"
27#include "llvm/CodeGen/MachineFunction.h"
28#include "llvm/CodeGen/MachineMemOperand.h"
29#include "llvm/CodeGen/MachineRegisterInfo.h"
30#include "llvm/CodeGen/RuntimeLibcalls.h"
31#include "llvm/CodeGen/SelectionDAG.h"
32#include "llvm/CodeGen/TargetCallingConv.h"
33#include "llvm/CodeGen/ValueTypes.h"
34#include "llvm/IR/BasicBlock.h"
35#include "llvm/IR/CallingConv.h"
36#include "llvm/IR/DataLayout.h"
37#include "llvm/IR/DerivedTypes.h"
38#include "llvm/IR/DiagnosticInfo.h"
39#include "llvm/IR/DiagnosticPrinter.h"
40#include "llvm/IR/Function.h"
41#include "llvm/IR/GlobalValue.h"
42#include "llvm/IR/InlineAsm.h"
43#include "llvm/IR/Instructions.h"
44#include "llvm/IR/IntrinsicInst.h"
45#include "llvm/IR/Intrinsics.h"
46#include "llvm/IR/IntrinsicsHexagon.h"
47#include "llvm/IR/IRBuilder.h"
48#include "llvm/IR/Module.h"
49#include "llvm/IR/Type.h"
50#include "llvm/IR/Value.h"
51#include "llvm/MC/MCRegisterInfo.h"
52#include "llvm/Support/Casting.h"
53#include "llvm/Support/CodeGen.h"
54#include "llvm/Support/CommandLine.h"
55#include "llvm/Support/Debug.h"
56#include "llvm/Support/ErrorHandling.h"
57#include "llvm/Support/MathExtras.h"
58#include "llvm/Support/raw_ostream.h"
59#include "llvm/Target/TargetMachine.h"
60#include <algorithm>
61#include <cassert>
62#include <cstddef>
63#include <cstdint>
64#include <limits>
65#include <utility>
66
67using namespace llvm;
68
69#define DEBUG_TYPE "hexagon-lowering"
70
71static cl::opt<bool> EmitJumpTables("hexagon-emit-jump-tables",
72  cl::init(true), cl::Hidden,
73  cl::desc("Control jump table emission on Hexagon target"));
74
75static cl::opt<bool>
76    EnableHexSDNodeSched("enable-hexagon-sdnode-sched", cl::Hidden,
77                         cl::desc("Enable Hexagon SDNode scheduling"));
78
79static cl::opt<bool> EnableFastMath("ffast-math", cl::Hidden,
80                                    cl::desc("Enable Fast Math processing"));
81
82static cl::opt<int> MinimumJumpTables("minimum-jump-tables", cl::Hidden,
83                                      cl::init(5),
84                                      cl::desc("Set minimum jump tables"));
85
86static cl::opt<int>
87    MaxStoresPerMemcpyCL("max-store-memcpy", cl::Hidden, cl::init(6),
88                         cl::desc("Max #stores to inline memcpy"));
89
90static cl::opt<int>
91    MaxStoresPerMemcpyOptSizeCL("max-store-memcpy-Os", cl::Hidden, cl::init(4),
92                                cl::desc("Max #stores to inline memcpy"));
93
94static cl::opt<int>
95    MaxStoresPerMemmoveCL("max-store-memmove", cl::Hidden, cl::init(6),
96                          cl::desc("Max #stores to inline memmove"));
97
98static cl::opt<int>
99    MaxStoresPerMemmoveOptSizeCL("max-store-memmove-Os", cl::Hidden,
100                                 cl::init(4),
101                                 cl::desc("Max #stores to inline memmove"));
102
103static cl::opt<int>
104    MaxStoresPerMemsetCL("max-store-memset", cl::Hidden, cl::init(8),
105                         cl::desc("Max #stores to inline memset"));
106
107static cl::opt<int>
108    MaxStoresPerMemsetOptSizeCL("max-store-memset-Os", cl::Hidden, cl::init(4),
109                                cl::desc("Max #stores to inline memset"));
110
111static cl::opt<bool> AlignLoads("hexagon-align-loads",
112  cl::Hidden, cl::init(false),
113  cl::desc("Rewrite unaligned loads as a pair of aligned loads"));
114
115static cl::opt<bool>
116    DisableArgsMinAlignment("hexagon-disable-args-min-alignment", cl::Hidden,
117                            cl::init(false),
118                            cl::desc("Disable minimum alignment of 1 for "
119                                     "arguments passed by value on stack"));
120
121namespace {
122
123  class HexagonCCState : public CCState {
124    unsigned NumNamedVarArgParams = 0;
125
126  public:
127    HexagonCCState(CallingConv::ID CC, bool IsVarArg, MachineFunction &MF,
128                   SmallVectorImpl<CCValAssign> &locs, LLVMContext &C,
129                   unsigned NumNamedArgs)
130        : CCState(CC, IsVarArg, MF, locs, C),
131          NumNamedVarArgParams(NumNamedArgs) {}
132    unsigned getNumNamedVarArgParams() const { return NumNamedVarArgParams; }
133  };
134
135} // end anonymous namespace
136
137
138// Implement calling convention for Hexagon.
139
140static bool CC_SkipOdd(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
141                       CCValAssign::LocInfo &LocInfo,
142                       ISD::ArgFlagsTy &ArgFlags, CCState &State) {
143  static const MCPhysReg ArgRegs[] = {
144    Hexagon::R0, Hexagon::R1, Hexagon::R2,
145    Hexagon::R3, Hexagon::R4, Hexagon::R5
146  };
147  const unsigned NumArgRegs = std::size(ArgRegs);
148  unsigned RegNum = State.getFirstUnallocated(ArgRegs);
149
150  // RegNum is an index into ArgRegs: skip a register if RegNum is odd.
151  if (RegNum != NumArgRegs && RegNum % 2 == 1)
152    State.AllocateReg(ArgRegs[RegNum]);
153
154  // Always return false here, as this function only makes sure that the first
155  // unallocated register has an even register number and does not actually
156  // allocate a register for the current argument.
157  return false;
158}
159
160#include "HexagonGenCallingConv.inc"
161
162
163SDValue
164HexagonTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG)
165      const {
166  return SDValue();
167}
168
169/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified
170/// by "Src" to address "Dst" of size "Size".  Alignment information is
171/// specified by the specific parameter attribute. The copy will be passed as
172/// a byval function parameter.  Sometimes what we are copying is the end of a
173/// larger object, the part that does not fit in registers.
174static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
175                                         SDValue Chain, ISD::ArgFlagsTy Flags,
176                                         SelectionDAG &DAG, const SDLoc &dl) {
177  SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
178  return DAG.getMemcpy(
179      Chain, dl, Dst, Src, SizeNode, Flags.getNonZeroByValAlign(),
180      /*isVolatile=*/false, /*AlwaysInline=*/false,
181      /*isTailCall=*/false, MachinePointerInfo(), MachinePointerInfo());
182}
183
184bool
185HexagonTargetLowering::CanLowerReturn(
186    CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
187    const SmallVectorImpl<ISD::OutputArg> &Outs,
188    LLVMContext &Context) const {
189  SmallVector<CCValAssign, 16> RVLocs;
190  CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
191
192  if (MF.getSubtarget<HexagonSubtarget>().useHVXOps())
193    return CCInfo.CheckReturn(Outs, RetCC_Hexagon_HVX);
194  return CCInfo.CheckReturn(Outs, RetCC_Hexagon);
195}
196
197// LowerReturn - Lower ISD::RET. If a struct is larger than 8 bytes and is
198// passed by value, the function prototype is modified to return void and
199// the value is stored in memory pointed by a pointer passed by caller.
200SDValue
201HexagonTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
202                                   bool IsVarArg,
203                                   const SmallVectorImpl<ISD::OutputArg> &Outs,
204                                   const SmallVectorImpl<SDValue> &OutVals,
205                                   const SDLoc &dl, SelectionDAG &DAG) const {
206  // CCValAssign - represent the assignment of the return value to locations.
207  SmallVector<CCValAssign, 16> RVLocs;
208
209  // CCState - Info about the registers and stack slot.
210  CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
211                 *DAG.getContext());
212
213  // Analyze return values of ISD::RET
214  if (Subtarget.useHVXOps())
215    CCInfo.AnalyzeReturn(Outs, RetCC_Hexagon_HVX);
216  else
217    CCInfo.AnalyzeReturn(Outs, RetCC_Hexagon);
218
219  SDValue Flag;
220  SmallVector<SDValue, 4> RetOps(1, Chain);
221
222  // Copy the result values into the output registers.
223  for (unsigned i = 0; i != RVLocs.size(); ++i) {
224    CCValAssign &VA = RVLocs[i];
225    SDValue Val = OutVals[i];
226
227    switch (VA.getLocInfo()) {
228      default:
229        // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt.
230        llvm_unreachable("Unknown loc info!");
231      case CCValAssign::Full:
232        break;
233      case CCValAssign::BCvt:
234        Val = DAG.getBitcast(VA.getLocVT(), Val);
235        break;
236      case CCValAssign::SExt:
237        Val = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Val);
238        break;
239      case CCValAssign::ZExt:
240        Val = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Val);
241        break;
242      case CCValAssign::AExt:
243        Val = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Val);
244        break;
245    }
246
247    Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Val, Flag);
248
249    // Guarantee that all emitted copies are stuck together with flags.
250    Flag = Chain.getValue(1);
251    RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
252  }
253
254  RetOps[0] = Chain;  // Update chain.
255
256  // Add the flag if we have it.
257  if (Flag.getNode())
258    RetOps.push_back(Flag);
259
260  return DAG.getNode(HexagonISD::RET_FLAG, dl, MVT::Other, RetOps);
261}
262
263bool HexagonTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
264  // If either no tail call or told not to tail call at all, don't.
265  return CI->isTailCall();
266}
267
268Register HexagonTargetLowering::getRegisterByName(
269      const char* RegName, LLT VT, const MachineFunction &) const {
270  // Just support r19, the linux kernel uses it.
271  Register Reg = StringSwitch<Register>(RegName)
272                     .Case("r0", Hexagon::R0)
273                     .Case("r1", Hexagon::R1)
274                     .Case("r2", Hexagon::R2)
275                     .Case("r3", Hexagon::R3)
276                     .Case("r4", Hexagon::R4)
277                     .Case("r5", Hexagon::R5)
278                     .Case("r6", Hexagon::R6)
279                     .Case("r7", Hexagon::R7)
280                     .Case("r8", Hexagon::R8)
281                     .Case("r9", Hexagon::R9)
282                     .Case("r10", Hexagon::R10)
283                     .Case("r11", Hexagon::R11)
284                     .Case("r12", Hexagon::R12)
285                     .Case("r13", Hexagon::R13)
286                     .Case("r14", Hexagon::R14)
287                     .Case("r15", Hexagon::R15)
288                     .Case("r16", Hexagon::R16)
289                     .Case("r17", Hexagon::R17)
290                     .Case("r18", Hexagon::R18)
291                     .Case("r19", Hexagon::R19)
292                     .Case("r20", Hexagon::R20)
293                     .Case("r21", Hexagon::R21)
294                     .Case("r22", Hexagon::R22)
295                     .Case("r23", Hexagon::R23)
296                     .Case("r24", Hexagon::R24)
297                     .Case("r25", Hexagon::R25)
298                     .Case("r26", Hexagon::R26)
299                     .Case("r27", Hexagon::R27)
300                     .Case("r28", Hexagon::R28)
301                     .Case("r29", Hexagon::R29)
302                     .Case("r30", Hexagon::R30)
303                     .Case("r31", Hexagon::R31)
304                     .Case("r1:0", Hexagon::D0)
305                     .Case("r3:2", Hexagon::D1)
306                     .Case("r5:4", Hexagon::D2)
307                     .Case("r7:6", Hexagon::D3)
308                     .Case("r9:8", Hexagon::D4)
309                     .Case("r11:10", Hexagon::D5)
310                     .Case("r13:12", Hexagon::D6)
311                     .Case("r15:14", Hexagon::D7)
312                     .Case("r17:16", Hexagon::D8)
313                     .Case("r19:18", Hexagon::D9)
314                     .Case("r21:20", Hexagon::D10)
315                     .Case("r23:22", Hexagon::D11)
316                     .Case("r25:24", Hexagon::D12)
317                     .Case("r27:26", Hexagon::D13)
318                     .Case("r29:28", Hexagon::D14)
319                     .Case("r31:30", Hexagon::D15)
320                     .Case("sp", Hexagon::R29)
321                     .Case("fp", Hexagon::R30)
322                     .Case("lr", Hexagon::R31)
323                     .Case("p0", Hexagon::P0)
324                     .Case("p1", Hexagon::P1)
325                     .Case("p2", Hexagon::P2)
326                     .Case("p3", Hexagon::P3)
327                     .Case("sa0", Hexagon::SA0)
328                     .Case("lc0", Hexagon::LC0)
329                     .Case("sa1", Hexagon::SA1)
330                     .Case("lc1", Hexagon::LC1)
331                     .Case("m0", Hexagon::M0)
332                     .Case("m1", Hexagon::M1)
333                     .Case("usr", Hexagon::USR)
334                     .Case("ugp", Hexagon::UGP)
335                     .Case("cs0", Hexagon::CS0)
336                     .Case("cs1", Hexagon::CS1)
337                     .Default(Register());
338  if (Reg)
339    return Reg;
340
341  report_fatal_error("Invalid register name global variable");
342}
343
344/// LowerCallResult - Lower the result values of an ISD::CALL into the
345/// appropriate copies out of appropriate physical registers.  This assumes that
346/// Chain/Glue are the input chain/glue to use, and that TheCall is the call
347/// being lowered. Returns a SDNode with the same number of values as the
348/// ISD::CALL.
349SDValue HexagonTargetLowering::LowerCallResult(
350    SDValue Chain, SDValue Glue, CallingConv::ID CallConv, bool IsVarArg,
351    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
352    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
353    const SmallVectorImpl<SDValue> &OutVals, SDValue Callee) const {
354  // Assign locations to each value returned by this call.
355  SmallVector<CCValAssign, 16> RVLocs;
356
357  CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
358                 *DAG.getContext());
359
360  if (Subtarget.useHVXOps())
361    CCInfo.AnalyzeCallResult(Ins, RetCC_Hexagon_HVX);
362  else
363    CCInfo.AnalyzeCallResult(Ins, RetCC_Hexagon);
364
365  // Copy all of the result registers out of their specified physreg.
366  for (unsigned i = 0; i != RVLocs.size(); ++i) {
367    SDValue RetVal;
368    if (RVLocs[i].getValVT() == MVT::i1) {
369      // Return values of type MVT::i1 require special handling. The reason
370      // is that MVT::i1 is associated with the PredRegs register class, but
371      // values of that type are still returned in R0. Generate an explicit
372      // copy into a predicate register from R0, and treat the value of the
373      // predicate register as the call result.
374      auto &MRI = DAG.getMachineFunction().getRegInfo();
375      SDValue FR0 = DAG.getCopyFromReg(Chain, dl, RVLocs[i].getLocReg(),
376                                       MVT::i32, Glue);
377      // FR0 = (Value, Chain, Glue)
378      Register PredR = MRI.createVirtualRegister(&Hexagon::PredRegsRegClass);
379      SDValue TPR = DAG.getCopyToReg(FR0.getValue(1), dl, PredR,
380                                     FR0.getValue(0), FR0.getValue(2));
381      // TPR = (Chain, Glue)
382      // Don't glue this CopyFromReg, because it copies from a virtual
383      // register. If it is glued to the call, InstrEmitter will add it
384      // as an implicit def to the call (EmitMachineNode).
385      RetVal = DAG.getCopyFromReg(TPR.getValue(0), dl, PredR, MVT::i1);
386      Glue = TPR.getValue(1);
387      Chain = TPR.getValue(0);
388    } else {
389      RetVal = DAG.getCopyFromReg(Chain, dl, RVLocs[i].getLocReg(),
390                                  RVLocs[i].getValVT(), Glue);
391      Glue = RetVal.getValue(2);
392      Chain = RetVal.getValue(1);
393    }
394    InVals.push_back(RetVal.getValue(0));
395  }
396
397  return Chain;
398}
399
400/// LowerCall - Functions arguments are copied from virtual regs to
401/// (physical regs)/(stack frame), CALLSEQ_START and CALLSEQ_END are emitted.
402SDValue
403HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
404                                 SmallVectorImpl<SDValue> &InVals) const {
405  SelectionDAG &DAG                     = CLI.DAG;
406  SDLoc &dl                             = CLI.DL;
407  SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
408  SmallVectorImpl<SDValue> &OutVals     = CLI.OutVals;
409  SmallVectorImpl<ISD::InputArg> &Ins   = CLI.Ins;
410  SDValue Chain                         = CLI.Chain;
411  SDValue Callee                        = CLI.Callee;
412  CallingConv::ID CallConv              = CLI.CallConv;
413  bool IsVarArg                         = CLI.IsVarArg;
414  bool DoesNotReturn                    = CLI.DoesNotReturn;
415
416  bool IsStructRet    = Outs.empty() ? false : Outs[0].Flags.isSRet();
417  MachineFunction &MF = DAG.getMachineFunction();
418  MachineFrameInfo &MFI = MF.getFrameInfo();
419  auto PtrVT = getPointerTy(MF.getDataLayout());
420
421  unsigned NumParams = CLI.CB ? CLI.CB->getFunctionType()->getNumParams() : 0;
422  if (GlobalAddressSDNode *GAN = dyn_cast<GlobalAddressSDNode>(Callee))
423    Callee = DAG.getTargetGlobalAddress(GAN->getGlobal(), dl, MVT::i32);
424
425  // Linux ABI treats var-arg calls the same way as regular ones.
426  bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg;
427
428  // Analyze operands of the call, assigning locations to each operand.
429  SmallVector<CCValAssign, 16> ArgLocs;
430  HexagonCCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, *DAG.getContext(),
431                        NumParams);
432
433  if (Subtarget.useHVXOps())
434    CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_HVX);
435  else if (DisableArgsMinAlignment)
436    CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_Legacy);
437  else
438    CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon);
439
440  if (CLI.IsTailCall) {
441    bool StructAttrFlag = MF.getFunction().hasStructRetAttr();
442    CLI.IsTailCall = IsEligibleForTailCallOptimization(Callee, CallConv,
443                        IsVarArg, IsStructRet, StructAttrFlag, Outs,
444                        OutVals, Ins, DAG);
445    for (const CCValAssign &VA : ArgLocs) {
446      if (VA.isMemLoc()) {
447        CLI.IsTailCall = false;
448        break;
449      }
450    }
451    LLVM_DEBUG(dbgs() << (CLI.IsTailCall ? "Eligible for Tail Call\n"
452                                         : "Argument must be passed on stack. "
453                                           "Not eligible for Tail Call\n"));
454  }
455  // Get a count of how many bytes are to be pushed on the stack.
456  unsigned NumBytes = CCInfo.getNextStackOffset();
457  SmallVector<std::pair<unsigned, SDValue>, 16> RegsToPass;
458  SmallVector<SDValue, 8> MemOpChains;
459
460  const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
461  SDValue StackPtr =
462      DAG.getCopyFromReg(Chain, dl, HRI.getStackRegister(), PtrVT);
463
464  bool NeedsArgAlign = false;
465  Align LargestAlignSeen;
466  // Walk the register/memloc assignments, inserting copies/loads.
467  for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
468    CCValAssign &VA = ArgLocs[i];
469    SDValue Arg = OutVals[i];
470    ISD::ArgFlagsTy Flags = Outs[i].Flags;
471    // Record if we need > 8 byte alignment on an argument.
472    bool ArgAlign = Subtarget.isHVXVectorType(VA.getValVT());
473    NeedsArgAlign |= ArgAlign;
474
475    // Promote the value if needed.
476    switch (VA.getLocInfo()) {
477      default:
478        // Loc info must be one of Full, BCvt, SExt, ZExt, or AExt.
479        llvm_unreachable("Unknown loc info!");
480      case CCValAssign::Full:
481        break;
482      case CCValAssign::BCvt:
483        Arg = DAG.getBitcast(VA.getLocVT(), Arg);
484        break;
485      case CCValAssign::SExt:
486        Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg);
487        break;
488      case CCValAssign::ZExt:
489        Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg);
490        break;
491      case CCValAssign::AExt:
492        Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg);
493        break;
494    }
495
496    if (VA.isMemLoc()) {
497      unsigned LocMemOffset = VA.getLocMemOffset();
498      SDValue MemAddr = DAG.getConstant(LocMemOffset, dl,
499                                        StackPtr.getValueType());
500      MemAddr = DAG.getNode(ISD::ADD, dl, MVT::i32, StackPtr, MemAddr);
501      if (ArgAlign)
502        LargestAlignSeen = std::max(
503            LargestAlignSeen, Align(VA.getLocVT().getStoreSizeInBits() / 8));
504      if (Flags.isByVal()) {
505        // The argument is a struct passed by value. According to LLVM, "Arg"
506        // is a pointer.
507        MemOpChains.push_back(CreateCopyOfByValArgument(Arg, MemAddr, Chain,
508                                                        Flags, DAG, dl));
509      } else {
510        MachinePointerInfo LocPI = MachinePointerInfo::getStack(
511            DAG.getMachineFunction(), LocMemOffset);
512        SDValue S = DAG.getStore(Chain, dl, Arg, MemAddr, LocPI);
513        MemOpChains.push_back(S);
514      }
515      continue;
516    }
517
518    // Arguments that can be passed on register must be kept at RegsToPass
519    // vector.
520    if (VA.isRegLoc())
521      RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
522  }
523
524  if (NeedsArgAlign && Subtarget.hasV60Ops()) {
525    LLVM_DEBUG(dbgs() << "Function needs byte stack align due to call args\n");
526    Align VecAlign = HRI.getSpillAlign(Hexagon::HvxVRRegClass);
527    LargestAlignSeen = std::max(LargestAlignSeen, VecAlign);
528    MFI.ensureMaxAlignment(LargestAlignSeen);
529  }
530  // Transform all store nodes into one single node because all store
531  // nodes are independent of each other.
532  if (!MemOpChains.empty())
533    Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
534
535  SDValue Glue;
536  if (!CLI.IsTailCall) {
537    Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl);
538    Glue = Chain.getValue(1);
539  }
540
541  // Build a sequence of copy-to-reg nodes chained together with token
542  // chain and flag operands which copy the outgoing args into registers.
543  // The Glue is necessary since all emitted instructions must be
544  // stuck together.
545  if (!CLI.IsTailCall) {
546    for (const auto &R : RegsToPass) {
547      Chain = DAG.getCopyToReg(Chain, dl, R.first, R.second, Glue);
548      Glue = Chain.getValue(1);
549    }
550  } else {
551    // For tail calls lower the arguments to the 'real' stack slot.
552    //
553    // Force all the incoming stack arguments to be loaded from the stack
554    // before any new outgoing arguments are stored to the stack, because the
555    // outgoing stack slots may alias the incoming argument stack slots, and
556    // the alias isn't otherwise explicit. This is slightly more conservative
557    // than necessary, because it means that each store effectively depends
558    // on every argument instead of just those arguments it would clobber.
559    //
560    // Do not flag preceding copytoreg stuff together with the following stuff.
561    Glue = SDValue();
562    for (const auto &R : RegsToPass) {
563      Chain = DAG.getCopyToReg(Chain, dl, R.first, R.second, Glue);
564      Glue = Chain.getValue(1);
565    }
566    Glue = SDValue();
567  }
568
569  bool LongCalls = MF.getSubtarget<HexagonSubtarget>().useLongCalls();
570  unsigned Flags = LongCalls ? HexagonII::HMOTF_ConstExtended : 0;
571
572  // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
573  // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
574  // node so that legalize doesn't hack it.
575  if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
576    Callee = DAG.getTargetGlobalAddress(G->getGlobal(), dl, PtrVT, 0, Flags);
577  } else if (ExternalSymbolSDNode *S =
578             dyn_cast<ExternalSymbolSDNode>(Callee)) {
579    Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, Flags);
580  }
581
582  // Returns a chain & a flag for retval copy to use.
583  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
584  SmallVector<SDValue, 8> Ops;
585  Ops.push_back(Chain);
586  Ops.push_back(Callee);
587
588  // Add argument registers to the end of the list so that they are
589  // known live into the call.
590  for (const auto &R : RegsToPass)
591    Ops.push_back(DAG.getRegister(R.first, R.second.getValueType()));
592
593  const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallConv);
594  assert(Mask && "Missing call preserved mask for calling convention");
595  Ops.push_back(DAG.getRegisterMask(Mask));
596
597  if (Glue.getNode())
598    Ops.push_back(Glue);
599
600  if (CLI.IsTailCall) {
601    MFI.setHasTailCall();
602    return DAG.getNode(HexagonISD::TC_RETURN, dl, NodeTys, Ops);
603  }
604
605  // Set this here because we need to know this for "hasFP" in frame lowering.
606  // The target-independent code calls getFrameRegister before setting it, and
607  // getFrameRegister uses hasFP to determine whether the function has FP.
608  MFI.setHasCalls(true);
609
610  unsigned OpCode = DoesNotReturn ? HexagonISD::CALLnr : HexagonISD::CALL;
611  Chain = DAG.getNode(OpCode, dl, NodeTys, Ops);
612  Glue = Chain.getValue(1);
613
614  // Create the CALLSEQ_END node.
615  Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, Glue, dl);
616  Glue = Chain.getValue(1);
617
618  // Handle result values, copying them out of physregs into vregs that we
619  // return.
620  return LowerCallResult(Chain, Glue, CallConv, IsVarArg, Ins, dl, DAG,
621                         InVals, OutVals, Callee);
622}
623
624/// Returns true by value, base pointer and offset pointer and addressing
625/// mode by reference if this node can be combined with a load / store to
626/// form a post-indexed load / store.
627bool HexagonTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
628      SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM,
629      SelectionDAG &DAG) const {
630  LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(N);
631  if (!LSN)
632    return false;
633  EVT VT = LSN->getMemoryVT();
634  if (!VT.isSimple())
635    return false;
636  bool IsLegalType = VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 ||
637                     VT == MVT::i64 || VT == MVT::f32 || VT == MVT::f64 ||
638                     VT == MVT::v2i16 || VT == MVT::v2i32 || VT == MVT::v4i8 ||
639                     VT == MVT::v4i16 || VT == MVT::v8i8 ||
640                     Subtarget.isHVXVectorType(VT.getSimpleVT());
641  if (!IsLegalType)
642    return false;
643
644  if (Op->getOpcode() != ISD::ADD)
645    return false;
646  Base = Op->getOperand(0);
647  Offset = Op->getOperand(1);
648  if (!isa<ConstantSDNode>(Offset.getNode()))
649    return false;
650  AM = ISD::POST_INC;
651
652  int32_t V = cast<ConstantSDNode>(Offset.getNode())->getSExtValue();
653  return Subtarget.getInstrInfo()->isValidAutoIncImm(VT, V);
654}
655
656SDValue
657HexagonTargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {
658  MachineFunction &MF = DAG.getMachineFunction();
659  auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>();
660  const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
661  unsigned LR = HRI.getRARegister();
662
663  if ((Op.getOpcode() != ISD::INLINEASM &&
664       Op.getOpcode() != ISD::INLINEASM_BR) || HMFI.hasClobberLR())
665    return Op;
666
667  unsigned NumOps = Op.getNumOperands();
668  if (Op.getOperand(NumOps-1).getValueType() == MVT::Glue)
669    --NumOps;  // Ignore the flag operand.
670
671  for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
672    unsigned Flags = cast<ConstantSDNode>(Op.getOperand(i))->getZExtValue();
673    unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags);
674    ++i;  // Skip the ID value.
675
676    switch (InlineAsm::getKind(Flags)) {
677      default:
678        llvm_unreachable("Bad flags!");
679      case InlineAsm::Kind_RegUse:
680      case InlineAsm::Kind_Imm:
681      case InlineAsm::Kind_Mem:
682        i += NumVals;
683        break;
684      case InlineAsm::Kind_Clobber:
685      case InlineAsm::Kind_RegDef:
686      case InlineAsm::Kind_RegDefEarlyClobber: {
687        for (; NumVals; --NumVals, ++i) {
688          Register Reg = cast<RegisterSDNode>(Op.getOperand(i))->getReg();
689          if (Reg != LR)
690            continue;
691          HMFI.setHasClobberLR(true);
692          return Op;
693        }
694        break;
695      }
696    }
697  }
698
699  return Op;
700}
701
702// Need to transform ISD::PREFETCH into something that doesn't inherit
703// all of the properties of ISD::PREFETCH, specifically SDNPMayLoad and
704// SDNPMayStore.
705SDValue HexagonTargetLowering::LowerPREFETCH(SDValue Op,
706                                             SelectionDAG &DAG) const {
707  SDValue Chain = Op.getOperand(0);
708  SDValue Addr = Op.getOperand(1);
709  // Lower it to DCFETCH($reg, #0).  A "pat" will try to merge the offset in,
710  // if the "reg" is fed by an "add".
711  SDLoc DL(Op);
712  SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
713  return DAG.getNode(HexagonISD::DCFETCH, DL, MVT::Other, Chain, Addr, Zero);
714}
715
716// Custom-handle ISD::READCYCLECOUNTER because the target-independent SDNode
717// is marked as having side-effects, while the register read on Hexagon does
718// not have any. TableGen refuses to accept the direct pattern from that node
719// to the A4_tfrcpp.
720SDValue HexagonTargetLowering::LowerREADCYCLECOUNTER(SDValue Op,
721                                                     SelectionDAG &DAG) const {
722  SDValue Chain = Op.getOperand(0);
723  SDLoc dl(Op);
724  SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other);
725  return DAG.getNode(HexagonISD::READCYCLE, dl, VTs, Chain);
726}
727
728SDValue HexagonTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
729      SelectionDAG &DAG) const {
730  SDValue Chain = Op.getOperand(0);
731  unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
732  // Lower the hexagon_prefetch builtin to DCFETCH, as above.
733  if (IntNo == Intrinsic::hexagon_prefetch) {
734    SDValue Addr = Op.getOperand(2);
735    SDLoc DL(Op);
736    SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
737    return DAG.getNode(HexagonISD::DCFETCH, DL, MVT::Other, Chain, Addr, Zero);
738  }
739  return SDValue();
740}
741
742SDValue
743HexagonTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
744                                               SelectionDAG &DAG) const {
745  SDValue Chain = Op.getOperand(0);
746  SDValue Size = Op.getOperand(1);
747  SDValue Align = Op.getOperand(2);
748  SDLoc dl(Op);
749
750  ConstantSDNode *AlignConst = dyn_cast<ConstantSDNode>(Align);
751  assert(AlignConst && "Non-constant Align in LowerDYNAMIC_STACKALLOC");
752
753  unsigned A = AlignConst->getSExtValue();
754  auto &HFI = *Subtarget.getFrameLowering();
755  // "Zero" means natural stack alignment.
756  if (A == 0)
757    A = HFI.getStackAlign().value();
758
759  LLVM_DEBUG({
760    dbgs () << __func__ << " Align: " << A << " Size: ";
761    Size.getNode()->dump(&DAG);
762    dbgs() << "\n";
763  });
764
765  SDValue AC = DAG.getConstant(A, dl, MVT::i32);
766  SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
767  SDValue AA = DAG.getNode(HexagonISD::ALLOCA, dl, VTs, Chain, Size, AC);
768
769  DAG.ReplaceAllUsesOfValueWith(Op, AA);
770  return AA;
771}
772
773SDValue HexagonTargetLowering::LowerFormalArguments(
774    SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
775    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
776    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
777  MachineFunction &MF = DAG.getMachineFunction();
778  MachineFrameInfo &MFI = MF.getFrameInfo();
779  MachineRegisterInfo &MRI = MF.getRegInfo();
780
781  // Linux ABI treats var-arg calls the same way as regular ones.
782  bool TreatAsVarArg = !Subtarget.isEnvironmentMusl() && IsVarArg;
783
784  // Assign locations to all of the incoming arguments.
785  SmallVector<CCValAssign, 16> ArgLocs;
786  HexagonCCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs,
787                        *DAG.getContext(),
788                        MF.getFunction().getFunctionType()->getNumParams());
789
790  if (Subtarget.useHVXOps())
791    CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon_HVX);
792  else if (DisableArgsMinAlignment)
793    CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon_Legacy);
794  else
795    CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon);
796
797  // For LLVM, in the case when returning a struct by value (>8byte),
798  // the first argument is a pointer that points to the location on caller's
799  // stack where the return value will be stored. For Hexagon, the location on
800  // caller's stack is passed only when the struct size is smaller than (and
801  // equal to) 8 bytes. If not, no address will be passed into callee and
802  // callee return the result direclty through R0/R1.
803  auto NextSingleReg = [] (const TargetRegisterClass &RC, unsigned Reg) {
804    switch (RC.getID()) {
805    case Hexagon::IntRegsRegClassID:
806      return Reg - Hexagon::R0 + 1;
807    case Hexagon::DoubleRegsRegClassID:
808      return (Reg - Hexagon::D0 + 1) * 2;
809    case Hexagon::HvxVRRegClassID:
810      return Reg - Hexagon::V0 + 1;
811    case Hexagon::HvxWRRegClassID:
812      return (Reg - Hexagon::W0 + 1) * 2;
813    }
814    llvm_unreachable("Unexpected register class");
815  };
816
817  auto &HFL = const_cast<HexagonFrameLowering&>(*Subtarget.getFrameLowering());
818  auto &HMFI = *MF.getInfo<HexagonMachineFunctionInfo>();
819  HFL.FirstVarArgSavedReg = 0;
820  HMFI.setFirstNamedArgFrameIndex(-int(MFI.getNumFixedObjects()));
821
822  for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
823    CCValAssign &VA = ArgLocs[i];
824    ISD::ArgFlagsTy Flags = Ins[i].Flags;
825    bool ByVal = Flags.isByVal();
826
827    // Arguments passed in registers:
828    // 1. 32- and 64-bit values and HVX vectors are passed directly,
829    // 2. Large structs are passed via an address, and the address is
830    //    passed in a register.
831    if (VA.isRegLoc() && ByVal && Flags.getByValSize() <= 8)
832      llvm_unreachable("ByValSize must be bigger than 8 bytes");
833
834    bool InReg = VA.isRegLoc() &&
835                 (!ByVal || (ByVal && Flags.getByValSize() > 8));
836
837    if (InReg) {
838      MVT RegVT = VA.getLocVT();
839      if (VA.getLocInfo() == CCValAssign::BCvt)
840        RegVT = VA.getValVT();
841
842      const TargetRegisterClass *RC = getRegClassFor(RegVT);
843      Register VReg = MRI.createVirtualRegister(RC);
844      SDValue Copy = DAG.getCopyFromReg(Chain, dl, VReg, RegVT);
845
846      // Treat values of type MVT::i1 specially: they are passed in
847      // registers of type i32, but they need to remain as values of
848      // type i1 for consistency of the argument lowering.
849      if (VA.getValVT() == MVT::i1) {
850        assert(RegVT.getSizeInBits() <= 32);
851        SDValue T = DAG.getNode(ISD::AND, dl, RegVT,
852                                Copy, DAG.getConstant(1, dl, RegVT));
853        Copy = DAG.getSetCC(dl, MVT::i1, T, DAG.getConstant(0, dl, RegVT),
854                            ISD::SETNE);
855      } else {
856#ifndef NDEBUG
857        unsigned RegSize = RegVT.getSizeInBits();
858        assert(RegSize == 32 || RegSize == 64 ||
859               Subtarget.isHVXVectorType(RegVT));
860#endif
861      }
862      InVals.push_back(Copy);
863      MRI.addLiveIn(VA.getLocReg(), VReg);
864      HFL.FirstVarArgSavedReg = NextSingleReg(*RC, VA.getLocReg());
865    } else {
866      assert(VA.isMemLoc() && "Argument should be passed in memory");
867
868      // If it's a byval parameter, then we need to compute the
869      // "real" size, not the size of the pointer.
870      unsigned ObjSize = Flags.isByVal()
871                            ? Flags.getByValSize()
872                            : VA.getLocVT().getStoreSizeInBits() / 8;
873
874      // Create the frame index object for this incoming parameter.
875      int Offset = HEXAGON_LRFP_SIZE + VA.getLocMemOffset();
876      int FI = MFI.CreateFixedObject(ObjSize, Offset, true);
877      SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
878
879      if (Flags.isByVal()) {
880        // If it's a pass-by-value aggregate, then do not dereference the stack
881        // location. Instead, we should generate a reference to the stack
882        // location.
883        InVals.push_back(FIN);
884      } else {
885        SDValue L = DAG.getLoad(VA.getValVT(), dl, Chain, FIN,
886                                MachinePointerInfo::getFixedStack(MF, FI, 0));
887        InVals.push_back(L);
888      }
889    }
890  }
891
892  if (IsVarArg && Subtarget.isEnvironmentMusl()) {
893    for (int i = HFL.FirstVarArgSavedReg; i < 6; i++)
894      MRI.addLiveIn(Hexagon::R0+i);
895  }
896
897  if (IsVarArg && Subtarget.isEnvironmentMusl()) {
898    HMFI.setFirstNamedArgFrameIndex(HMFI.getFirstNamedArgFrameIndex() - 1);
899    HMFI.setLastNamedArgFrameIndex(-int(MFI.getNumFixedObjects()));
900
901    // Create Frame index for the start of register saved area.
902    int NumVarArgRegs = 6 - HFL.FirstVarArgSavedReg;
903    bool RequiresPadding = (NumVarArgRegs & 1);
904    int RegSaveAreaSizePlusPadding = RequiresPadding
905                                        ? (NumVarArgRegs + 1) * 4
906                                        : NumVarArgRegs * 4;
907
908    if (RegSaveAreaSizePlusPadding > 0) {
909      // The offset to saved register area should be 8 byte aligned.
910      int RegAreaStart = HEXAGON_LRFP_SIZE + CCInfo.getNextStackOffset();
911      if (!(RegAreaStart % 8))
912        RegAreaStart = (RegAreaStart + 7) & -8;
913
914      int RegSaveAreaFrameIndex =
915        MFI.CreateFixedObject(RegSaveAreaSizePlusPadding, RegAreaStart, true);
916      HMFI.setRegSavedAreaStartFrameIndex(RegSaveAreaFrameIndex);
917
918      // This will point to the next argument passed via stack.
919      int Offset = RegAreaStart + RegSaveAreaSizePlusPadding;
920      int FI = MFI.CreateFixedObject(Hexagon_PointerSize, Offset, true);
921      HMFI.setVarArgsFrameIndex(FI);
922    } else {
923      // This will point to the next argument passed via stack, when
924      // there is no saved register area.
925      int Offset = HEXAGON_LRFP_SIZE + CCInfo.getNextStackOffset();
926      int FI = MFI.CreateFixedObject(Hexagon_PointerSize, Offset, true);
927      HMFI.setRegSavedAreaStartFrameIndex(FI);
928      HMFI.setVarArgsFrameIndex(FI);
929    }
930  }
931
932
933  if (IsVarArg && !Subtarget.isEnvironmentMusl()) {
934    // This will point to the next argument passed via stack.
935    int Offset = HEXAGON_LRFP_SIZE + CCInfo.getNextStackOffset();
936    int FI = MFI.CreateFixedObject(Hexagon_PointerSize, Offset, true);
937    HMFI.setVarArgsFrameIndex(FI);
938  }
939
940  return Chain;
941}
942
943SDValue
944HexagonTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
945  // VASTART stores the address of the VarArgsFrameIndex slot into the
946  // memory location argument.
947  MachineFunction &MF = DAG.getMachineFunction();
948  HexagonMachineFunctionInfo *QFI = MF.getInfo<HexagonMachineFunctionInfo>();
949  SDValue Addr = DAG.getFrameIndex(QFI->getVarArgsFrameIndex(), MVT::i32);
950  const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
951
952  if (!Subtarget.isEnvironmentMusl()) {
953    return DAG.getStore(Op.getOperand(0), SDLoc(Op), Addr, Op.getOperand(1),
954                        MachinePointerInfo(SV));
955  }
956  auto &FuncInfo = *MF.getInfo<HexagonMachineFunctionInfo>();
957  auto &HFL = *Subtarget.getFrameLowering();
958  SDLoc DL(Op);
959  SmallVector<SDValue, 8> MemOps;
960
961  // Get frame index of va_list.
962  SDValue FIN = Op.getOperand(1);
963
964  // If first Vararg register is odd, add 4 bytes to start of
965  // saved register area to point to the first register location.
966  // This is because the saved register area has to be 8 byte aligned.
967  // Incase of an odd start register, there will be 4 bytes of padding in
968  // the beginning of saved register area. If all registers area used up,
969  // the following condition will handle it correctly.
970  SDValue SavedRegAreaStartFrameIndex =
971    DAG.getFrameIndex(FuncInfo.getRegSavedAreaStartFrameIndex(), MVT::i32);
972
973  auto PtrVT = getPointerTy(DAG.getDataLayout());
974
975  if (HFL.FirstVarArgSavedReg & 1)
976    SavedRegAreaStartFrameIndex =
977      DAG.getNode(ISD::ADD, DL, PtrVT,
978                  DAG.getFrameIndex(FuncInfo.getRegSavedAreaStartFrameIndex(),
979                                    MVT::i32),
980                  DAG.getIntPtrConstant(4, DL));
981
982  // Store the saved register area start pointer.
983  SDValue Store =
984    DAG.getStore(Op.getOperand(0), DL,
985                 SavedRegAreaStartFrameIndex,
986                 FIN, MachinePointerInfo(SV));
987  MemOps.push_back(Store);
988
989  // Store saved register area end pointer.
990  FIN = DAG.getNode(ISD::ADD, DL, PtrVT,
991                    FIN, DAG.getIntPtrConstant(4, DL));
992  Store = DAG.getStore(Op.getOperand(0), DL,
993                       DAG.getFrameIndex(FuncInfo.getVarArgsFrameIndex(),
994                                         PtrVT),
995                       FIN, MachinePointerInfo(SV, 4));
996  MemOps.push_back(Store);
997
998  // Store overflow area pointer.
999  FIN = DAG.getNode(ISD::ADD, DL, PtrVT,
1000                    FIN, DAG.getIntPtrConstant(4, DL));
1001  Store = DAG.getStore(Op.getOperand(0), DL,
1002                       DAG.getFrameIndex(FuncInfo.getVarArgsFrameIndex(),
1003                                         PtrVT),
1004                       FIN, MachinePointerInfo(SV, 8));
1005  MemOps.push_back(Store);
1006
1007  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
1008}
1009
1010SDValue
1011HexagonTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const {
1012  // Assert that the linux ABI is enabled for the current compilation.
1013  assert(Subtarget.isEnvironmentMusl() && "Linux ABI should be enabled");
1014  SDValue Chain = Op.getOperand(0);
1015  SDValue DestPtr = Op.getOperand(1);
1016  SDValue SrcPtr = Op.getOperand(2);
1017  const Value *DestSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
1018  const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
1019  SDLoc DL(Op);
1020  // Size of the va_list is 12 bytes as it has 3 pointers. Therefore,
1021  // we need to memcopy 12 bytes from va_list to another similar list.
1022  return DAG.getMemcpy(Chain, DL, DestPtr, SrcPtr,
1023                       DAG.getIntPtrConstant(12, DL), Align(4),
1024                       /*isVolatile*/ false, false, false,
1025                       MachinePointerInfo(DestSV), MachinePointerInfo(SrcSV));
1026}
1027
1028SDValue HexagonTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
1029  const SDLoc &dl(Op);
1030  SDValue LHS = Op.getOperand(0);
1031  SDValue RHS = Op.getOperand(1);
1032  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
1033  MVT ResTy = ty(Op);
1034  MVT OpTy = ty(LHS);
1035
1036  if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) {
1037    MVT ElemTy = OpTy.getVectorElementType();
1038    assert(ElemTy.isScalarInteger());
1039    MVT WideTy = MVT::getVectorVT(MVT::getIntegerVT(2*ElemTy.getSizeInBits()),
1040                                  OpTy.getVectorNumElements());
1041    return DAG.getSetCC(dl, ResTy,
1042                        DAG.getSExtOrTrunc(LHS, SDLoc(LHS), WideTy),
1043                        DAG.getSExtOrTrunc(RHS, SDLoc(RHS), WideTy), CC);
1044  }
1045
1046  // Treat all other vector types as legal.
1047  if (ResTy.isVector())
1048    return Op;
1049
1050  // Comparisons of short integers should use sign-extend, not zero-extend,
1051  // since we can represent small negative values in the compare instructions.
1052  // The LLVM default is to use zero-extend arbitrarily in these cases.
1053  auto isSExtFree = [this](SDValue N) {
1054    switch (N.getOpcode()) {
1055      case ISD::TRUNCATE: {
1056        // A sign-extend of a truncate of a sign-extend is free.
1057        SDValue Op = N.getOperand(0);
1058        if (Op.getOpcode() != ISD::AssertSext)
1059          return false;
1060        EVT OrigTy = cast<VTSDNode>(Op.getOperand(1))->getVT();
1061        unsigned ThisBW = ty(N).getSizeInBits();
1062        unsigned OrigBW = OrigTy.getSizeInBits();
1063        // The type that was sign-extended to get the AssertSext must be
1064        // narrower than the type of N (so that N has still the same value
1065        // as the original).
1066        return ThisBW >= OrigBW;
1067      }
1068      case ISD::LOAD:
1069        // We have sign-extended loads.
1070        return true;
1071    }
1072    return false;
1073  };
1074
1075  if (OpTy == MVT::i8 || OpTy == MVT::i16) {
1076    ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS);
1077    bool IsNegative = C && C->getAPIntValue().isNegative();
1078    if (IsNegative || isSExtFree(LHS) || isSExtFree(RHS))
1079      return DAG.getSetCC(dl, ResTy,
1080                          DAG.getSExtOrTrunc(LHS, SDLoc(LHS), MVT::i32),
1081                          DAG.getSExtOrTrunc(RHS, SDLoc(RHS), MVT::i32), CC);
1082  }
1083
1084  return SDValue();
1085}
1086
1087SDValue
1088HexagonTargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {
1089  SDValue PredOp = Op.getOperand(0);
1090  SDValue Op1 = Op.getOperand(1), Op2 = Op.getOperand(2);
1091  MVT OpTy = ty(Op1);
1092  const SDLoc &dl(Op);
1093
1094  if (OpTy == MVT::v2i16 || OpTy == MVT::v4i8) {
1095    MVT ElemTy = OpTy.getVectorElementType();
1096    assert(ElemTy.isScalarInteger());
1097    MVT WideTy = MVT::getVectorVT(MVT::getIntegerVT(2*ElemTy.getSizeInBits()),
1098                                  OpTy.getVectorNumElements());
1099    // Generate (trunc (select (_, sext, sext))).
1100    return DAG.getSExtOrTrunc(
1101              DAG.getSelect(dl, WideTy, PredOp,
1102                            DAG.getSExtOrTrunc(Op1, dl, WideTy),
1103                            DAG.getSExtOrTrunc(Op2, dl, WideTy)),
1104              dl, OpTy);
1105  }
1106
1107  return SDValue();
1108}
1109
1110SDValue
1111HexagonTargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
1112  EVT ValTy = Op.getValueType();
1113  ConstantPoolSDNode *CPN = cast<ConstantPoolSDNode>(Op);
1114  Constant *CVal = nullptr;
1115  bool isVTi1Type = false;
1116  if (auto *CV = dyn_cast<ConstantVector>(CPN->getConstVal())) {
1117    if (cast<VectorType>(CV->getType())->getElementType()->isIntegerTy(1)) {
1118      IRBuilder<> IRB(CV->getContext());
1119      SmallVector<Constant*, 128> NewConst;
1120      unsigned VecLen = CV->getNumOperands();
1121      assert(isPowerOf2_32(VecLen) &&
1122             "conversion only supported for pow2 VectorSize");
1123      for (unsigned i = 0; i < VecLen; ++i)
1124        NewConst.push_back(IRB.getInt8(CV->getOperand(i)->isZeroValue()));
1125
1126      CVal = ConstantVector::get(NewConst);
1127      isVTi1Type = true;
1128    }
1129  }
1130  Align Alignment = CPN->getAlign();
1131  bool IsPositionIndependent = isPositionIndependent();
1132  unsigned char TF = IsPositionIndependent ? HexagonII::MO_PCREL : 0;
1133
1134  unsigned Offset = 0;
1135  SDValue T;
1136  if (CPN->isMachineConstantPoolEntry())
1137    T = DAG.getTargetConstantPool(CPN->getMachineCPVal(), ValTy, Alignment,
1138                                  Offset, TF);
1139  else if (isVTi1Type)
1140    T = DAG.getTargetConstantPool(CVal, ValTy, Alignment, Offset, TF);
1141  else
1142    T = DAG.getTargetConstantPool(CPN->getConstVal(), ValTy, Alignment, Offset,
1143                                  TF);
1144
1145  assert(cast<ConstantPoolSDNode>(T)->getTargetFlags() == TF &&
1146         "Inconsistent target flag encountered");
1147
1148  if (IsPositionIndependent)
1149    return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), ValTy, T);
1150  return DAG.getNode(HexagonISD::CP, SDLoc(Op), ValTy, T);
1151}
1152
1153SDValue
1154HexagonTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
1155  EVT VT = Op.getValueType();
1156  int Idx = cast<JumpTableSDNode>(Op)->getIndex();
1157  if (isPositionIndependent()) {
1158    SDValue T = DAG.getTargetJumpTable(Idx, VT, HexagonII::MO_PCREL);
1159    return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), VT, T);
1160  }
1161
1162  SDValue T = DAG.getTargetJumpTable(Idx, VT);
1163  return DAG.getNode(HexagonISD::JT, SDLoc(Op), VT, T);
1164}
1165
1166SDValue
1167HexagonTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const {
1168  const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
1169  MachineFunction &MF = DAG.getMachineFunction();
1170  MachineFrameInfo &MFI = MF.getFrameInfo();
1171  MFI.setReturnAddressIsTaken(true);
1172
1173  if (verifyReturnAddressArgumentIsConstant(Op, DAG))
1174    return SDValue();
1175
1176  EVT VT = Op.getValueType();
1177  SDLoc dl(Op);
1178  unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1179  if (Depth) {
1180    SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
1181    SDValue Offset = DAG.getConstant(4, dl, MVT::i32);
1182    return DAG.getLoad(VT, dl, DAG.getEntryNode(),
1183                       DAG.getNode(ISD::ADD, dl, VT, FrameAddr, Offset),
1184                       MachinePointerInfo());
1185  }
1186
1187  // Return LR, which contains the return address. Mark it an implicit live-in.
1188  Register Reg = MF.addLiveIn(HRI.getRARegister(), getRegClassFor(MVT::i32));
1189  return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT);
1190}
1191
1192SDValue
1193HexagonTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
1194  const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
1195  MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
1196  MFI.setFrameAddressIsTaken(true);
1197
1198  EVT VT = Op.getValueType();
1199  SDLoc dl(Op);
1200  unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1201  SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl,
1202                                         HRI.getFrameRegister(), VT);
1203  while (Depth--)
1204    FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr,
1205                            MachinePointerInfo());
1206  return FrameAddr;
1207}
1208
1209SDValue
1210HexagonTargetLowering::LowerATOMIC_FENCE(SDValue Op, SelectionDAG& DAG) const {
1211  SDLoc dl(Op);
1212  return DAG.getNode(HexagonISD::BARRIER, dl, MVT::Other, Op.getOperand(0));
1213}
1214
1215SDValue
1216HexagonTargetLowering::LowerGLOBALADDRESS(SDValue Op, SelectionDAG &DAG) const {
1217  SDLoc dl(Op);
1218  auto *GAN = cast<GlobalAddressSDNode>(Op);
1219  auto PtrVT = getPointerTy(DAG.getDataLayout());
1220  auto *GV = GAN->getGlobal();
1221  int64_t Offset = GAN->getOffset();
1222
1223  auto &HLOF = *HTM.getObjFileLowering();
1224  Reloc::Model RM = HTM.getRelocationModel();
1225
1226  if (RM == Reloc::Static) {
1227    SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, Offset);
1228    const GlobalObject *GO = GV->getAliaseeObject();
1229    if (GO && Subtarget.useSmallData() && HLOF.isGlobalInSmallSection(GO, HTM))
1230      return DAG.getNode(HexagonISD::CONST32_GP, dl, PtrVT, GA);
1231    return DAG.getNode(HexagonISD::CONST32, dl, PtrVT, GA);
1232  }
1233
1234  bool UsePCRel = getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV);
1235  if (UsePCRel) {
1236    SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, Offset,
1237                                            HexagonII::MO_PCREL);
1238    return DAG.getNode(HexagonISD::AT_PCREL, dl, PtrVT, GA);
1239  }
1240
1241  // Use GOT index.
1242  SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(PtrVT);
1243  SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, HexagonII::MO_GOT);
1244  SDValue Off = DAG.getConstant(Offset, dl, MVT::i32);
1245  return DAG.getNode(HexagonISD::AT_GOT, dl, PtrVT, GOT, GA, Off);
1246}
1247
1248// Specifies that for loads and stores VT can be promoted to PromotedLdStVT.
1249SDValue
1250HexagonTargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
1251  const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
1252  SDLoc dl(Op);
1253  EVT PtrVT = getPointerTy(DAG.getDataLayout());
1254
1255  Reloc::Model RM = HTM.getRelocationModel();
1256  if (RM == Reloc::Static) {
1257    SDValue A = DAG.getTargetBlockAddress(BA, PtrVT);
1258    return DAG.getNode(HexagonISD::CONST32_GP, dl, PtrVT, A);
1259  }
1260
1261  SDValue A = DAG.getTargetBlockAddress(BA, PtrVT, 0, HexagonII::MO_PCREL);
1262  return DAG.getNode(HexagonISD::AT_PCREL, dl, PtrVT, A);
1263}
1264
1265SDValue
1266HexagonTargetLowering::LowerGLOBAL_OFFSET_TABLE(SDValue Op, SelectionDAG &DAG)
1267      const {
1268  EVT PtrVT = getPointerTy(DAG.getDataLayout());
1269  SDValue GOTSym = DAG.getTargetExternalSymbol(HEXAGON_GOT_SYM_NAME, PtrVT,
1270                                               HexagonII::MO_PCREL);
1271  return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Op), PtrVT, GOTSym);
1272}
1273
1274SDValue
1275HexagonTargetLowering::GetDynamicTLSAddr(SelectionDAG &DAG, SDValue Chain,
1276      GlobalAddressSDNode *GA, SDValue Glue, EVT PtrVT, unsigned ReturnReg,
1277      unsigned char OperandFlags) const {
1278  MachineFunction &MF = DAG.getMachineFunction();
1279  MachineFrameInfo &MFI = MF.getFrameInfo();
1280  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
1281  SDLoc dl(GA);
1282  SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
1283                                           GA->getValueType(0),
1284                                           GA->getOffset(),
1285                                           OperandFlags);
1286  // Create Operands for the call.The Operands should have the following:
1287  // 1. Chain SDValue
1288  // 2. Callee which in this case is the Global address value.
1289  // 3. Registers live into the call.In this case its R0, as we
1290  //    have just one argument to be passed.
1291  // 4. Glue.
1292  // Note: The order is important.
1293
1294  const auto &HRI = *Subtarget.getRegisterInfo();
1295  const uint32_t *Mask = HRI.getCallPreservedMask(MF, CallingConv::C);
1296  assert(Mask && "Missing call preserved mask for calling convention");
1297  SDValue Ops[] = { Chain, TGA, DAG.getRegister(Hexagon::R0, PtrVT),
1298                    DAG.getRegisterMask(Mask), Glue };
1299  Chain = DAG.getNode(HexagonISD::CALL, dl, NodeTys, Ops);
1300
1301  // Inform MFI that function has calls.
1302  MFI.setAdjustsStack(true);
1303
1304  Glue = Chain.getValue(1);
1305  return DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Glue);
1306}
1307
1308//
1309// Lower using the intial executable model for TLS addresses
1310//
1311SDValue
1312HexagonTargetLowering::LowerToTLSInitialExecModel(GlobalAddressSDNode *GA,
1313      SelectionDAG &DAG) const {
1314  SDLoc dl(GA);
1315  int64_t Offset = GA->getOffset();
1316  auto PtrVT = getPointerTy(DAG.getDataLayout());
1317
1318  // Get the thread pointer.
1319  SDValue TP = DAG.getCopyFromReg(DAG.getEntryNode(), dl, Hexagon::UGP, PtrVT);
1320
1321  bool IsPositionIndependent = isPositionIndependent();
1322  unsigned char TF =
1323      IsPositionIndependent ? HexagonII::MO_IEGOT : HexagonII::MO_IE;
1324
1325  // First generate the TLS symbol address
1326  SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT,
1327                                           Offset, TF);
1328
1329  SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA);
1330
1331  if (IsPositionIndependent) {
1332    // Generate the GOT pointer in case of position independent code
1333    SDValue GOT = LowerGLOBAL_OFFSET_TABLE(Sym, DAG);
1334
1335    // Add the TLS Symbol address to GOT pointer.This gives
1336    // GOT relative relocation for the symbol.
1337    Sym = DAG.getNode(ISD::ADD, dl, PtrVT, GOT, Sym);
1338  }
1339
1340  // Load the offset value for TLS symbol.This offset is relative to
1341  // thread pointer.
1342  SDValue LoadOffset =
1343      DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Sym, MachinePointerInfo());
1344
1345  // Address of the thread local variable is the add of thread
1346  // pointer and the offset of the variable.
1347  return DAG.getNode(ISD::ADD, dl, PtrVT, TP, LoadOffset);
1348}
1349
1350//
1351// Lower using the local executable model for TLS addresses
1352//
1353SDValue
1354HexagonTargetLowering::LowerToTLSLocalExecModel(GlobalAddressSDNode *GA,
1355      SelectionDAG &DAG) const {
1356  SDLoc dl(GA);
1357  int64_t Offset = GA->getOffset();
1358  auto PtrVT = getPointerTy(DAG.getDataLayout());
1359
1360  // Get the thread pointer.
1361  SDValue TP = DAG.getCopyFromReg(DAG.getEntryNode(), dl, Hexagon::UGP, PtrVT);
1362  // Generate the TLS symbol address
1363  SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT, Offset,
1364                                           HexagonII::MO_TPREL);
1365  SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA);
1366
1367  // Address of the thread local variable is the add of thread
1368  // pointer and the offset of the variable.
1369  return DAG.getNode(ISD::ADD, dl, PtrVT, TP, Sym);
1370}
1371
1372//
1373// Lower using the general dynamic model for TLS addresses
1374//
1375SDValue
1376HexagonTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
1377      SelectionDAG &DAG) const {
1378  SDLoc dl(GA);
1379  int64_t Offset = GA->getOffset();
1380  auto PtrVT = getPointerTy(DAG.getDataLayout());
1381
1382  // First generate the TLS symbol address
1383  SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, PtrVT, Offset,
1384                                           HexagonII::MO_GDGOT);
1385
1386  // Then, generate the GOT pointer
1387  SDValue GOT = LowerGLOBAL_OFFSET_TABLE(TGA, DAG);
1388
1389  // Add the TLS symbol and the GOT pointer
1390  SDValue Sym = DAG.getNode(HexagonISD::CONST32, dl, PtrVT, TGA);
1391  SDValue Chain = DAG.getNode(ISD::ADD, dl, PtrVT, GOT, Sym);
1392
1393  // Copy over the argument to R0
1394  SDValue InFlag;
1395  Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, Hexagon::R0, Chain, InFlag);
1396  InFlag = Chain.getValue(1);
1397
1398  unsigned Flags = DAG.getSubtarget<HexagonSubtarget>().useLongCalls()
1399                       ? HexagonII::MO_GDPLT | HexagonII::HMOTF_ConstExtended
1400                       : HexagonII::MO_GDPLT;
1401
1402  return GetDynamicTLSAddr(DAG, Chain, GA, InFlag, PtrVT,
1403                           Hexagon::R0, Flags);
1404}
1405
1406//
1407// Lower TLS addresses.
1408//
1409// For now for dynamic models, we only support the general dynamic model.
1410//
1411SDValue
1412HexagonTargetLowering::LowerGlobalTLSAddress(SDValue Op,
1413      SelectionDAG &DAG) const {
1414  GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
1415
1416  switch (HTM.getTLSModel(GA->getGlobal())) {
1417    case TLSModel::GeneralDynamic:
1418    case TLSModel::LocalDynamic:
1419      return LowerToTLSGeneralDynamicModel(GA, DAG);
1420    case TLSModel::InitialExec:
1421      return LowerToTLSInitialExecModel(GA, DAG);
1422    case TLSModel::LocalExec:
1423      return LowerToTLSLocalExecModel(GA, DAG);
1424  }
1425  llvm_unreachable("Bogus TLS model");
1426}
1427
1428//===----------------------------------------------------------------------===//
1429// TargetLowering Implementation
1430//===----------------------------------------------------------------------===//
1431
1432HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM,
1433                                             const HexagonSubtarget &ST)
1434    : TargetLowering(TM), HTM(static_cast<const HexagonTargetMachine&>(TM)),
1435      Subtarget(ST) {
1436  auto &HRI = *Subtarget.getRegisterInfo();
1437
1438  setPrefLoopAlignment(Align(16));
1439  setMinFunctionAlignment(Align(4));
1440  setPrefFunctionAlignment(Align(16));
1441  setStackPointerRegisterToSaveRestore(HRI.getStackRegister());
1442  setBooleanContents(TargetLoweringBase::UndefinedBooleanContent);
1443  setBooleanVectorContents(TargetLoweringBase::UndefinedBooleanContent);
1444
1445  setMaxAtomicSizeInBitsSupported(64);
1446  setMinCmpXchgSizeInBits(32);
1447
1448  if (EnableHexSDNodeSched)
1449    setSchedulingPreference(Sched::VLIW);
1450  else
1451    setSchedulingPreference(Sched::Source);
1452
1453  // Limits for inline expansion of memcpy/memmove
1454  MaxStoresPerMemcpy = MaxStoresPerMemcpyCL;
1455  MaxStoresPerMemcpyOptSize = MaxStoresPerMemcpyOptSizeCL;
1456  MaxStoresPerMemmove = MaxStoresPerMemmoveCL;
1457  MaxStoresPerMemmoveOptSize = MaxStoresPerMemmoveOptSizeCL;
1458  MaxStoresPerMemset = MaxStoresPerMemsetCL;
1459  MaxStoresPerMemsetOptSize = MaxStoresPerMemsetOptSizeCL;
1460
1461  //
1462  // Set up register classes.
1463  //
1464
1465  addRegisterClass(MVT::i1,    &Hexagon::PredRegsRegClass);
1466  addRegisterClass(MVT::v2i1,  &Hexagon::PredRegsRegClass);  // bbbbaaaa
1467  addRegisterClass(MVT::v4i1,  &Hexagon::PredRegsRegClass);  // ddccbbaa
1468  addRegisterClass(MVT::v8i1,  &Hexagon::PredRegsRegClass);  // hgfedcba
1469  addRegisterClass(MVT::i32,   &Hexagon::IntRegsRegClass);
1470  addRegisterClass(MVT::v2i16, &Hexagon::IntRegsRegClass);
1471  addRegisterClass(MVT::v4i8,  &Hexagon::IntRegsRegClass);
1472  addRegisterClass(MVT::i64,   &Hexagon::DoubleRegsRegClass);
1473  addRegisterClass(MVT::v8i8,  &Hexagon::DoubleRegsRegClass);
1474  addRegisterClass(MVT::v4i16, &Hexagon::DoubleRegsRegClass);
1475  addRegisterClass(MVT::v2i32, &Hexagon::DoubleRegsRegClass);
1476
1477  addRegisterClass(MVT::f32, &Hexagon::IntRegsRegClass);
1478  addRegisterClass(MVT::f64, &Hexagon::DoubleRegsRegClass);
1479
1480  //
1481  // Handling of scalar operations.
1482  //
1483  // All operations default to "legal", except:
1484  // - indexed loads and stores (pre-/post-incremented),
1485  // - ANY_EXTEND_VECTOR_INREG, ATOMIC_CMP_SWAP_WITH_SUCCESS, CONCAT_VECTORS,
1486  //   ConstantFP, DEBUGTRAP, FCEIL, FCOPYSIGN, FEXP, FEXP2, FFLOOR, FGETSIGN,
1487  //   FLOG, FLOG2, FLOG10, FMAXNUM, FMINNUM, FNEARBYINT, FRINT, FROUND, TRAP,
1488  //   FTRUNC, PREFETCH, SIGN_EXTEND_VECTOR_INREG, ZERO_EXTEND_VECTOR_INREG,
1489  // which default to "expand" for at least one type.
1490
1491  // Misc operations.
1492  setOperationAction(ISD::ConstantFP,           MVT::f32,   Legal);
1493  setOperationAction(ISD::ConstantFP,           MVT::f64,   Legal);
1494  setOperationAction(ISD::TRAP,                 MVT::Other, Legal);
1495  setOperationAction(ISD::ConstantPool,         MVT::i32,   Custom);
1496  setOperationAction(ISD::JumpTable,            MVT::i32,   Custom);
1497  setOperationAction(ISD::BUILD_PAIR,           MVT::i64,   Expand);
1498  setOperationAction(ISD::SIGN_EXTEND_INREG,    MVT::i1,    Expand);
1499  setOperationAction(ISD::INLINEASM,            MVT::Other, Custom);
1500  setOperationAction(ISD::INLINEASM_BR,         MVT::Other, Custom);
1501  setOperationAction(ISD::PREFETCH,             MVT::Other, Custom);
1502  setOperationAction(ISD::READCYCLECOUNTER,     MVT::i64,   Custom);
1503  setOperationAction(ISD::INTRINSIC_VOID,       MVT::Other, Custom);
1504  setOperationAction(ISD::EH_RETURN,            MVT::Other, Custom);
1505  setOperationAction(ISD::GLOBAL_OFFSET_TABLE,  MVT::i32,   Custom);
1506  setOperationAction(ISD::GlobalTLSAddress,     MVT::i32,   Custom);
1507  setOperationAction(ISD::ATOMIC_FENCE,         MVT::Other, Custom);
1508
1509  // Custom legalize GlobalAddress nodes into CONST32.
1510  setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
1511  setOperationAction(ISD::GlobalAddress, MVT::i8,  Custom);
1512  setOperationAction(ISD::BlockAddress,  MVT::i32, Custom);
1513
1514  // Hexagon needs to optimize cases with negative constants.
1515  setOperationAction(ISD::SETCC, MVT::i8,    Custom);
1516  setOperationAction(ISD::SETCC, MVT::i16,   Custom);
1517  setOperationAction(ISD::SETCC, MVT::v4i8,  Custom);
1518  setOperationAction(ISD::SETCC, MVT::v2i16, Custom);
1519
1520  // VASTART needs to be custom lowered to use the VarArgsFrameIndex.
1521  setOperationAction(ISD::VASTART, MVT::Other, Custom);
1522  setOperationAction(ISD::VAEND,   MVT::Other, Expand);
1523  setOperationAction(ISD::VAARG,   MVT::Other, Expand);
1524  if (Subtarget.isEnvironmentMusl())
1525    setOperationAction(ISD::VACOPY, MVT::Other, Custom);
1526  else
1527    setOperationAction(ISD::VACOPY,  MVT::Other, Expand);
1528
1529  setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
1530  setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
1531  setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom);
1532
1533  if (EmitJumpTables)
1534    setMinimumJumpTableEntries(MinimumJumpTables);
1535  else
1536    setMinimumJumpTableEntries(std::numeric_limits<unsigned>::max());
1537  setOperationAction(ISD::BR_JT, MVT::Other, Expand);
1538
1539  for (unsigned LegalIntOp :
1540       {ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}) {
1541    setOperationAction(LegalIntOp, MVT::i32, Legal);
1542    setOperationAction(LegalIntOp, MVT::i64, Legal);
1543  }
1544
1545  // Hexagon has A4_addp_c and A4_subp_c that take and generate a carry bit,
1546  // but they only operate on i64.
1547  for (MVT VT : MVT::integer_valuetypes()) {
1548    setOperationAction(ISD::UADDO,    VT, Custom);
1549    setOperationAction(ISD::USUBO,    VT, Custom);
1550    setOperationAction(ISD::SADDO,    VT, Expand);
1551    setOperationAction(ISD::SSUBO,    VT, Expand);
1552    setOperationAction(ISD::ADDCARRY, VT, Expand);
1553    setOperationAction(ISD::SUBCARRY, VT, Expand);
1554  }
1555  setOperationAction(ISD::ADDCARRY, MVT::i64, Custom);
1556  setOperationAction(ISD::SUBCARRY, MVT::i64, Custom);
1557
1558  setOperationAction(ISD::CTLZ, MVT::i8,  Promote);
1559  setOperationAction(ISD::CTLZ, MVT::i16, Promote);
1560  setOperationAction(ISD::CTTZ, MVT::i8,  Promote);
1561  setOperationAction(ISD::CTTZ, MVT::i16, Promote);
1562
1563  // Popcount can count # of 1s in i64 but returns i32.
1564  setOperationAction(ISD::CTPOP, MVT::i8,  Promote);
1565  setOperationAction(ISD::CTPOP, MVT::i16, Promote);
1566  setOperationAction(ISD::CTPOP, MVT::i32, Promote);
1567  setOperationAction(ISD::CTPOP, MVT::i64, Legal);
1568
1569  setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
1570  setOperationAction(ISD::BITREVERSE, MVT::i64, Legal);
1571  setOperationAction(ISD::BSWAP, MVT::i32, Legal);
1572  setOperationAction(ISD::BSWAP, MVT::i64, Legal);
1573
1574  setOperationAction(ISD::FSHL, MVT::i32, Legal);
1575  setOperationAction(ISD::FSHL, MVT::i64, Legal);
1576  setOperationAction(ISD::FSHR, MVT::i32, Legal);
1577  setOperationAction(ISD::FSHR, MVT::i64, Legal);
1578
1579  for (unsigned IntExpOp :
1580       {ISD::SDIV,      ISD::UDIV,      ISD::SREM,      ISD::UREM,
1581        ISD::SDIVREM,   ISD::UDIVREM,   ISD::ROTL,      ISD::ROTR,
1582        ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS,
1583        ISD::SMUL_LOHI, ISD::UMUL_LOHI}) {
1584    for (MVT VT : MVT::integer_valuetypes())
1585      setOperationAction(IntExpOp, VT, Expand);
1586  }
1587
1588  for (unsigned FPExpOp :
1589       {ISD::FDIV, ISD::FREM, ISD::FSQRT, ISD::FSIN, ISD::FCOS, ISD::FSINCOS,
1590        ISD::FPOW, ISD::FCOPYSIGN}) {
1591    for (MVT VT : MVT::fp_valuetypes())
1592      setOperationAction(FPExpOp, VT, Expand);
1593  }
1594
1595  // No extending loads from i32.
1596  for (MVT VT : MVT::integer_valuetypes()) {
1597    setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i32, Expand);
1598    setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i32, Expand);
1599    setLoadExtAction(ISD::EXTLOAD,  VT, MVT::i32, Expand);
1600  }
1601  // Turn FP truncstore into trunc + store.
1602  setTruncStoreAction(MVT::f64, MVT::f32, Expand);
1603  // Turn FP extload into load/fpextend.
1604  for (MVT VT : MVT::fp_valuetypes())
1605    setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
1606
1607  // Expand BR_CC and SELECT_CC for all integer and fp types.
1608  for (MVT VT : MVT::integer_valuetypes()) {
1609    setOperationAction(ISD::BR_CC,     VT, Expand);
1610    setOperationAction(ISD::SELECT_CC, VT, Expand);
1611  }
1612  for (MVT VT : MVT::fp_valuetypes()) {
1613    setOperationAction(ISD::BR_CC,     VT, Expand);
1614    setOperationAction(ISD::SELECT_CC, VT, Expand);
1615  }
1616  setOperationAction(ISD::BR_CC, MVT::Other, Expand);
1617
1618  //
1619  // Handling of vector operations.
1620  //
1621
1622  // Set the action for vector operations to "expand", then override it with
1623  // either "custom" or "legal" for specific cases.
1624  static const unsigned VectExpOps[] = {
1625    // Integer arithmetic:
1626    ISD::ADD,     ISD::SUB,     ISD::MUL,     ISD::SDIV,      ISD::UDIV,
1627    ISD::SREM,    ISD::UREM,    ISD::SDIVREM, ISD::UDIVREM,   ISD::SADDO,
1628    ISD::UADDO,   ISD::SSUBO,   ISD::USUBO,   ISD::SMUL_LOHI, ISD::UMUL_LOHI,
1629    // Logical/bit:
1630    ISD::AND,     ISD::OR,      ISD::XOR,     ISD::ROTL,    ISD::ROTR,
1631    ISD::CTPOP,   ISD::CTLZ,    ISD::CTTZ,    ISD::BSWAP,   ISD::BITREVERSE,
1632    // Floating point arithmetic/math functions:
1633    ISD::FADD,    ISD::FSUB,    ISD::FMUL,    ISD::FMA,     ISD::FDIV,
1634    ISD::FREM,    ISD::FNEG,    ISD::FABS,    ISD::FSQRT,   ISD::FSIN,
1635    ISD::FCOS,    ISD::FPOW,    ISD::FLOG,    ISD::FLOG2,
1636    ISD::FLOG10,  ISD::FEXP,    ISD::FEXP2,   ISD::FCEIL,   ISD::FTRUNC,
1637    ISD::FRINT,   ISD::FNEARBYINT,            ISD::FROUND,  ISD::FFLOOR,
1638    ISD::FMINNUM, ISD::FMAXNUM, ISD::FSINCOS,
1639    // Misc:
1640    ISD::BR_CC,   ISD::SELECT_CC,             ISD::ConstantPool,
1641    // Vector:
1642    ISD::BUILD_VECTOR,          ISD::SCALAR_TO_VECTOR,
1643    ISD::EXTRACT_VECTOR_ELT,    ISD::INSERT_VECTOR_ELT,
1644    ISD::EXTRACT_SUBVECTOR,     ISD::INSERT_SUBVECTOR,
1645    ISD::CONCAT_VECTORS,        ISD::VECTOR_SHUFFLE,
1646    ISD::SPLAT_VECTOR,
1647  };
1648
1649  for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
1650    for (unsigned VectExpOp : VectExpOps)
1651      setOperationAction(VectExpOp, VT, Expand);
1652
1653    // Expand all extending loads and truncating stores:
1654    for (MVT TargetVT : MVT::fixedlen_vector_valuetypes()) {
1655      if (TargetVT == VT)
1656        continue;
1657      setLoadExtAction(ISD::EXTLOAD, TargetVT, VT, Expand);
1658      setLoadExtAction(ISD::ZEXTLOAD, TargetVT, VT, Expand);
1659      setLoadExtAction(ISD::SEXTLOAD, TargetVT, VT, Expand);
1660      setTruncStoreAction(VT, TargetVT, Expand);
1661    }
1662
1663    // Normalize all inputs to SELECT to be vectors of i32.
1664    if (VT.getVectorElementType() != MVT::i32) {
1665      MVT VT32 = MVT::getVectorVT(MVT::i32, VT.getSizeInBits()/32);
1666      setOperationAction(ISD::SELECT, VT, Promote);
1667      AddPromotedToType(ISD::SELECT, VT, VT32);
1668    }
1669    setOperationAction(ISD::SRA, VT, Custom);
1670    setOperationAction(ISD::SHL, VT, Custom);
1671    setOperationAction(ISD::SRL, VT, Custom);
1672  }
1673
1674  // Extending loads from (native) vectors of i8 into (native) vectors of i16
1675  // are legal.
1676  setLoadExtAction(ISD::EXTLOAD,  MVT::v2i16, MVT::v2i8, Legal);
1677  setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i16, MVT::v2i8, Legal);
1678  setLoadExtAction(ISD::SEXTLOAD, MVT::v2i16, MVT::v2i8, Legal);
1679  setLoadExtAction(ISD::EXTLOAD,  MVT::v4i16, MVT::v4i8, Legal);
1680  setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i16, MVT::v4i8, Legal);
1681  setLoadExtAction(ISD::SEXTLOAD, MVT::v4i16, MVT::v4i8, Legal);
1682
1683  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8,  Legal);
1684  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Legal);
1685  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i32, Legal);
1686
1687  // Types natively supported:
1688  for (MVT NativeVT : {MVT::v8i1, MVT::v4i1, MVT::v2i1, MVT::v4i8,
1689                       MVT::v8i8, MVT::v2i16, MVT::v4i16, MVT::v2i32}) {
1690    setOperationAction(ISD::BUILD_VECTOR,       NativeVT, Custom);
1691    setOperationAction(ISD::EXTRACT_VECTOR_ELT, NativeVT, Custom);
1692    setOperationAction(ISD::INSERT_VECTOR_ELT,  NativeVT, Custom);
1693    setOperationAction(ISD::EXTRACT_SUBVECTOR,  NativeVT, Custom);
1694    setOperationAction(ISD::INSERT_SUBVECTOR,   NativeVT, Custom);
1695    setOperationAction(ISD::CONCAT_VECTORS,     NativeVT, Custom);
1696
1697    setOperationAction(ISD::ADD, NativeVT, Legal);
1698    setOperationAction(ISD::SUB, NativeVT, Legal);
1699    setOperationAction(ISD::MUL, NativeVT, Legal);
1700    setOperationAction(ISD::AND, NativeVT, Legal);
1701    setOperationAction(ISD::OR,  NativeVT, Legal);
1702    setOperationAction(ISD::XOR, NativeVT, Legal);
1703
1704    if (NativeVT.getVectorElementType() != MVT::i1) {
1705      setOperationAction(ISD::SPLAT_VECTOR, NativeVT, Legal);
1706      setOperationAction(ISD::BSWAP,        NativeVT, Legal);
1707      setOperationAction(ISD::BITREVERSE,   NativeVT, Legal);
1708    }
1709  }
1710
1711  for (MVT VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32}) {
1712    setOperationAction(ISD::SMIN, VT, Legal);
1713    setOperationAction(ISD::SMAX, VT, Legal);
1714    setOperationAction(ISD::UMIN, VT, Legal);
1715    setOperationAction(ISD::UMAX, VT, Legal);
1716  }
1717
1718  // Custom lower unaligned loads.
1719  // Also, for both loads and stores, verify the alignment of the address
1720  // in case it is a compile-time constant. This is a usability feature to
1721  // provide a meaningful error message to users.
1722  for (MVT VT : {MVT::i16, MVT::i32, MVT::v4i8, MVT::i64, MVT::v8i8,
1723                 MVT::v2i16, MVT::v4i16, MVT::v2i32}) {
1724    setOperationAction(ISD::LOAD,  VT, Custom);
1725    setOperationAction(ISD::STORE, VT, Custom);
1726  }
1727
1728  // Custom-lower load/stores of boolean vectors.
1729  for (MVT VT : {MVT::v2i1, MVT::v4i1, MVT::v8i1}) {
1730    setOperationAction(ISD::LOAD,  VT, Custom);
1731    setOperationAction(ISD::STORE, VT, Custom);
1732  }
1733
1734  for (MVT VT : {MVT::v2i16, MVT::v4i8, MVT::v8i8, MVT::v2i32, MVT::v4i16,
1735                 MVT::v2i32}) {
1736    setCondCodeAction(ISD::SETNE,  VT, Expand);
1737    setCondCodeAction(ISD::SETLE,  VT, Expand);
1738    setCondCodeAction(ISD::SETGE,  VT, Expand);
1739    setCondCodeAction(ISD::SETLT,  VT, Expand);
1740    setCondCodeAction(ISD::SETULE, VT, Expand);
1741    setCondCodeAction(ISD::SETUGE, VT, Expand);
1742    setCondCodeAction(ISD::SETULT, VT, Expand);
1743  }
1744
1745  // Custom-lower bitcasts from i8 to v8i1.
1746  setOperationAction(ISD::BITCAST,        MVT::i8,    Custom);
1747  setOperationAction(ISD::SETCC,          MVT::v2i16, Custom);
1748  setOperationAction(ISD::VSELECT,        MVT::v4i8,  Custom);
1749  setOperationAction(ISD::VSELECT,        MVT::v2i16, Custom);
1750  setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i8,  Custom);
1751  setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i16, Custom);
1752  setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i8,  Custom);
1753
1754  // V5+.
1755  setOperationAction(ISD::FMA,  MVT::f64, Expand);
1756  setOperationAction(ISD::FADD, MVT::f64, Expand);
1757  setOperationAction(ISD::FSUB, MVT::f64, Expand);
1758  setOperationAction(ISD::FMUL, MVT::f64, Expand);
1759
1760  setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
1761  setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
1762
1763  setOperationAction(ISD::FP_TO_UINT, MVT::i1,  Promote);
1764  setOperationAction(ISD::FP_TO_UINT, MVT::i8,  Promote);
1765  setOperationAction(ISD::FP_TO_UINT, MVT::i16, Promote);
1766  setOperationAction(ISD::FP_TO_SINT, MVT::i1,  Promote);
1767  setOperationAction(ISD::FP_TO_SINT, MVT::i8,  Promote);
1768  setOperationAction(ISD::FP_TO_SINT, MVT::i16, Promote);
1769  setOperationAction(ISD::UINT_TO_FP, MVT::i1,  Promote);
1770  setOperationAction(ISD::UINT_TO_FP, MVT::i8,  Promote);
1771  setOperationAction(ISD::UINT_TO_FP, MVT::i16, Promote);
1772  setOperationAction(ISD::SINT_TO_FP, MVT::i1,  Promote);
1773  setOperationAction(ISD::SINT_TO_FP, MVT::i8,  Promote);
1774  setOperationAction(ISD::SINT_TO_FP, MVT::i16, Promote);
1775
1776  // Special handling for half-precision floating point conversions.
1777  // Lower half float conversions into library calls.
1778  setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand);
1779  setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
1780  setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand);
1781  setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand);
1782
1783  setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
1784  setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
1785  setTruncStoreAction(MVT::f32, MVT::f16, Expand);
1786  setTruncStoreAction(MVT::f64, MVT::f16, Expand);
1787
1788  // Handling of indexed loads/stores: default is "expand".
1789  //
1790  for (MVT VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64, MVT::f32, MVT::f64,
1791                 MVT::v2i16, MVT::v2i32, MVT::v4i8, MVT::v4i16, MVT::v8i8}) {
1792    setIndexedLoadAction(ISD::POST_INC, VT, Legal);
1793    setIndexedStoreAction(ISD::POST_INC, VT, Legal);
1794  }
1795
1796  // Subtarget-specific operation actions.
1797  //
1798  if (Subtarget.hasV60Ops()) {
1799    setOperationAction(ISD::ROTL, MVT::i32, Legal);
1800    setOperationAction(ISD::ROTL, MVT::i64, Legal);
1801    setOperationAction(ISD::ROTR, MVT::i32, Legal);
1802    setOperationAction(ISD::ROTR, MVT::i64, Legal);
1803  }
1804  if (Subtarget.hasV66Ops()) {
1805    setOperationAction(ISD::FADD, MVT::f64, Legal);
1806    setOperationAction(ISD::FSUB, MVT::f64, Legal);
1807  }
1808  if (Subtarget.hasV67Ops()) {
1809    setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
1810    setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
1811    setOperationAction(ISD::FMUL,    MVT::f64, Legal);
1812  }
1813
1814  setTargetDAGCombine(ISD::OR);
1815  setTargetDAGCombine(ISD::TRUNCATE);
1816  setTargetDAGCombine(ISD::VSELECT);
1817
1818  if (Subtarget.useHVXOps())
1819    initializeHVXLowering();
1820
1821  computeRegisterProperties(&HRI);
1822
1823  //
1824  // Library calls for unsupported operations
1825  //
1826  bool FastMath  = EnableFastMath;
1827
1828  setLibcallName(RTLIB::SDIV_I32, "__hexagon_divsi3");
1829  setLibcallName(RTLIB::SDIV_I64, "__hexagon_divdi3");
1830  setLibcallName(RTLIB::UDIV_I32, "__hexagon_udivsi3");
1831  setLibcallName(RTLIB::UDIV_I64, "__hexagon_udivdi3");
1832  setLibcallName(RTLIB::SREM_I32, "__hexagon_modsi3");
1833  setLibcallName(RTLIB::SREM_I64, "__hexagon_moddi3");
1834  setLibcallName(RTLIB::UREM_I32, "__hexagon_umodsi3");
1835  setLibcallName(RTLIB::UREM_I64, "__hexagon_umoddi3");
1836
1837  setLibcallName(RTLIB::SINTTOFP_I128_F64, "__hexagon_floattidf");
1838  setLibcallName(RTLIB::SINTTOFP_I128_F32, "__hexagon_floattisf");
1839  setLibcallName(RTLIB::FPTOUINT_F32_I128, "__hexagon_fixunssfti");
1840  setLibcallName(RTLIB::FPTOUINT_F64_I128, "__hexagon_fixunsdfti");
1841  setLibcallName(RTLIB::FPTOSINT_F32_I128, "__hexagon_fixsfti");
1842  setLibcallName(RTLIB::FPTOSINT_F64_I128, "__hexagon_fixdfti");
1843
1844  // This is the only fast library function for sqrtd.
1845  if (FastMath)
1846    setLibcallName(RTLIB::SQRT_F64, "__hexagon_fast2_sqrtdf2");
1847
1848  // Prefix is: nothing  for "slow-math",
1849  //            "fast2_" for V5+ fast-math double-precision
1850  // (actually, keep fast-math and fast-math2 separate for now)
1851  if (FastMath) {
1852    setLibcallName(RTLIB::ADD_F64, "__hexagon_fast_adddf3");
1853    setLibcallName(RTLIB::SUB_F64, "__hexagon_fast_subdf3");
1854    setLibcallName(RTLIB::MUL_F64, "__hexagon_fast_muldf3");
1855    setLibcallName(RTLIB::DIV_F64, "__hexagon_fast_divdf3");
1856    setLibcallName(RTLIB::DIV_F32, "__hexagon_fast_divsf3");
1857  } else {
1858    setLibcallName(RTLIB::ADD_F64, "__hexagon_adddf3");
1859    setLibcallName(RTLIB::SUB_F64, "__hexagon_subdf3");
1860    setLibcallName(RTLIB::MUL_F64, "__hexagon_muldf3");
1861    setLibcallName(RTLIB::DIV_F64, "__hexagon_divdf3");
1862    setLibcallName(RTLIB::DIV_F32, "__hexagon_divsf3");
1863  }
1864
1865  if (FastMath)
1866    setLibcallName(RTLIB::SQRT_F32, "__hexagon_fast2_sqrtf");
1867  else
1868    setLibcallName(RTLIB::SQRT_F32, "__hexagon_sqrtf");
1869
1870  // Routines to handle fp16 storage type.
1871  setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
1872  setLibcallName(RTLIB::FPROUND_F64_F16, "__truncdfhf2");
1873  setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
1874
1875  // These cause problems when the shift amount is non-constant.
1876  setLibcallName(RTLIB::SHL_I128, nullptr);
1877  setLibcallName(RTLIB::SRL_I128, nullptr);
1878  setLibcallName(RTLIB::SRA_I128, nullptr);
1879}
1880
1881const char* HexagonTargetLowering::getTargetNodeName(unsigned Opcode) const {
1882  switch ((HexagonISD::NodeType)Opcode) {
1883  case HexagonISD::ADDC:          return "HexagonISD::ADDC";
1884  case HexagonISD::SUBC:          return "HexagonISD::SUBC";
1885  case HexagonISD::ALLOCA:        return "HexagonISD::ALLOCA";
1886  case HexagonISD::AT_GOT:        return "HexagonISD::AT_GOT";
1887  case HexagonISD::AT_PCREL:      return "HexagonISD::AT_PCREL";
1888  case HexagonISD::BARRIER:       return "HexagonISD::BARRIER";
1889  case HexagonISD::CALL:          return "HexagonISD::CALL";
1890  case HexagonISD::CALLnr:        return "HexagonISD::CALLnr";
1891  case HexagonISD::CALLR:         return "HexagonISD::CALLR";
1892  case HexagonISD::COMBINE:       return "HexagonISD::COMBINE";
1893  case HexagonISD::CONST32_GP:    return "HexagonISD::CONST32_GP";
1894  case HexagonISD::CONST32:       return "HexagonISD::CONST32";
1895  case HexagonISD::CP:            return "HexagonISD::CP";
1896  case HexagonISD::DCFETCH:       return "HexagonISD::DCFETCH";
1897  case HexagonISD::EH_RETURN:     return "HexagonISD::EH_RETURN";
1898  case HexagonISD::TSTBIT:        return "HexagonISD::TSTBIT";
1899  case HexagonISD::EXTRACTU:      return "HexagonISD::EXTRACTU";
1900  case HexagonISD::INSERT:        return "HexagonISD::INSERT";
1901  case HexagonISD::JT:            return "HexagonISD::JT";
1902  case HexagonISD::RET_FLAG:      return "HexagonISD::RET_FLAG";
1903  case HexagonISD::TC_RETURN:     return "HexagonISD::TC_RETURN";
1904  case HexagonISD::VASL:          return "HexagonISD::VASL";
1905  case HexagonISD::VASR:          return "HexagonISD::VASR";
1906  case HexagonISD::VLSR:          return "HexagonISD::VLSR";
1907  case HexagonISD::MFSHL:         return "HexagonISD::MFSHL";
1908  case HexagonISD::MFSHR:         return "HexagonISD::MFSHR";
1909  case HexagonISD::SSAT:          return "HexagonISD::SSAT";
1910  case HexagonISD::USAT:          return "HexagonISD::USAT";
1911  case HexagonISD::SMUL_LOHI:     return "HexagonISD::SMUL_LOHI";
1912  case HexagonISD::UMUL_LOHI:     return "HexagonISD::UMUL_LOHI";
1913  case HexagonISD::USMUL_LOHI:    return "HexagonISD::USMUL_LOHI";
1914  case HexagonISD::VEXTRACTW:     return "HexagonISD::VEXTRACTW";
1915  case HexagonISD::VINSERTW0:     return "HexagonISD::VINSERTW0";
1916  case HexagonISD::VROR:          return "HexagonISD::VROR";
1917  case HexagonISD::READCYCLE:     return "HexagonISD::READCYCLE";
1918  case HexagonISD::PTRUE:         return "HexagonISD::PTRUE";
1919  case HexagonISD::PFALSE:        return "HexagonISD::PFALSE";
1920  case HexagonISD::D2P:           return "HexagonISD::D2P";
1921  case HexagonISD::P2D:           return "HexagonISD::P2D";
1922  case HexagonISD::V2Q:           return "HexagonISD::V2Q";
1923  case HexagonISD::Q2V:           return "HexagonISD::Q2V";
1924  case HexagonISD::QCAT:          return "HexagonISD::QCAT";
1925  case HexagonISD::QTRUE:         return "HexagonISD::QTRUE";
1926  case HexagonISD::QFALSE:        return "HexagonISD::QFALSE";
1927  case HexagonISD::TL_EXTEND:     return "HexagonISD::TL_EXTEND";
1928  case HexagonISD::TL_TRUNCATE:   return "HexagonISD::TL_TRUNCATE";
1929  case HexagonISD::TYPECAST:      return "HexagonISD::TYPECAST";
1930  case HexagonISD::VALIGN:        return "HexagonISD::VALIGN";
1931  case HexagonISD::VALIGNADDR:    return "HexagonISD::VALIGNADDR";
1932  case HexagonISD::ISEL:          return "HexagonISD::ISEL";
1933  case HexagonISD::OP_END:        break;
1934  }
1935  return nullptr;
1936}
1937
1938bool
1939HexagonTargetLowering::validateConstPtrAlignment(SDValue Ptr, Align NeedAlign,
1940      const SDLoc &dl, SelectionDAG &DAG) const {
1941  auto *CA = dyn_cast<ConstantSDNode>(Ptr);
1942  if (!CA)
1943    return true;
1944  unsigned Addr = CA->getZExtValue();
1945  Align HaveAlign =
1946      Addr != 0 ? Align(1ull << countTrailingZeros(Addr)) : NeedAlign;
1947  if (HaveAlign >= NeedAlign)
1948    return true;
1949
1950  static int DK_MisalignedTrap = llvm::getNextAvailablePluginDiagnosticKind();
1951
1952  struct DiagnosticInfoMisalignedTrap : public DiagnosticInfo {
1953    DiagnosticInfoMisalignedTrap(StringRef M)
1954      : DiagnosticInfo(DK_MisalignedTrap, DS_Remark), Msg(M) {}
1955    void print(DiagnosticPrinter &DP) const override {
1956      DP << Msg;
1957    }
1958    static bool classof(const DiagnosticInfo *DI) {
1959      return DI->getKind() == DK_MisalignedTrap;
1960    }
1961    StringRef Msg;
1962  };
1963
1964  std::string ErrMsg;
1965  raw_string_ostream O(ErrMsg);
1966  O << "Misaligned constant address: " << format_hex(Addr, 10)
1967    << " has alignment " << HaveAlign.value()
1968    << ", but the memory access requires " << NeedAlign.value();
1969  if (DebugLoc DL = dl.getDebugLoc())
1970    DL.print(O << ", at ");
1971  O << ". The instruction has been replaced with a trap.";
1972
1973  DAG.getContext()->diagnose(DiagnosticInfoMisalignedTrap(O.str()));
1974  return false;
1975}
1976
1977SDValue
1978HexagonTargetLowering::replaceMemWithUndef(SDValue Op, SelectionDAG &DAG)
1979      const {
1980  const SDLoc &dl(Op);
1981  auto *LS = cast<LSBaseSDNode>(Op.getNode());
1982  assert(!LS->isIndexed() && "Not expecting indexed ops on constant address");
1983
1984  SDValue Chain = LS->getChain();
1985  SDValue Trap = DAG.getNode(ISD::TRAP, dl, MVT::Other, Chain);
1986  if (LS->getOpcode() == ISD::LOAD)
1987    return DAG.getMergeValues({DAG.getUNDEF(ty(Op)), Trap}, dl);
1988  return Trap;
1989}
1990
1991// Bit-reverse Load Intrinsic: Check if the instruction is a bit reverse load
1992// intrinsic.
1993static bool isBrevLdIntrinsic(const Value *Inst) {
1994  unsigned ID = cast<IntrinsicInst>(Inst)->getIntrinsicID();
1995  return (ID == Intrinsic::hexagon_L2_loadrd_pbr ||
1996          ID == Intrinsic::hexagon_L2_loadri_pbr ||
1997          ID == Intrinsic::hexagon_L2_loadrh_pbr ||
1998          ID == Intrinsic::hexagon_L2_loadruh_pbr ||
1999          ID == Intrinsic::hexagon_L2_loadrb_pbr ||
2000          ID == Intrinsic::hexagon_L2_loadrub_pbr);
2001}
2002
2003// Bit-reverse Load Intrinsic :Crawl up and figure out the object from previous
2004// instruction. So far we only handle bitcast, extract value and bit reverse
2005// load intrinsic instructions. Should we handle CGEP ?
2006static Value *getBrevLdObject(Value *V) {
2007  if (Operator::getOpcode(V) == Instruction::ExtractValue ||
2008      Operator::getOpcode(V) == Instruction::BitCast)
2009    V = cast<Operator>(V)->getOperand(0);
2010  else if (isa<IntrinsicInst>(V) && isBrevLdIntrinsic(V))
2011    V = cast<Instruction>(V)->getOperand(0);
2012  return V;
2013}
2014
2015// Bit-reverse Load Intrinsic: For a PHI Node return either an incoming edge or
2016// a back edge. If the back edge comes from the intrinsic itself, the incoming
2017// edge is returned.
2018static Value *returnEdge(const PHINode *PN, Value *IntrBaseVal) {
2019  const BasicBlock *Parent = PN->getParent();
2020  int Idx = -1;
2021  for (unsigned i = 0, e = PN->getNumIncomingValues(); i < e; ++i) {
2022    BasicBlock *Blk = PN->getIncomingBlock(i);
2023    // Determine if the back edge is originated from intrinsic.
2024    if (Blk == Parent) {
2025      Value *BackEdgeVal = PN->getIncomingValue(i);
2026      Value *BaseVal;
2027      // Loop over till we return the same Value or we hit the IntrBaseVal.
2028      do {
2029        BaseVal = BackEdgeVal;
2030        BackEdgeVal = getBrevLdObject(BackEdgeVal);
2031      } while ((BaseVal != BackEdgeVal) && (IntrBaseVal != BackEdgeVal));
2032      // If the getBrevLdObject returns IntrBaseVal, we should return the
2033      // incoming edge.
2034      if (IntrBaseVal == BackEdgeVal)
2035        continue;
2036      Idx = i;
2037      break;
2038    } else // Set the node to incoming edge.
2039      Idx = i;
2040  }
2041  assert(Idx >= 0 && "Unexpected index to incoming argument in PHI");
2042  return PN->getIncomingValue(Idx);
2043}
2044
2045// Bit-reverse Load Intrinsic: Figure out the underlying object the base
2046// pointer points to, for the bit-reverse load intrinsic. Setting this to
2047// memoperand might help alias analysis to figure out the dependencies.
2048static Value *getUnderLyingObjectForBrevLdIntr(Value *V) {
2049  Value *IntrBaseVal = V;
2050  Value *BaseVal;
2051  // Loop over till we return the same Value, implies we either figure out
2052  // the object or we hit a PHI
2053  do {
2054    BaseVal = V;
2055    V = getBrevLdObject(V);
2056  } while (BaseVal != V);
2057
2058  // Identify the object from PHINode.
2059  if (const PHINode *PN = dyn_cast<PHINode>(V))
2060    return returnEdge(PN, IntrBaseVal);
2061  // For non PHI nodes, the object is the last value returned by getBrevLdObject
2062  else
2063    return V;
2064}
2065
2066/// Given an intrinsic, checks if on the target the intrinsic will need to map
2067/// to a MemIntrinsicNode (touches memory). If this is the case, it returns
2068/// true and store the intrinsic information into the IntrinsicInfo that was
2069/// passed to the function.
2070bool HexagonTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
2071                                               const CallInst &I,
2072                                               MachineFunction &MF,
2073                                               unsigned Intrinsic) const {
2074  switch (Intrinsic) {
2075  case Intrinsic::hexagon_L2_loadrd_pbr:
2076  case Intrinsic::hexagon_L2_loadri_pbr:
2077  case Intrinsic::hexagon_L2_loadrh_pbr:
2078  case Intrinsic::hexagon_L2_loadruh_pbr:
2079  case Intrinsic::hexagon_L2_loadrb_pbr:
2080  case Intrinsic::hexagon_L2_loadrub_pbr: {
2081    Info.opc = ISD::INTRINSIC_W_CHAIN;
2082    auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
2083    auto &Cont = I.getCalledFunction()->getParent()->getContext();
2084    // The intrinsic function call is of the form { ElTy, i8* }
2085    // @llvm.hexagon.L2.loadXX.pbr(i8*, i32). The pointer and memory access type
2086    // should be derived from ElTy.
2087    Type *ElTy = I.getCalledFunction()->getReturnType()->getStructElementType(0);
2088    Info.memVT = MVT::getVT(ElTy);
2089    llvm::Value *BasePtrVal = I.getOperand(0);
2090    Info.ptrVal = getUnderLyingObjectForBrevLdIntr(BasePtrVal);
2091    // The offset value comes through Modifier register. For now, assume the
2092    // offset is 0.
2093    Info.offset = 0;
2094    Info.align = DL.getABITypeAlign(Info.memVT.getTypeForEVT(Cont));
2095    Info.flags = MachineMemOperand::MOLoad;
2096    return true;
2097  }
2098  case Intrinsic::hexagon_V6_vgathermw:
2099  case Intrinsic::hexagon_V6_vgathermw_128B:
2100  case Intrinsic::hexagon_V6_vgathermh:
2101  case Intrinsic::hexagon_V6_vgathermh_128B:
2102  case Intrinsic::hexagon_V6_vgathermhw:
2103  case Intrinsic::hexagon_V6_vgathermhw_128B:
2104  case Intrinsic::hexagon_V6_vgathermwq:
2105  case Intrinsic::hexagon_V6_vgathermwq_128B:
2106  case Intrinsic::hexagon_V6_vgathermhq:
2107  case Intrinsic::hexagon_V6_vgathermhq_128B:
2108  case Intrinsic::hexagon_V6_vgathermhwq:
2109  case Intrinsic::hexagon_V6_vgathermhwq_128B: {
2110    const Module &M = *I.getParent()->getParent()->getParent();
2111    Info.opc = ISD::INTRINSIC_W_CHAIN;
2112    Type *VecTy = I.getArgOperand(1)->getType();
2113    Info.memVT = MVT::getVT(VecTy);
2114    Info.ptrVal = I.getArgOperand(0);
2115    Info.offset = 0;
2116    Info.align =
2117        MaybeAlign(M.getDataLayout().getTypeAllocSizeInBits(VecTy) / 8);
2118    Info.flags = MachineMemOperand::MOLoad |
2119                 MachineMemOperand::MOStore |
2120                 MachineMemOperand::MOVolatile;
2121    return true;
2122  }
2123  default:
2124    break;
2125  }
2126  return false;
2127}
2128
2129bool HexagonTargetLowering::hasBitTest(SDValue X, SDValue Y) const {
2130  return X.getValueType().isScalarInteger(); // 'tstbit'
2131}
2132
2133bool HexagonTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
2134  return isTruncateFree(EVT::getEVT(Ty1), EVT::getEVT(Ty2));
2135}
2136
2137bool HexagonTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
2138  if (!VT1.isSimple() || !VT2.isSimple())
2139    return false;
2140  return VT1.getSimpleVT() == MVT::i64 && VT2.getSimpleVT() == MVT::i32;
2141}
2142
2143bool HexagonTargetLowering::isFMAFasterThanFMulAndFAdd(
2144    const MachineFunction &MF, EVT VT) const {
2145  return isOperationLegalOrCustom(ISD::FMA, VT);
2146}
2147
2148// Should we expand the build vector with shuffles?
2149bool HexagonTargetLowering::shouldExpandBuildVectorWithShuffles(EVT VT,
2150      unsigned DefinedValues) const {
2151  return false;
2152}
2153
2154bool HexagonTargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
2155      unsigned Index) const {
2156  assert(ResVT.getVectorElementType() == SrcVT.getVectorElementType());
2157  if (!ResVT.isSimple() || !SrcVT.isSimple())
2158    return false;
2159
2160  MVT ResTy = ResVT.getSimpleVT(), SrcTy = SrcVT.getSimpleVT();
2161  if (ResTy.getVectorElementType() != MVT::i1)
2162    return true;
2163
2164  // Non-HVX bool vectors are relatively cheap.
2165  return SrcTy.getVectorNumElements() <= 8;
2166}
2167
2168bool HexagonTargetLowering::isTargetCanonicalConstantNode(SDValue Op) const {
2169  return Op.getOpcode() == ISD::CONCAT_VECTORS ||
2170         TargetLowering::isTargetCanonicalConstantNode(Op);
2171}
2172
2173bool HexagonTargetLowering::isShuffleMaskLegal(ArrayRef<int> Mask,
2174                                               EVT VT) const {
2175  return true;
2176}
2177
2178TargetLoweringBase::LegalizeTypeAction
2179HexagonTargetLowering::getPreferredVectorAction(MVT VT) const {
2180  unsigned VecLen = VT.getVectorMinNumElements();
2181  MVT ElemTy = VT.getVectorElementType();
2182
2183  if (VecLen == 1 || VT.isScalableVector())
2184    return TargetLoweringBase::TypeScalarizeVector;
2185
2186  if (Subtarget.useHVXOps()) {
2187    unsigned Action = getPreferredHvxVectorAction(VT);
2188    if (Action != ~0u)
2189      return static_cast<TargetLoweringBase::LegalizeTypeAction>(Action);
2190  }
2191
2192  // Always widen (remaining) vectors of i1.
2193  if (ElemTy == MVT::i1)
2194    return TargetLoweringBase::TypeWidenVector;
2195  // Widen non-power-of-2 vectors. Such types cannot be split right now,
2196  // and computeRegisterProperties will override "split" with "widen",
2197  // which can cause other issues.
2198  if (!isPowerOf2_32(VecLen))
2199    return TargetLoweringBase::TypeWidenVector;
2200
2201  return TargetLoweringBase::TypeSplitVector;
2202}
2203
2204TargetLoweringBase::LegalizeAction
2205HexagonTargetLowering::getCustomOperationAction(SDNode &Op) const {
2206  if (Subtarget.useHVXOps()) {
2207    unsigned Action = getCustomHvxOperationAction(Op);
2208    if (Action != ~0u)
2209      return static_cast<TargetLoweringBase::LegalizeAction>(Action);
2210  }
2211  return TargetLoweringBase::Legal;
2212}
2213
2214std::pair<SDValue, int>
2215HexagonTargetLowering::getBaseAndOffset(SDValue Addr) const {
2216  if (Addr.getOpcode() == ISD::ADD) {
2217    SDValue Op1 = Addr.getOperand(1);
2218    if (auto *CN = dyn_cast<const ConstantSDNode>(Op1.getNode()))
2219      return { Addr.getOperand(0), CN->getSExtValue() };
2220  }
2221  return { Addr, 0 };
2222}
2223
2224// Lower a vector shuffle (V1, V2, V3).  V1 and V2 are the two vectors
2225// to select data from, V3 is the permutation.
2226SDValue
2227HexagonTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG)
2228      const {
2229  const auto *SVN = cast<ShuffleVectorSDNode>(Op);
2230  ArrayRef<int> AM = SVN->getMask();
2231  assert(AM.size() <= 8 && "Unexpected shuffle mask");
2232  unsigned VecLen = AM.size();
2233
2234  MVT VecTy = ty(Op);
2235  assert(!Subtarget.isHVXVectorType(VecTy, true) &&
2236         "HVX shuffles should be legal");
2237  assert(VecTy.getSizeInBits() <= 64 && "Unexpected vector length");
2238
2239  SDValue Op0 = Op.getOperand(0);
2240  SDValue Op1 = Op.getOperand(1);
2241  const SDLoc &dl(Op);
2242
2243  // If the inputs are not the same as the output, bail. This is not an
2244  // error situation, but complicates the handling and the default expansion
2245  // (into BUILD_VECTOR) should be adequate.
2246  if (ty(Op0) != VecTy || ty(Op1) != VecTy)
2247    return SDValue();
2248
2249  // Normalize the mask so that the first non-negative index comes from
2250  // the first operand.
2251  SmallVector<int,8> Mask(AM.begin(), AM.end());
2252  unsigned F = llvm::find_if(AM, [](int M) { return M >= 0; }) - AM.data();
2253  if (F == AM.size())
2254    return DAG.getUNDEF(VecTy);
2255  if (AM[F] >= int(VecLen)) {
2256    ShuffleVectorSDNode::commuteMask(Mask);
2257    std::swap(Op0, Op1);
2258  }
2259
2260  // Express the shuffle mask in terms of bytes.
2261  SmallVector<int,8> ByteMask;
2262  unsigned ElemBytes = VecTy.getVectorElementType().getSizeInBits() / 8;
2263  for (int M : Mask) {
2264    if (M < 0) {
2265      for (unsigned j = 0; j != ElemBytes; ++j)
2266        ByteMask.push_back(-1);
2267    } else {
2268      for (unsigned j = 0; j != ElemBytes; ++j)
2269        ByteMask.push_back(M*ElemBytes + j);
2270    }
2271  }
2272  assert(ByteMask.size() <= 8);
2273
2274  // All non-undef (non-negative) indexes are well within [0..127], so they
2275  // fit in a single byte. Build two 64-bit words:
2276  // - MaskIdx where each byte is the corresponding index (for non-negative
2277  //   indexes), and 0xFF for negative indexes, and
2278  // - MaskUnd that has 0xFF for each negative index.
2279  uint64_t MaskIdx = 0;
2280  uint64_t MaskUnd = 0;
2281  for (unsigned i = 0, e = ByteMask.size(); i != e; ++i) {
2282    unsigned S = 8*i;
2283    uint64_t M = ByteMask[i] & 0xFF;
2284    if (M == 0xFF)
2285      MaskUnd |= M << S;
2286    MaskIdx |= M << S;
2287  }
2288
2289  if (ByteMask.size() == 4) {
2290    // Identity.
2291    if (MaskIdx == (0x03020100 | MaskUnd))
2292      return Op0;
2293    // Byte swap.
2294    if (MaskIdx == (0x00010203 | MaskUnd)) {
2295      SDValue T0 = DAG.getBitcast(MVT::i32, Op0);
2296      SDValue T1 = DAG.getNode(ISD::BSWAP, dl, MVT::i32, T0);
2297      return DAG.getBitcast(VecTy, T1);
2298    }
2299
2300    // Byte packs.
2301    SDValue Concat10 =
2302        getCombine(Op1, Op0, dl, typeJoin({ty(Op1), ty(Op0)}), DAG);
2303    if (MaskIdx == (0x06040200 | MaskUnd))
2304      return getInstr(Hexagon::S2_vtrunehb, dl, VecTy, {Concat10}, DAG);
2305    if (MaskIdx == (0x07050301 | MaskUnd))
2306      return getInstr(Hexagon::S2_vtrunohb, dl, VecTy, {Concat10}, DAG);
2307
2308    SDValue Concat01 =
2309        getCombine(Op0, Op1, dl, typeJoin({ty(Op0), ty(Op1)}), DAG);
2310    if (MaskIdx == (0x02000604 | MaskUnd))
2311      return getInstr(Hexagon::S2_vtrunehb, dl, VecTy, {Concat01}, DAG);
2312    if (MaskIdx == (0x03010705 | MaskUnd))
2313      return getInstr(Hexagon::S2_vtrunohb, dl, VecTy, {Concat01}, DAG);
2314  }
2315
2316  if (ByteMask.size() == 8) {
2317    // Identity.
2318    if (MaskIdx == (0x0706050403020100ull | MaskUnd))
2319      return Op0;
2320    // Byte swap.
2321    if (MaskIdx == (0x0001020304050607ull | MaskUnd)) {
2322      SDValue T0 = DAG.getBitcast(MVT::i64, Op0);
2323      SDValue T1 = DAG.getNode(ISD::BSWAP, dl, MVT::i64, T0);
2324      return DAG.getBitcast(VecTy, T1);
2325    }
2326
2327    // Halfword picks.
2328    if (MaskIdx == (0x0d0c050409080100ull | MaskUnd))
2329      return getInstr(Hexagon::S2_shuffeh, dl, VecTy, {Op1, Op0}, DAG);
2330    if (MaskIdx == (0x0f0e07060b0a0302ull | MaskUnd))
2331      return getInstr(Hexagon::S2_shuffoh, dl, VecTy, {Op1, Op0}, DAG);
2332    if (MaskIdx == (0x0d0c090805040100ull | MaskUnd))
2333      return getInstr(Hexagon::S2_vtrunewh, dl, VecTy, {Op1, Op0}, DAG);
2334    if (MaskIdx == (0x0f0e0b0a07060302ull | MaskUnd))
2335      return getInstr(Hexagon::S2_vtrunowh, dl, VecTy, {Op1, Op0}, DAG);
2336    if (MaskIdx == (0x0706030205040100ull | MaskUnd)) {
2337      VectorPair P = opSplit(Op0, dl, DAG);
2338      return getInstr(Hexagon::S2_packhl, dl, VecTy, {P.second, P.first}, DAG);
2339    }
2340
2341    // Byte packs.
2342    if (MaskIdx == (0x0e060c040a020800ull | MaskUnd))
2343      return getInstr(Hexagon::S2_shuffeb, dl, VecTy, {Op1, Op0}, DAG);
2344    if (MaskIdx == (0x0f070d050b030901ull | MaskUnd))
2345      return getInstr(Hexagon::S2_shuffob, dl, VecTy, {Op1, Op0}, DAG);
2346  }
2347
2348  return SDValue();
2349}
2350
2351SDValue
2352HexagonTargetLowering::getSplatValue(SDValue Op, SelectionDAG &DAG) const {
2353  switch (Op.getOpcode()) {
2354    case ISD::BUILD_VECTOR:
2355      if (SDValue S = cast<BuildVectorSDNode>(Op)->getSplatValue())
2356        return S;
2357      break;
2358    case ISD::SPLAT_VECTOR:
2359      return Op.getOperand(0);
2360  }
2361  return SDValue();
2362}
2363
2364// Create a Hexagon-specific node for shifting a vector by an integer.
2365SDValue
2366HexagonTargetLowering::getVectorShiftByInt(SDValue Op, SelectionDAG &DAG)
2367      const {
2368  unsigned NewOpc;
2369  switch (Op.getOpcode()) {
2370    case ISD::SHL:
2371      NewOpc = HexagonISD::VASL;
2372      break;
2373    case ISD::SRA:
2374      NewOpc = HexagonISD::VASR;
2375      break;
2376    case ISD::SRL:
2377      NewOpc = HexagonISD::VLSR;
2378      break;
2379    default:
2380      llvm_unreachable("Unexpected shift opcode");
2381  }
2382
2383  if (SDValue Sp = getSplatValue(Op.getOperand(1), DAG))
2384    return DAG.getNode(NewOpc, SDLoc(Op), ty(Op), Op.getOperand(0), Sp);
2385  return SDValue();
2386}
2387
2388SDValue
2389HexagonTargetLowering::LowerVECTOR_SHIFT(SDValue Op, SelectionDAG &DAG) const {
2390  const SDLoc &dl(Op);
2391
2392  // First try to convert the shift (by vector) to a shift by a scalar.
2393  // If we first split the shift, the shift amount will become 'extract
2394  // subvector', and will no longer be recognized as scalar.
2395  SDValue Res = Op;
2396  if (SDValue S = getVectorShiftByInt(Op, DAG))
2397    Res = S;
2398
2399  unsigned Opc = Res.getOpcode();
2400  switch (Opc) {
2401  case HexagonISD::VASR:
2402  case HexagonISD::VLSR:
2403  case HexagonISD::VASL:
2404    break;
2405  default:
2406    // No instructions for shifts by non-scalars.
2407    return SDValue();
2408  }
2409
2410  MVT ResTy = ty(Res);
2411  if (ResTy.getVectorElementType() != MVT::i8)
2412    return Res;
2413
2414  // For shifts of i8, extend the inputs to i16, then truncate back to i8.
2415  assert(ResTy.getVectorElementType() == MVT::i8);
2416  SDValue Val = Res.getOperand(0), Amt = Res.getOperand(1);
2417
2418  auto ShiftPartI8 = [&dl, &DAG, this](unsigned Opc, SDValue V, SDValue A) {
2419    MVT Ty = ty(V);
2420    MVT ExtTy = MVT::getVectorVT(MVT::i16, Ty.getVectorNumElements());
2421    SDValue ExtV = Opc == HexagonISD::VASR ? DAG.getSExtOrTrunc(V, dl, ExtTy)
2422                                           : DAG.getZExtOrTrunc(V, dl, ExtTy);
2423    SDValue ExtS = DAG.getNode(Opc, dl, ExtTy, {ExtV, A});
2424    return DAG.getZExtOrTrunc(ExtS, dl, Ty);
2425  };
2426
2427  if (ResTy.getSizeInBits() == 32)
2428    return ShiftPartI8(Opc, Val, Amt);
2429
2430  auto [LoV, HiV] = opSplit(Val, dl, DAG);
2431  return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResTy,
2432                     {ShiftPartI8(Opc, LoV, Amt), ShiftPartI8(Opc, HiV, Amt)});
2433}
2434
2435SDValue
2436HexagonTargetLowering::LowerROTL(SDValue Op, SelectionDAG &DAG) const {
2437  if (isa<ConstantSDNode>(Op.getOperand(1).getNode()))
2438    return Op;
2439  return SDValue();
2440}
2441
2442SDValue
2443HexagonTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
2444  MVT ResTy = ty(Op);
2445  SDValue InpV = Op.getOperand(0);
2446  MVT InpTy = ty(InpV);
2447  assert(ResTy.getSizeInBits() == InpTy.getSizeInBits());
2448  const SDLoc &dl(Op);
2449
2450  // Handle conversion from i8 to v8i1.
2451  if (InpTy == MVT::i8) {
2452    if (ResTy == MVT::v8i1) {
2453      SDValue Sc = DAG.getBitcast(tyScalar(InpTy), InpV);
2454      SDValue Ext = DAG.getZExtOrTrunc(Sc, dl, MVT::i32);
2455      return getInstr(Hexagon::C2_tfrrp, dl, ResTy, Ext, DAG);
2456    }
2457    return SDValue();
2458  }
2459
2460  return Op;
2461}
2462
2463bool
2464HexagonTargetLowering::getBuildVectorConstInts(ArrayRef<SDValue> Values,
2465      MVT VecTy, SelectionDAG &DAG,
2466      MutableArrayRef<ConstantInt*> Consts) const {
2467  MVT ElemTy = VecTy.getVectorElementType();
2468  unsigned ElemWidth = ElemTy.getSizeInBits();
2469  IntegerType *IntTy = IntegerType::get(*DAG.getContext(), ElemWidth);
2470  bool AllConst = true;
2471
2472  for (unsigned i = 0, e = Values.size(); i != e; ++i) {
2473    SDValue V = Values[i];
2474    if (V.isUndef()) {
2475      Consts[i] = ConstantInt::get(IntTy, 0);
2476      continue;
2477    }
2478    // Make sure to always cast to IntTy.
2479    if (auto *CN = dyn_cast<ConstantSDNode>(V.getNode())) {
2480      const ConstantInt *CI = CN->getConstantIntValue();
2481      Consts[i] = ConstantInt::get(IntTy, CI->getValue().getSExtValue());
2482    } else if (auto *CN = dyn_cast<ConstantFPSDNode>(V.getNode())) {
2483      const ConstantFP *CF = CN->getConstantFPValue();
2484      APInt A = CF->getValueAPF().bitcastToAPInt();
2485      Consts[i] = ConstantInt::get(IntTy, A.getZExtValue());
2486    } else {
2487      AllConst = false;
2488    }
2489  }
2490  return AllConst;
2491}
2492
2493SDValue
2494HexagonTargetLowering::buildVector32(ArrayRef<SDValue> Elem, const SDLoc &dl,
2495                                     MVT VecTy, SelectionDAG &DAG) const {
2496  MVT ElemTy = VecTy.getVectorElementType();
2497  assert(VecTy.getVectorNumElements() == Elem.size());
2498
2499  SmallVector<ConstantInt*,4> Consts(Elem.size());
2500  bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts);
2501
2502  unsigned First, Num = Elem.size();
2503  for (First = 0; First != Num; ++First) {
2504    if (!isUndef(Elem[First]))
2505      break;
2506  }
2507  if (First == Num)
2508    return DAG.getUNDEF(VecTy);
2509
2510  if (AllConst &&
2511      llvm::all_of(Consts, [](ConstantInt *CI) { return CI->isZero(); }))
2512    return getZero(dl, VecTy, DAG);
2513
2514  if (ElemTy == MVT::i16 || ElemTy == MVT::f16) {
2515    assert(Elem.size() == 2);
2516    if (AllConst) {
2517      // The 'Consts' array will have all values as integers regardless
2518      // of the vector element type.
2519      uint32_t V = (Consts[0]->getZExtValue() & 0xFFFF) |
2520                   Consts[1]->getZExtValue() << 16;
2521      return DAG.getBitcast(VecTy, DAG.getConstant(V, dl, MVT::i32));
2522    }
2523    SDValue E0, E1;
2524    if (ElemTy == MVT::f16) {
2525      E0 = DAG.getZExtOrTrunc(DAG.getBitcast(MVT::i16, Elem[0]), dl, MVT::i32);
2526      E1 = DAG.getZExtOrTrunc(DAG.getBitcast(MVT::i16, Elem[1]), dl, MVT::i32);
2527    } else {
2528      E0 = Elem[0];
2529      E1 = Elem[1];
2530    }
2531    SDValue N = getInstr(Hexagon::A2_combine_ll, dl, MVT::i32, {E1, E0}, DAG);
2532    return DAG.getBitcast(VecTy, N);
2533  }
2534
2535  if (ElemTy == MVT::i8) {
2536    // First try generating a constant.
2537    if (AllConst) {
2538      int32_t V = (Consts[0]->getZExtValue() & 0xFF) |
2539                  (Consts[1]->getZExtValue() & 0xFF) << 8 |
2540                  (Consts[2]->getZExtValue() & 0xFF) << 16 |
2541                  Consts[3]->getZExtValue() << 24;
2542      return DAG.getBitcast(MVT::v4i8, DAG.getConstant(V, dl, MVT::i32));
2543    }
2544
2545    // Then try splat.
2546    bool IsSplat = true;
2547    for (unsigned i = First+1; i != Num; ++i) {
2548      if (Elem[i] == Elem[First] || isUndef(Elem[i]))
2549        continue;
2550      IsSplat = false;
2551      break;
2552    }
2553    if (IsSplat) {
2554      // Legalize the operand of SPLAT_VECTOR.
2555      SDValue Ext = DAG.getZExtOrTrunc(Elem[First], dl, MVT::i32);
2556      return DAG.getNode(ISD::SPLAT_VECTOR, dl, VecTy, Ext);
2557    }
2558
2559    // Generate
2560    //   (zxtb(Elem[0]) | (zxtb(Elem[1]) << 8)) |
2561    //   (zxtb(Elem[2]) | (zxtb(Elem[3]) << 8)) << 16
2562    assert(Elem.size() == 4);
2563    SDValue Vs[4];
2564    for (unsigned i = 0; i != 4; ++i) {
2565      Vs[i] = DAG.getZExtOrTrunc(Elem[i], dl, MVT::i32);
2566      Vs[i] = DAG.getZeroExtendInReg(Vs[i], dl, MVT::i8);
2567    }
2568    SDValue S8 = DAG.getConstant(8, dl, MVT::i32);
2569    SDValue T0 = DAG.getNode(ISD::SHL, dl, MVT::i32, {Vs[1], S8});
2570    SDValue T1 = DAG.getNode(ISD::SHL, dl, MVT::i32, {Vs[3], S8});
2571    SDValue B0 = DAG.getNode(ISD::OR, dl, MVT::i32, {Vs[0], T0});
2572    SDValue B1 = DAG.getNode(ISD::OR, dl, MVT::i32, {Vs[2], T1});
2573
2574    SDValue R = getInstr(Hexagon::A2_combine_ll, dl, MVT::i32, {B1, B0}, DAG);
2575    return DAG.getBitcast(MVT::v4i8, R);
2576  }
2577
2578#ifndef NDEBUG
2579  dbgs() << "VecTy: " << EVT(VecTy).getEVTString() << '\n';
2580#endif
2581  llvm_unreachable("Unexpected vector element type");
2582}
2583
2584SDValue
2585HexagonTargetLowering::buildVector64(ArrayRef<SDValue> Elem, const SDLoc &dl,
2586                                     MVT VecTy, SelectionDAG &DAG) const {
2587  MVT ElemTy = VecTy.getVectorElementType();
2588  assert(VecTy.getVectorNumElements() == Elem.size());
2589
2590  SmallVector<ConstantInt*,8> Consts(Elem.size());
2591  bool AllConst = getBuildVectorConstInts(Elem, VecTy, DAG, Consts);
2592
2593  unsigned First, Num = Elem.size();
2594  for (First = 0; First != Num; ++First) {
2595    if (!isUndef(Elem[First]))
2596      break;
2597  }
2598  if (First == Num)
2599    return DAG.getUNDEF(VecTy);
2600
2601  if (AllConst &&
2602      llvm::all_of(Consts, [](ConstantInt *CI) { return CI->isZero(); }))
2603    return getZero(dl, VecTy, DAG);
2604
2605  // First try splat if possible.
2606  if (ElemTy == MVT::i16 || ElemTy == MVT::f16) {
2607    bool IsSplat = true;
2608    for (unsigned i = First+1; i != Num; ++i) {
2609      if (Elem[i] == Elem[First] || isUndef(Elem[i]))
2610        continue;
2611      IsSplat = false;
2612      break;
2613    }
2614    if (IsSplat) {
2615      // Legalize the operand of SPLAT_VECTOR
2616      SDValue S = ElemTy == MVT::f16 ? DAG.getBitcast(MVT::i16, Elem[First])
2617                                     : Elem[First];
2618      SDValue Ext = DAG.getZExtOrTrunc(S, dl, MVT::i32);
2619      return DAG.getNode(ISD::SPLAT_VECTOR, dl, VecTy, Ext);
2620    }
2621  }
2622
2623  // Then try constant.
2624  if (AllConst) {
2625    uint64_t Val = 0;
2626    unsigned W = ElemTy.getSizeInBits();
2627    uint64_t Mask = (1ull << W) - 1;
2628    for (unsigned i = 0; i != Num; ++i)
2629      Val = (Val << W) | (Consts[Num-1-i]->getZExtValue() & Mask);
2630    SDValue V0 = DAG.getConstant(Val, dl, MVT::i64);
2631    return DAG.getBitcast(VecTy, V0);
2632  }
2633
2634  // Build two 32-bit vectors and concatenate.
2635  MVT HalfTy = MVT::getVectorVT(ElemTy, Num/2);
2636  SDValue L = (ElemTy == MVT::i32)
2637                ? Elem[0]
2638                : buildVector32(Elem.take_front(Num/2), dl, HalfTy, DAG);
2639  SDValue H = (ElemTy == MVT::i32)
2640                ? Elem[1]
2641                : buildVector32(Elem.drop_front(Num/2), dl, HalfTy, DAG);
2642  return getCombine(H, L, dl, VecTy, DAG);
2643}
2644
2645SDValue
2646HexagonTargetLowering::extractVector(SDValue VecV, SDValue IdxV,
2647                                     const SDLoc &dl, MVT ValTy, MVT ResTy,
2648                                     SelectionDAG &DAG) const {
2649  MVT VecTy = ty(VecV);
2650  assert(!ValTy.isVector() ||
2651         VecTy.getVectorElementType() == ValTy.getVectorElementType());
2652  if (VecTy.getVectorElementType() == MVT::i1)
2653    return extractVectorPred(VecV, IdxV, dl, ValTy, ResTy, DAG);
2654
2655  unsigned VecWidth = VecTy.getSizeInBits();
2656  unsigned ValWidth = ValTy.getSizeInBits();
2657  unsigned ElemWidth = VecTy.getVectorElementType().getSizeInBits();
2658  assert((VecWidth % ElemWidth) == 0);
2659  assert(VecWidth == 32 || VecWidth == 64);
2660
2661  // Cast everything to scalar integer types.
2662  MVT ScalarTy = tyScalar(VecTy);
2663  VecV = DAG.getBitcast(ScalarTy, VecV);
2664
2665  SDValue WidthV = DAG.getConstant(ValWidth, dl, MVT::i32);
2666  SDValue ExtV;
2667
2668  if (auto *IdxN = dyn_cast<ConstantSDNode>(IdxV)) {
2669    unsigned Off = IdxN->getZExtValue() * ElemWidth;
2670    if (VecWidth == 64 && ValWidth == 32) {
2671      assert(Off == 0 || Off == 32);
2672      ExtV = Off == 0 ? LoHalf(VecV, DAG) : HiHalf(VecV, DAG);
2673    } else if (Off == 0 && (ValWidth % 8) == 0) {
2674      ExtV = DAG.getZeroExtendInReg(VecV, dl, tyScalar(ValTy));
2675    } else {
2676      SDValue OffV = DAG.getConstant(Off, dl, MVT::i32);
2677      // The return type of EXTRACTU must be the same as the type of the
2678      // input vector.
2679      ExtV = DAG.getNode(HexagonISD::EXTRACTU, dl, ScalarTy,
2680                         {VecV, WidthV, OffV});
2681    }
2682  } else {
2683    if (ty(IdxV) != MVT::i32)
2684      IdxV = DAG.getZExtOrTrunc(IdxV, dl, MVT::i32);
2685    SDValue OffV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
2686                               DAG.getConstant(ElemWidth, dl, MVT::i32));
2687    ExtV = DAG.getNode(HexagonISD::EXTRACTU, dl, ScalarTy,
2688                       {VecV, WidthV, OffV});
2689  }
2690
2691  // Cast ExtV to the requested result type.
2692  ExtV = DAG.getZExtOrTrunc(ExtV, dl, tyScalar(ResTy));
2693  ExtV = DAG.getBitcast(ResTy, ExtV);
2694  return ExtV;
2695}
2696
2697SDValue
2698HexagonTargetLowering::extractVectorPred(SDValue VecV, SDValue IdxV,
2699                                         const SDLoc &dl, MVT ValTy, MVT ResTy,
2700                                         SelectionDAG &DAG) const {
2701  // Special case for v{8,4,2}i1 (the only boolean vectors legal in Hexagon
2702  // without any coprocessors).
2703  MVT VecTy = ty(VecV);
2704  unsigned VecWidth = VecTy.getSizeInBits();
2705  unsigned ValWidth = ValTy.getSizeInBits();
2706  assert(VecWidth == VecTy.getVectorNumElements() &&
2707         "Vector elements should equal vector width size");
2708  assert(VecWidth == 8 || VecWidth == 4 || VecWidth == 2);
2709
2710  // Check if this is an extract of the lowest bit.
2711  if (auto *IdxN = dyn_cast<ConstantSDNode>(IdxV)) {
2712    // Extracting the lowest bit is a no-op, but it changes the type,
2713    // so it must be kept as an operation to avoid errors related to
2714    // type mismatches.
2715    if (IdxN->isZero() && ValTy.getSizeInBits() == 1)
2716      return DAG.getNode(HexagonISD::TYPECAST, dl, MVT::i1, VecV);
2717  }
2718
2719  // If the value extracted is a single bit, use tstbit.
2720  if (ValWidth == 1) {
2721    SDValue A0 = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32, {VecV}, DAG);
2722    SDValue M0 = DAG.getConstant(8 / VecWidth, dl, MVT::i32);
2723    SDValue I0 = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, M0);
2724    return DAG.getNode(HexagonISD::TSTBIT, dl, MVT::i1, A0, I0);
2725  }
2726
2727  // Each bool vector (v2i1, v4i1, v8i1) always occupies 8 bits in
2728  // a predicate register. The elements of the vector are repeated
2729  // in the register (if necessary) so that the total number is 8.
2730  // The extracted subvector will need to be expanded in such a way.
2731  unsigned Scale = VecWidth / ValWidth;
2732
2733  // Generate (p2d VecV) >> 8*Idx to move the interesting bytes to
2734  // position 0.
2735  assert(ty(IdxV) == MVT::i32);
2736  unsigned VecRep = 8 / VecWidth;
2737  SDValue S0 = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
2738                           DAG.getConstant(8*VecRep, dl, MVT::i32));
2739  SDValue T0 = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, VecV);
2740  SDValue T1 = DAG.getNode(ISD::SRL, dl, MVT::i64, T0, S0);
2741  while (Scale > 1) {
2742    // The longest possible subvector is at most 32 bits, so it is always
2743    // contained in the low subregister.
2744    T1 = LoHalf(T1, DAG);
2745    T1 = expandPredicate(T1, dl, DAG);
2746    Scale /= 2;
2747  }
2748
2749  return DAG.getNode(HexagonISD::D2P, dl, ResTy, T1);
2750}
2751
2752SDValue
2753HexagonTargetLowering::insertVector(SDValue VecV, SDValue ValV, SDValue IdxV,
2754                                    const SDLoc &dl, MVT ValTy,
2755                                    SelectionDAG &DAG) const {
2756  MVT VecTy = ty(VecV);
2757  if (VecTy.getVectorElementType() == MVT::i1)
2758    return insertVectorPred(VecV, ValV, IdxV, dl, ValTy, DAG);
2759
2760  unsigned VecWidth = VecTy.getSizeInBits();
2761  unsigned ValWidth = ValTy.getSizeInBits();
2762  assert(VecWidth == 32 || VecWidth == 64);
2763  assert((VecWidth % ValWidth) == 0);
2764
2765  // Cast everything to scalar integer types.
2766  MVT ScalarTy = MVT::getIntegerVT(VecWidth);
2767  // The actual type of ValV may be different than ValTy (which is related
2768  // to the vector type).
2769  unsigned VW = ty(ValV).getSizeInBits();
2770  ValV = DAG.getBitcast(MVT::getIntegerVT(VW), ValV);
2771  VecV = DAG.getBitcast(ScalarTy, VecV);
2772  if (VW != VecWidth)
2773    ValV = DAG.getAnyExtOrTrunc(ValV, dl, ScalarTy);
2774
2775  SDValue WidthV = DAG.getConstant(ValWidth, dl, MVT::i32);
2776  SDValue InsV;
2777
2778  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(IdxV)) {
2779    unsigned W = C->getZExtValue() * ValWidth;
2780    SDValue OffV = DAG.getConstant(W, dl, MVT::i32);
2781    InsV = DAG.getNode(HexagonISD::INSERT, dl, ScalarTy,
2782                       {VecV, ValV, WidthV, OffV});
2783  } else {
2784    if (ty(IdxV) != MVT::i32)
2785      IdxV = DAG.getZExtOrTrunc(IdxV, dl, MVT::i32);
2786    SDValue OffV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, WidthV);
2787    InsV = DAG.getNode(HexagonISD::INSERT, dl, ScalarTy,
2788                       {VecV, ValV, WidthV, OffV});
2789  }
2790
2791  return DAG.getNode(ISD::BITCAST, dl, VecTy, InsV);
2792}
2793
2794SDValue
2795HexagonTargetLowering::insertVectorPred(SDValue VecV, SDValue ValV,
2796                                        SDValue IdxV, const SDLoc &dl,
2797                                        MVT ValTy, SelectionDAG &DAG) const {
2798  MVT VecTy = ty(VecV);
2799  unsigned VecLen = VecTy.getVectorNumElements();
2800
2801  if (ValTy == MVT::i1) {
2802    SDValue ToReg = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32, {VecV}, DAG);
2803    SDValue Ext = DAG.getSExtOrTrunc(ValV, dl, MVT::i32);
2804    SDValue Width = DAG.getConstant(8 / VecLen, dl, MVT::i32);
2805    SDValue Idx = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, Width);
2806    SDValue Ins =
2807        DAG.getNode(HexagonISD::INSERT, dl, MVT::i32, {ToReg, Ext, Width, Idx});
2808    return getInstr(Hexagon::C2_tfrrp, dl, VecTy, {Ins}, DAG);
2809  }
2810
2811  assert(ValTy.getVectorElementType() == MVT::i1);
2812  SDValue ValR = ValTy.isVector()
2813                     ? DAG.getNode(HexagonISD::P2D, dl, MVT::i64, ValV)
2814                     : DAG.getSExtOrTrunc(ValV, dl, MVT::i64);
2815
2816  unsigned Scale = VecLen / ValTy.getVectorNumElements();
2817  assert(Scale > 1);
2818
2819  for (unsigned R = Scale; R > 1; R /= 2) {
2820    ValR = contractPredicate(ValR, dl, DAG);
2821    ValR = getCombine(DAG.getUNDEF(MVT::i32), ValR, dl, MVT::i64, DAG);
2822  }
2823
2824  SDValue Width = DAG.getConstant(64 / Scale, dl, MVT::i32);
2825  SDValue Idx = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, Width);
2826  SDValue VecR = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, VecV);
2827  SDValue Ins =
2828      DAG.getNode(HexagonISD::INSERT, dl, MVT::i64, {VecR, ValR, Width, Idx});
2829  return DAG.getNode(HexagonISD::D2P, dl, VecTy, Ins);
2830}
2831
2832SDValue
2833HexagonTargetLowering::expandPredicate(SDValue Vec32, const SDLoc &dl,
2834                                       SelectionDAG &DAG) const {
2835  assert(ty(Vec32).getSizeInBits() == 32);
2836  if (isUndef(Vec32))
2837    return DAG.getUNDEF(MVT::i64);
2838  SDValue P = DAG.getBitcast(MVT::v4i8, Vec32);
2839  SDValue X = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i16, P);
2840  return DAG.getBitcast(MVT::i64, X);
2841}
2842
2843SDValue
2844HexagonTargetLowering::contractPredicate(SDValue Vec64, const SDLoc &dl,
2845                                         SelectionDAG &DAG) const {
2846  assert(ty(Vec64).getSizeInBits() == 64);
2847  if (isUndef(Vec64))
2848    return DAG.getUNDEF(MVT::i32);
2849  // Collect even bytes:
2850  SDValue A = DAG.getBitcast(MVT::v8i8, Vec64);
2851  SDValue S = DAG.getVectorShuffle(MVT::v8i8, dl, A, DAG.getUNDEF(MVT::v8i8),
2852                                   {0, 2, 4, 6, 1, 3, 5, 7});
2853  return extractVector(S, DAG.getConstant(0, dl, MVT::i32), dl, MVT::v4i8,
2854                       MVT::i32, DAG);
2855}
2856
2857SDValue
2858HexagonTargetLowering::getZero(const SDLoc &dl, MVT Ty, SelectionDAG &DAG)
2859      const {
2860  if (Ty.isVector()) {
2861    unsigned W = Ty.getSizeInBits();
2862    if (W <= 64)
2863      return DAG.getBitcast(Ty, DAG.getConstant(0, dl, MVT::getIntegerVT(W)));
2864    return DAG.getNode(ISD::SPLAT_VECTOR, dl, Ty, getZero(dl, MVT::i32, DAG));
2865  }
2866
2867  if (Ty.isInteger())
2868    return DAG.getConstant(0, dl, Ty);
2869  if (Ty.isFloatingPoint())
2870    return DAG.getConstantFP(0.0, dl, Ty);
2871  llvm_unreachable("Invalid type for zero");
2872}
2873
2874SDValue
2875HexagonTargetLowering::appendUndef(SDValue Val, MVT ResTy, SelectionDAG &DAG)
2876      const {
2877  MVT ValTy = ty(Val);
2878  assert(ValTy.getVectorElementType() == ResTy.getVectorElementType());
2879
2880  unsigned ValLen = ValTy.getVectorNumElements();
2881  unsigned ResLen = ResTy.getVectorNumElements();
2882  if (ValLen == ResLen)
2883    return Val;
2884
2885  const SDLoc &dl(Val);
2886  assert(ValLen < ResLen);
2887  assert(ResLen % ValLen == 0);
2888
2889  SmallVector<SDValue, 4> Concats = {Val};
2890  for (unsigned i = 1, e = ResLen / ValLen; i < e; ++i)
2891    Concats.push_back(DAG.getUNDEF(ValTy));
2892
2893  return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResTy, Concats);
2894}
2895
2896SDValue
2897HexagonTargetLowering::getCombine(SDValue Hi, SDValue Lo, const SDLoc &dl,
2898                                  MVT ResTy, SelectionDAG &DAG) const {
2899  MVT ElemTy = ty(Hi);
2900  assert(ElemTy == ty(Lo));
2901
2902  if (!ElemTy.isVector()) {
2903    assert(ElemTy.isScalarInteger());
2904    MVT PairTy = MVT::getIntegerVT(2 * ElemTy.getSizeInBits());
2905    SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, dl, PairTy, Lo, Hi);
2906    return DAG.getBitcast(ResTy, Pair);
2907  }
2908
2909  unsigned Width = ElemTy.getSizeInBits();
2910  MVT IntTy = MVT::getIntegerVT(Width);
2911  MVT PairTy = MVT::getIntegerVT(2 * Width);
2912  SDValue Pair =
2913      DAG.getNode(ISD::BUILD_PAIR, dl, PairTy,
2914                  {DAG.getBitcast(IntTy, Lo), DAG.getBitcast(IntTy, Hi)});
2915  return DAG.getBitcast(ResTy, Pair);
2916}
2917
2918SDValue
2919HexagonTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
2920  MVT VecTy = ty(Op);
2921  unsigned BW = VecTy.getSizeInBits();
2922  const SDLoc &dl(Op);
2923  SmallVector<SDValue,8> Ops;
2924  for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i)
2925    Ops.push_back(Op.getOperand(i));
2926
2927  if (BW == 32)
2928    return buildVector32(Ops, dl, VecTy, DAG);
2929  if (BW == 64)
2930    return buildVector64(Ops, dl, VecTy, DAG);
2931
2932  if (VecTy == MVT::v8i1 || VecTy == MVT::v4i1 || VecTy == MVT::v2i1) {
2933    // Check if this is a special case or all-0 or all-1.
2934    bool All0 = true, All1 = true;
2935    for (SDValue P : Ops) {
2936      auto *CN = dyn_cast<ConstantSDNode>(P.getNode());
2937      if (CN == nullptr) {
2938        All0 = All1 = false;
2939        break;
2940      }
2941      uint32_t C = CN->getZExtValue();
2942      All0 &= (C == 0);
2943      All1 &= (C == 1);
2944    }
2945    if (All0)
2946      return DAG.getNode(HexagonISD::PFALSE, dl, VecTy);
2947    if (All1)
2948      return DAG.getNode(HexagonISD::PTRUE, dl, VecTy);
2949
2950    // For each i1 element in the resulting predicate register, put 1
2951    // shifted by the index of the element into a general-purpose register,
2952    // then or them together and transfer it back into a predicate register.
2953    SDValue Rs[8];
2954    SDValue Z = getZero(dl, MVT::i32, DAG);
2955    // Always produce 8 bits, repeat inputs if necessary.
2956    unsigned Rep = 8 / VecTy.getVectorNumElements();
2957    for (unsigned i = 0; i != 8; ++i) {
2958      SDValue S = DAG.getConstant(1ull << i, dl, MVT::i32);
2959      Rs[i] = DAG.getSelect(dl, MVT::i32, Ops[i/Rep], S, Z);
2960    }
2961    for (ArrayRef<SDValue> A(Rs); A.size() != 1; A = A.drop_back(A.size()/2)) {
2962      for (unsigned i = 0, e = A.size()/2; i != e; ++i)
2963        Rs[i] = DAG.getNode(ISD::OR, dl, MVT::i32, Rs[2*i], Rs[2*i+1]);
2964    }
2965    // Move the value directly to a predicate register.
2966    return getInstr(Hexagon::C2_tfrrp, dl, VecTy, {Rs[0]}, DAG);
2967  }
2968
2969  return SDValue();
2970}
2971
2972SDValue
2973HexagonTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
2974                                           SelectionDAG &DAG) const {
2975  MVT VecTy = ty(Op);
2976  const SDLoc &dl(Op);
2977  if (VecTy.getSizeInBits() == 64) {
2978    assert(Op.getNumOperands() == 2);
2979    return getCombine(Op.getOperand(1), Op.getOperand(0), dl, VecTy, DAG);
2980  }
2981
2982  MVT ElemTy = VecTy.getVectorElementType();
2983  if (ElemTy == MVT::i1) {
2984    assert(VecTy == MVT::v2i1 || VecTy == MVT::v4i1 || VecTy == MVT::v8i1);
2985    MVT OpTy = ty(Op.getOperand(0));
2986    // Scale is how many times the operands need to be contracted to match
2987    // the representation in the target register.
2988    unsigned Scale = VecTy.getVectorNumElements() / OpTy.getVectorNumElements();
2989    assert(Scale == Op.getNumOperands() && Scale > 1);
2990
2991    // First, convert all bool vectors to integers, then generate pairwise
2992    // inserts to form values of doubled length. Up until there are only
2993    // two values left to concatenate, all of these values will fit in a
2994    // 32-bit integer, so keep them as i32 to use 32-bit inserts.
2995    SmallVector<SDValue,4> Words[2];
2996    unsigned IdxW = 0;
2997
2998    for (SDValue P : Op.getNode()->op_values()) {
2999      SDValue W = DAG.getNode(HexagonISD::P2D, dl, MVT::i64, P);
3000      for (unsigned R = Scale; R > 1; R /= 2) {
3001        W = contractPredicate(W, dl, DAG);
3002        W = getCombine(DAG.getUNDEF(MVT::i32), W, dl, MVT::i64, DAG);
3003      }
3004      W = LoHalf(W, DAG);
3005      Words[IdxW].push_back(W);
3006    }
3007
3008    while (Scale > 2) {
3009      SDValue WidthV = DAG.getConstant(64 / Scale, dl, MVT::i32);
3010      Words[IdxW ^ 1].clear();
3011
3012      for (unsigned i = 0, e = Words[IdxW].size(); i != e; i += 2) {
3013        SDValue W0 = Words[IdxW][i], W1 = Words[IdxW][i+1];
3014        // Insert W1 into W0 right next to the significant bits of W0.
3015        SDValue T = DAG.getNode(HexagonISD::INSERT, dl, MVT::i32,
3016                                {W0, W1, WidthV, WidthV});
3017        Words[IdxW ^ 1].push_back(T);
3018      }
3019      IdxW ^= 1;
3020      Scale /= 2;
3021    }
3022
3023    // At this point there should only be two words left, and Scale should be 2.
3024    assert(Scale == 2 && Words[IdxW].size() == 2);
3025
3026    SDValue WW = getCombine(Words[IdxW][1], Words[IdxW][0], dl, MVT::i64, DAG);
3027    return DAG.getNode(HexagonISD::D2P, dl, VecTy, WW);
3028  }
3029
3030  return SDValue();
3031}
3032
3033SDValue
3034HexagonTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
3035                                               SelectionDAG &DAG) const {
3036  SDValue Vec = Op.getOperand(0);
3037  MVT ElemTy = ty(Vec).getVectorElementType();
3038  return extractVector(Vec, Op.getOperand(1), SDLoc(Op), ElemTy, ty(Op), DAG);
3039}
3040
3041SDValue
3042HexagonTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
3043                                              SelectionDAG &DAG) const {
3044  return extractVector(Op.getOperand(0), Op.getOperand(1), SDLoc(Op),
3045                       ty(Op), ty(Op), DAG);
3046}
3047
3048SDValue
3049HexagonTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
3050                                              SelectionDAG &DAG) const {
3051  return insertVector(Op.getOperand(0), Op.getOperand(1), Op.getOperand(2),
3052                      SDLoc(Op), ty(Op).getVectorElementType(), DAG);
3053}
3054
3055SDValue
3056HexagonTargetLowering::LowerINSERT_SUBVECTOR(SDValue Op,
3057                                             SelectionDAG &DAG) const {
3058  SDValue ValV = Op.getOperand(1);
3059  return insertVector(Op.getOperand(0), ValV, Op.getOperand(2),
3060                      SDLoc(Op), ty(ValV), DAG);
3061}
3062
3063bool
3064HexagonTargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const {
3065  // Assuming the caller does not have either a signext or zeroext modifier, and
3066  // only one value is accepted, any reasonable truncation is allowed.
3067  if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
3068    return false;
3069
3070  // FIXME: in principle up to 64-bit could be made safe, but it would be very
3071  // fragile at the moment: any support for multiple value returns would be
3072  // liable to disallow tail calls involving i64 -> iN truncation in many cases.
3073  return Ty1->getPrimitiveSizeInBits() <= 32;
3074}
3075
3076SDValue
3077HexagonTargetLowering::LowerLoad(SDValue Op, SelectionDAG &DAG) const {
3078  MVT Ty = ty(Op);
3079  const SDLoc &dl(Op);
3080  LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());
3081  MVT MemTy = LN->getMemoryVT().getSimpleVT();
3082  ISD::LoadExtType ET = LN->getExtensionType();
3083
3084  bool LoadPred = MemTy == MVT::v2i1 || MemTy == MVT::v4i1 || MemTy == MVT::v8i1;
3085  if (LoadPred) {
3086    SDValue NL = DAG.getLoad(
3087        LN->getAddressingMode(), ISD::ZEXTLOAD, MVT::i32, dl, LN->getChain(),
3088        LN->getBasePtr(), LN->getOffset(), LN->getPointerInfo(),
3089        /*MemoryVT*/ MVT::i8, LN->getAlign(), LN->getMemOperand()->getFlags(),
3090        LN->getAAInfo(), LN->getRanges());
3091    LN = cast<LoadSDNode>(NL.getNode());
3092  }
3093
3094  Align ClaimAlign = LN->getAlign();
3095  if (!validateConstPtrAlignment(LN->getBasePtr(), ClaimAlign, dl, DAG))
3096    return replaceMemWithUndef(Op, DAG);
3097
3098  // Call LowerUnalignedLoad for all loads, it recognizes loads that
3099  // don't need extra aligning.
3100  SDValue LU = LowerUnalignedLoad(SDValue(LN, 0), DAG);
3101  if (LoadPred) {
3102    SDValue TP = getInstr(Hexagon::C2_tfrrp, dl, MemTy, {LU}, DAG);
3103    if (ET == ISD::SEXTLOAD) {
3104      TP = DAG.getSExtOrTrunc(TP, dl, Ty);
3105    } else if (ET != ISD::NON_EXTLOAD) {
3106      TP = DAG.getZExtOrTrunc(TP, dl, Ty);
3107    }
3108    SDValue Ch = cast<LoadSDNode>(LU.getNode())->getChain();
3109    return DAG.getMergeValues({TP, Ch}, dl);
3110  }
3111  return LU;
3112}
3113
3114SDValue
3115HexagonTargetLowering::LowerStore(SDValue Op, SelectionDAG &DAG) const {
3116  const SDLoc &dl(Op);
3117  StoreSDNode *SN = cast<StoreSDNode>(Op.getNode());
3118  SDValue Val = SN->getValue();
3119  MVT Ty = ty(Val);
3120
3121  if (Ty == MVT::v2i1 || Ty == MVT::v4i1 || Ty == MVT::v8i1) {
3122    // Store the exact predicate (all bits).
3123    SDValue TR = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32, {Val}, DAG);
3124    SDValue NS = DAG.getTruncStore(SN->getChain(), dl, TR, SN->getBasePtr(),
3125                                   MVT::i8, SN->getMemOperand());
3126    if (SN->isIndexed()) {
3127      NS = DAG.getIndexedStore(NS, dl, SN->getBasePtr(), SN->getOffset(),
3128                               SN->getAddressingMode());
3129    }
3130    SN = cast<StoreSDNode>(NS.getNode());
3131  }
3132
3133  Align ClaimAlign = SN->getAlign();
3134  if (!validateConstPtrAlignment(SN->getBasePtr(), ClaimAlign, dl, DAG))
3135    return replaceMemWithUndef(Op, DAG);
3136
3137  MVT StoreTy = SN->getMemoryVT().getSimpleVT();
3138  Align NeedAlign = Subtarget.getTypeAlignment(StoreTy);
3139  if (ClaimAlign < NeedAlign)
3140    return expandUnalignedStore(SN, DAG);
3141  return SDValue(SN, 0);
3142}
3143
3144SDValue
3145HexagonTargetLowering::LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG)
3146      const {
3147  LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());
3148  MVT LoadTy = ty(Op);
3149  unsigned NeedAlign = Subtarget.getTypeAlignment(LoadTy).value();
3150  unsigned HaveAlign = LN->getAlign().value();
3151  if (HaveAlign >= NeedAlign)
3152    return Op;
3153
3154  const SDLoc &dl(Op);
3155  const DataLayout &DL = DAG.getDataLayout();
3156  LLVMContext &Ctx = *DAG.getContext();
3157
3158  // If the load aligning is disabled or the load can be broken up into two
3159  // smaller legal loads, do the default (target-independent) expansion.
3160  bool DoDefault = false;
3161  // Handle it in the default way if this is an indexed load.
3162  if (!LN->isUnindexed())
3163    DoDefault = true;
3164
3165  if (!AlignLoads) {
3166    if (allowsMemoryAccessForAlignment(Ctx, DL, LN->getMemoryVT(),
3167                                       *LN->getMemOperand()))
3168      return Op;
3169    DoDefault = true;
3170  }
3171  if (!DoDefault && (2 * HaveAlign) == NeedAlign) {
3172    // The PartTy is the equivalent of "getLoadableTypeOfSize(HaveAlign)".
3173    MVT PartTy = HaveAlign <= 8 ? MVT::getIntegerVT(8 * HaveAlign)
3174                                : MVT::getVectorVT(MVT::i8, HaveAlign);
3175    DoDefault =
3176        allowsMemoryAccessForAlignment(Ctx, DL, PartTy, *LN->getMemOperand());
3177  }
3178  if (DoDefault) {
3179    std::pair<SDValue, SDValue> P = expandUnalignedLoad(LN, DAG);
3180    return DAG.getMergeValues({P.first, P.second}, dl);
3181  }
3182
3183  // The code below generates two loads, both aligned as NeedAlign, and
3184  // with the distance of NeedAlign between them. For that to cover the
3185  // bits that need to be loaded (and without overlapping), the size of
3186  // the loads should be equal to NeedAlign. This is true for all loadable
3187  // types, but add an assertion in case something changes in the future.
3188  assert(LoadTy.getSizeInBits() == 8*NeedAlign);
3189
3190  unsigned LoadLen = NeedAlign;
3191  SDValue Base = LN->getBasePtr();
3192  SDValue Chain = LN->getChain();
3193  auto BO = getBaseAndOffset(Base);
3194  unsigned BaseOpc = BO.first.getOpcode();
3195  if (BaseOpc == HexagonISD::VALIGNADDR && BO.second % LoadLen == 0)
3196    return Op;
3197
3198  if (BO.second % LoadLen != 0) {
3199    BO.first = DAG.getNode(ISD::ADD, dl, MVT::i32, BO.first,
3200                           DAG.getConstant(BO.second % LoadLen, dl, MVT::i32));
3201    BO.second -= BO.second % LoadLen;
3202  }
3203  SDValue BaseNoOff = (BaseOpc != HexagonISD::VALIGNADDR)
3204      ? DAG.getNode(HexagonISD::VALIGNADDR, dl, MVT::i32, BO.first,
3205                    DAG.getConstant(NeedAlign, dl, MVT::i32))
3206      : BO.first;
3207  SDValue Base0 =
3208      DAG.getMemBasePlusOffset(BaseNoOff, TypeSize::Fixed(BO.second), dl);
3209  SDValue Base1 = DAG.getMemBasePlusOffset(
3210      BaseNoOff, TypeSize::Fixed(BO.second + LoadLen), dl);
3211
3212  MachineMemOperand *WideMMO = nullptr;
3213  if (MachineMemOperand *MMO = LN->getMemOperand()) {
3214    MachineFunction &MF = DAG.getMachineFunction();
3215    WideMMO = MF.getMachineMemOperand(
3216        MMO->getPointerInfo(), MMO->getFlags(), 2 * LoadLen, Align(LoadLen),
3217        MMO->getAAInfo(), MMO->getRanges(), MMO->getSyncScopeID(),
3218        MMO->getSuccessOrdering(), MMO->getFailureOrdering());
3219  }
3220
3221  SDValue Load0 = DAG.getLoad(LoadTy, dl, Chain, Base0, WideMMO);
3222  SDValue Load1 = DAG.getLoad(LoadTy, dl, Chain, Base1, WideMMO);
3223
3224  SDValue Aligned = DAG.getNode(HexagonISD::VALIGN, dl, LoadTy,
3225                                {Load1, Load0, BaseNoOff.getOperand(0)});
3226  SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3227                                 Load0.getValue(1), Load1.getValue(1));
3228  SDValue M = DAG.getMergeValues({Aligned, NewChain}, dl);
3229  return M;
3230}
3231
3232SDValue
3233HexagonTargetLowering::LowerUAddSubO(SDValue Op, SelectionDAG &DAG) const {
3234  SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
3235  auto *CY = dyn_cast<ConstantSDNode>(Y);
3236  if (!CY)
3237    return SDValue();
3238
3239  const SDLoc &dl(Op);
3240  SDVTList VTs = Op.getNode()->getVTList();
3241  assert(VTs.NumVTs == 2);
3242  assert(VTs.VTs[1] == MVT::i1);
3243  unsigned Opc = Op.getOpcode();
3244
3245  if (CY) {
3246    uint32_t VY = CY->getZExtValue();
3247    assert(VY != 0 && "This should have been folded");
3248    // X +/- 1
3249    if (VY != 1)
3250      return SDValue();
3251
3252    if (Opc == ISD::UADDO) {
3253      SDValue Op = DAG.getNode(ISD::ADD, dl, VTs.VTs[0], {X, Y});
3254      SDValue Ov = DAG.getSetCC(dl, MVT::i1, Op, getZero(dl, ty(Op), DAG),
3255                                ISD::SETEQ);
3256      return DAG.getMergeValues({Op, Ov}, dl);
3257    }
3258    if (Opc == ISD::USUBO) {
3259      SDValue Op = DAG.getNode(ISD::SUB, dl, VTs.VTs[0], {X, Y});
3260      SDValue Ov = DAG.getSetCC(dl, MVT::i1, Op,
3261                                DAG.getConstant(-1, dl, ty(Op)), ISD::SETEQ);
3262      return DAG.getMergeValues({Op, Ov}, dl);
3263    }
3264  }
3265
3266  return SDValue();
3267}
3268
3269SDValue
3270HexagonTargetLowering::LowerAddSubCarry(SDValue Op, SelectionDAG &DAG) const {
3271  const SDLoc &dl(Op);
3272  unsigned Opc = Op.getOpcode();
3273  SDValue X = Op.getOperand(0), Y = Op.getOperand(1), C = Op.getOperand(2);
3274
3275  if (Opc == ISD::ADDCARRY)
3276    return DAG.getNode(HexagonISD::ADDC, dl, Op.getNode()->getVTList(),
3277                       { X, Y, C });
3278
3279  EVT CarryTy = C.getValueType();
3280  SDValue SubC = DAG.getNode(HexagonISD::SUBC, dl, Op.getNode()->getVTList(),
3281                             { X, Y, DAG.getLogicalNOT(dl, C, CarryTy) });
3282  SDValue Out[] = { SubC.getValue(0),
3283                    DAG.getLogicalNOT(dl, SubC.getValue(1), CarryTy) };
3284  return DAG.getMergeValues(Out, dl);
3285}
3286
3287SDValue
3288HexagonTargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
3289  SDValue Chain     = Op.getOperand(0);
3290  SDValue Offset    = Op.getOperand(1);
3291  SDValue Handler   = Op.getOperand(2);
3292  SDLoc dl(Op);
3293  auto PtrVT = getPointerTy(DAG.getDataLayout());
3294
3295  // Mark function as containing a call to EH_RETURN.
3296  HexagonMachineFunctionInfo *FuncInfo =
3297    DAG.getMachineFunction().getInfo<HexagonMachineFunctionInfo>();
3298  FuncInfo->setHasEHReturn();
3299
3300  unsigned OffsetReg = Hexagon::R28;
3301
3302  SDValue StoreAddr =
3303      DAG.getNode(ISD::ADD, dl, PtrVT, DAG.getRegister(Hexagon::R30, PtrVT),
3304                  DAG.getIntPtrConstant(4, dl));
3305  Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo());
3306  Chain = DAG.getCopyToReg(Chain, dl, OffsetReg, Offset);
3307
3308  // Not needed we already use it as explict input to EH_RETURN.
3309  // MF.getRegInfo().addLiveOut(OffsetReg);
3310
3311  return DAG.getNode(HexagonISD::EH_RETURN, dl, MVT::Other, Chain);
3312}
3313
3314SDValue
3315HexagonTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
3316  unsigned Opc = Op.getOpcode();
3317
3318  // Handle INLINEASM first.
3319  if (Opc == ISD::INLINEASM || Opc == ISD::INLINEASM_BR)
3320    return LowerINLINEASM(Op, DAG);
3321
3322  if (isHvxOperation(Op.getNode(), DAG)) {
3323    // If HVX lowering returns nothing, try the default lowering.
3324    if (SDValue V = LowerHvxOperation(Op, DAG))
3325      return V;
3326  }
3327
3328  switch (Opc) {
3329    default:
3330#ifndef NDEBUG
3331      Op.getNode()->dumpr(&DAG);
3332      if (Opc > HexagonISD::OP_BEGIN && Opc < HexagonISD::OP_END)
3333        errs() << "Error: check for a non-legal type in this operation\n";
3334#endif
3335      llvm_unreachable("Should not custom lower this!");
3336    case ISD::CONCAT_VECTORS:       return LowerCONCAT_VECTORS(Op, DAG);
3337    case ISD::INSERT_SUBVECTOR:     return LowerINSERT_SUBVECTOR(Op, DAG);
3338    case ISD::INSERT_VECTOR_ELT:    return LowerINSERT_VECTOR_ELT(Op, DAG);
3339    case ISD::EXTRACT_SUBVECTOR:    return LowerEXTRACT_SUBVECTOR(Op, DAG);
3340    case ISD::EXTRACT_VECTOR_ELT:   return LowerEXTRACT_VECTOR_ELT(Op, DAG);
3341    case ISD::BUILD_VECTOR:         return LowerBUILD_VECTOR(Op, DAG);
3342    case ISD::VECTOR_SHUFFLE:       return LowerVECTOR_SHUFFLE(Op, DAG);
3343    case ISD::BITCAST:              return LowerBITCAST(Op, DAG);
3344    case ISD::LOAD:                 return LowerLoad(Op, DAG);
3345    case ISD::STORE:                return LowerStore(Op, DAG);
3346    case ISD::UADDO:
3347    case ISD::USUBO:                return LowerUAddSubO(Op, DAG);
3348    case ISD::ADDCARRY:
3349    case ISD::SUBCARRY:             return LowerAddSubCarry(Op, DAG);
3350    case ISD::SRA:
3351    case ISD::SHL:
3352    case ISD::SRL:                  return LowerVECTOR_SHIFT(Op, DAG);
3353    case ISD::ROTL:                 return LowerROTL(Op, DAG);
3354    case ISD::ConstantPool:         return LowerConstantPool(Op, DAG);
3355    case ISD::JumpTable:            return LowerJumpTable(Op, DAG);
3356    case ISD::EH_RETURN:            return LowerEH_RETURN(Op, DAG);
3357    case ISD::RETURNADDR:           return LowerRETURNADDR(Op, DAG);
3358    case ISD::FRAMEADDR:            return LowerFRAMEADDR(Op, DAG);
3359    case ISD::GlobalTLSAddress:     return LowerGlobalTLSAddress(Op, DAG);
3360    case ISD::ATOMIC_FENCE:         return LowerATOMIC_FENCE(Op, DAG);
3361    case ISD::GlobalAddress:        return LowerGLOBALADDRESS(Op, DAG);
3362    case ISD::BlockAddress:         return LowerBlockAddress(Op, DAG);
3363    case ISD::GLOBAL_OFFSET_TABLE:  return LowerGLOBAL_OFFSET_TABLE(Op, DAG);
3364    case ISD::VACOPY:               return LowerVACOPY(Op, DAG);
3365    case ISD::VASTART:              return LowerVASTART(Op, DAG);
3366    case ISD::DYNAMIC_STACKALLOC:   return LowerDYNAMIC_STACKALLOC(Op, DAG);
3367    case ISD::SETCC:                return LowerSETCC(Op, DAG);
3368    case ISD::VSELECT:              return LowerVSELECT(Op, DAG);
3369    case ISD::INTRINSIC_WO_CHAIN:   return LowerINTRINSIC_WO_CHAIN(Op, DAG);
3370    case ISD::INTRINSIC_VOID:       return LowerINTRINSIC_VOID(Op, DAG);
3371    case ISD::PREFETCH:             return LowerPREFETCH(Op, DAG);
3372    case ISD::READCYCLECOUNTER:     return LowerREADCYCLECOUNTER(Op, DAG);
3373      break;
3374  }
3375
3376  return SDValue();
3377}
3378
3379void
3380HexagonTargetLowering::LowerOperationWrapper(SDNode *N,
3381                                             SmallVectorImpl<SDValue> &Results,
3382                                             SelectionDAG &DAG) const {
3383  if (isHvxOperation(N, DAG)) {
3384    LowerHvxOperationWrapper(N, Results, DAG);
3385    if (!Results.empty())
3386      return;
3387  }
3388
3389  SDValue Op(N, 0);
3390  unsigned Opc = N->getOpcode();
3391
3392  switch (Opc) {
3393    case HexagonISD::SSAT:
3394    case HexagonISD::USAT:
3395      Results.push_back(opJoin(SplitVectorOp(Op, DAG), SDLoc(Op), DAG));
3396      break;
3397    case ISD::STORE:
3398      // We are only custom-lowering stores to verify the alignment of the
3399      // address if it is a compile-time constant. Since a store can be
3400      // modified during type-legalization (the value being stored may need
3401      // legalization), return empty Results here to indicate that we don't
3402      // really make any changes in the custom lowering.
3403      return;
3404    default:
3405      TargetLowering::LowerOperationWrapper(N, Results, DAG);
3406      break;
3407  }
3408}
3409
3410void
3411HexagonTargetLowering::ReplaceNodeResults(SDNode *N,
3412                                          SmallVectorImpl<SDValue> &Results,
3413                                          SelectionDAG &DAG) const {
3414  if (isHvxOperation(N, DAG)) {
3415    ReplaceHvxNodeResults(N, Results, DAG);
3416    if (!Results.empty())
3417      return;
3418  }
3419
3420  const SDLoc &dl(N);
3421  switch (N->getOpcode()) {
3422    case ISD::SRL:
3423    case ISD::SRA:
3424    case ISD::SHL:
3425      return;
3426    case ISD::BITCAST:
3427      // Handle a bitcast from v8i1 to i8.
3428      if (N->getValueType(0) == MVT::i8) {
3429        if (N->getOperand(0).getValueType() == MVT::v8i1) {
3430          SDValue P = getInstr(Hexagon::C2_tfrpr, dl, MVT::i32,
3431                               N->getOperand(0), DAG);
3432          SDValue T = DAG.getAnyExtOrTrunc(P, dl, MVT::i8);
3433          Results.push_back(T);
3434        }
3435      }
3436      break;
3437  }
3438}
3439
3440SDValue
3441HexagonTargetLowering::PerformDAGCombine(SDNode *N,
3442                                         DAGCombinerInfo &DCI) const {
3443  if (isHvxOperation(N, DCI.DAG)) {
3444    if (SDValue V = PerformHvxDAGCombine(N, DCI))
3445      return V;
3446    return SDValue();
3447  }
3448
3449  SDValue Op(N, 0);
3450  const SDLoc &dl(Op);
3451  unsigned Opc = Op.getOpcode();
3452
3453  if (Opc == ISD::TRUNCATE) {
3454    SDValue Op0 = Op.getOperand(0);
3455    // fold (truncate (build pair x, y)) -> (truncate x) or x
3456    if (Op0.getOpcode() == ISD::BUILD_PAIR) {
3457      EVT TruncTy = Op.getValueType();
3458      SDValue Elem0 = Op0.getOperand(0);
3459      // if we match the low element of the pair, just return it.
3460      if (Elem0.getValueType() == TruncTy)
3461        return Elem0;
3462      // otherwise, if the low part is still too large, apply the truncate.
3463      if (Elem0.getValueType().bitsGT(TruncTy))
3464        return DCI.DAG.getNode(ISD::TRUNCATE, dl, TruncTy, Elem0);
3465    }
3466  }
3467
3468  if (DCI.isBeforeLegalizeOps())
3469    return SDValue();
3470
3471  if (Opc == HexagonISD::P2D) {
3472    SDValue P = Op.getOperand(0);
3473    switch (P.getOpcode()) {
3474    case HexagonISD::PTRUE:
3475      return DCI.DAG.getConstant(-1, dl, ty(Op));
3476    case HexagonISD::PFALSE:
3477      return getZero(dl, ty(Op), DCI.DAG);
3478    default:
3479      break;
3480    }
3481  } else if (Opc == ISD::VSELECT) {
3482    // This is pretty much duplicated in HexagonISelLoweringHVX...
3483    //
3484    // (vselect (xor x, ptrue), v0, v1) -> (vselect x, v1, v0)
3485    SDValue Cond = Op.getOperand(0);
3486    if (Cond->getOpcode() == ISD::XOR) {
3487      SDValue C0 = Cond.getOperand(0), C1 = Cond.getOperand(1);
3488      if (C1->getOpcode() == HexagonISD::PTRUE) {
3489        SDValue VSel = DCI.DAG.getNode(ISD::VSELECT, dl, ty(Op), C0,
3490                                       Op.getOperand(2), Op.getOperand(1));
3491        return VSel;
3492      }
3493    }
3494  } else if (Opc == ISD::TRUNCATE) {
3495    SDValue Op0 = Op.getOperand(0);
3496    // fold (truncate (build pair x, y)) -> (truncate x) or x
3497    if (Op0.getOpcode() == ISD::BUILD_PAIR) {
3498      MVT TruncTy = ty(Op);
3499      SDValue Elem0 = Op0.getOperand(0);
3500      // if we match the low element of the pair, just return it.
3501      if (ty(Elem0) == TruncTy)
3502        return Elem0;
3503      // otherwise, if the low part is still too large, apply the truncate.
3504      if (ty(Elem0).bitsGT(TruncTy))
3505        return DCI.DAG.getNode(ISD::TRUNCATE, dl, TruncTy, Elem0);
3506    }
3507  } else if (Opc == ISD::OR) {
3508    // fold (or (shl xx, s), (zext y)) -> (COMBINE (shl xx, s-32), y)
3509    // if s >= 32
3510    auto fold0 = [&, this](SDValue Op) {
3511      if (ty(Op) != MVT::i64)
3512        return SDValue();
3513      SDValue Shl = Op.getOperand(0);
3514      SDValue Zxt = Op.getOperand(1);
3515      if (Shl.getOpcode() != ISD::SHL)
3516        std::swap(Shl, Zxt);
3517
3518      if (Shl.getOpcode() != ISD::SHL || Zxt.getOpcode() != ISD::ZERO_EXTEND)
3519        return SDValue();
3520
3521      SDValue Z = Zxt.getOperand(0);
3522      auto *Amt = dyn_cast<ConstantSDNode>(Shl.getOperand(1));
3523      if (Amt && Amt->getZExtValue() >= 32 && ty(Z).getSizeInBits() <= 32) {
3524        unsigned A = Amt->getZExtValue();
3525        SDValue S = Shl.getOperand(0);
3526        SDValue T0 = DCI.DAG.getNode(ISD::SHL, dl, ty(S), S,
3527                                     DCI.DAG.getConstant(32 - A, dl, MVT::i32));
3528        SDValue T1 = DCI.DAG.getZExtOrTrunc(T0, dl, MVT::i32);
3529        SDValue T2 = DCI.DAG.getZExtOrTrunc(Z, dl, MVT::i32);
3530        return DCI.DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64, {T1, T2});
3531      }
3532      return SDValue();
3533    };
3534
3535    if (SDValue R = fold0(Op))
3536      return R;
3537  }
3538
3539  return SDValue();
3540}
3541
3542/// Returns relocation base for the given PIC jumptable.
3543SDValue
3544HexagonTargetLowering::getPICJumpTableRelocBase(SDValue Table,
3545                                                SelectionDAG &DAG) const {
3546  int Idx = cast<JumpTableSDNode>(Table)->getIndex();
3547  EVT VT = Table.getValueType();
3548  SDValue T = DAG.getTargetJumpTable(Idx, VT, HexagonII::MO_PCREL);
3549  return DAG.getNode(HexagonISD::AT_PCREL, SDLoc(Table), VT, T);
3550}
3551
3552//===----------------------------------------------------------------------===//
3553// Inline Assembly Support
3554//===----------------------------------------------------------------------===//
3555
3556TargetLowering::ConstraintType
3557HexagonTargetLowering::getConstraintType(StringRef Constraint) const {
3558  if (Constraint.size() == 1) {
3559    switch (Constraint[0]) {
3560      case 'q':
3561      case 'v':
3562        if (Subtarget.useHVXOps())
3563          return C_RegisterClass;
3564        break;
3565      case 'a':
3566        return C_RegisterClass;
3567      default:
3568        break;
3569    }
3570  }
3571  return TargetLowering::getConstraintType(Constraint);
3572}
3573
3574std::pair<unsigned, const TargetRegisterClass*>
3575HexagonTargetLowering::getRegForInlineAsmConstraint(
3576    const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
3577
3578  if (Constraint.size() == 1) {
3579    switch (Constraint[0]) {
3580    case 'r':   // R0-R31
3581      switch (VT.SimpleTy) {
3582      default:
3583        return {0u, nullptr};
3584      case MVT::i1:
3585      case MVT::i8:
3586      case MVT::i16:
3587      case MVT::i32:
3588      case MVT::f32:
3589        return {0u, &Hexagon::IntRegsRegClass};
3590      case MVT::i64:
3591      case MVT::f64:
3592        return {0u, &Hexagon::DoubleRegsRegClass};
3593      }
3594      break;
3595    case 'a': // M0-M1
3596      if (VT != MVT::i32)
3597        return {0u, nullptr};
3598      return {0u, &Hexagon::ModRegsRegClass};
3599    case 'q': // q0-q3
3600      switch (VT.getSizeInBits()) {
3601      default:
3602        return {0u, nullptr};
3603      case 64:
3604      case 128:
3605        return {0u, &Hexagon::HvxQRRegClass};
3606      }
3607      break;
3608    case 'v': // V0-V31
3609      switch (VT.getSizeInBits()) {
3610      default:
3611        return {0u, nullptr};
3612      case 512:
3613        return {0u, &Hexagon::HvxVRRegClass};
3614      case 1024:
3615        if (Subtarget.hasV60Ops() && Subtarget.useHVX128BOps())
3616          return {0u, &Hexagon::HvxVRRegClass};
3617        return {0u, &Hexagon::HvxWRRegClass};
3618      case 2048:
3619        return {0u, &Hexagon::HvxWRRegClass};
3620      }
3621      break;
3622    default:
3623      return {0u, nullptr};
3624    }
3625  }
3626
3627  return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
3628}
3629
3630/// isFPImmLegal - Returns true if the target can instruction select the
3631/// specified FP immediate natively. If false, the legalizer will
3632/// materialize the FP immediate as a load from a constant pool.
3633bool HexagonTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
3634                                         bool ForCodeSize) const {
3635  return true;
3636}
3637
3638/// isLegalAddressingMode - Return true if the addressing mode represented by
3639/// AM is legal for this target, for a load/store of the specified type.
3640bool HexagonTargetLowering::isLegalAddressingMode(const DataLayout &DL,
3641                                                  const AddrMode &AM, Type *Ty,
3642                                                  unsigned AS, Instruction *I) const {
3643  if (Ty->isSized()) {
3644    // When LSR detects uses of the same base address to access different
3645    // types (e.g. unions), it will assume a conservative type for these
3646    // uses:
3647    //   LSR Use: Kind=Address of void in addrspace(4294967295), ...
3648    // The type Ty passed here would then be "void". Skip the alignment
3649    // checks, but do not return false right away, since that confuses
3650    // LSR into crashing.
3651    Align A = DL.getABITypeAlign(Ty);
3652    // The base offset must be a multiple of the alignment.
3653    if (!isAligned(A, AM.BaseOffs))
3654      return false;
3655    // The shifted offset must fit in 11 bits.
3656    if (!isInt<11>(AM.BaseOffs >> Log2(A)))
3657      return false;
3658  }
3659
3660  // No global is ever allowed as a base.
3661  if (AM.BaseGV)
3662    return false;
3663
3664  int Scale = AM.Scale;
3665  if (Scale < 0)
3666    Scale = -Scale;
3667  switch (Scale) {
3668  case 0:  // No scale reg, "r+i", "r", or just "i".
3669    break;
3670  default: // No scaled addressing mode.
3671    return false;
3672  }
3673  return true;
3674}
3675
3676/// Return true if folding a constant offset with the given GlobalAddress is
3677/// legal.  It is frequently not legal in PIC relocation models.
3678bool HexagonTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA)
3679      const {
3680  return HTM.getRelocationModel() == Reloc::Static;
3681}
3682
3683/// isLegalICmpImmediate - Return true if the specified immediate is legal
3684/// icmp immediate, that is the target has icmp instructions which can compare
3685/// a register against the immediate without having to materialize the
3686/// immediate into a register.
3687bool HexagonTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
3688  return Imm >= -512 && Imm <= 511;
3689}
3690
3691/// IsEligibleForTailCallOptimization - Check whether the call is eligible
3692/// for tail call optimization. Targets which want to do tail call
3693/// optimization should implement this function.
3694bool HexagonTargetLowering::IsEligibleForTailCallOptimization(
3695                                 SDValue Callee,
3696                                 CallingConv::ID CalleeCC,
3697                                 bool IsVarArg,
3698                                 bool IsCalleeStructRet,
3699                                 bool IsCallerStructRet,
3700                                 const SmallVectorImpl<ISD::OutputArg> &Outs,
3701                                 const SmallVectorImpl<SDValue> &OutVals,
3702                                 const SmallVectorImpl<ISD::InputArg> &Ins,
3703                                 SelectionDAG& DAG) const {
3704  const Function &CallerF = DAG.getMachineFunction().getFunction();
3705  CallingConv::ID CallerCC = CallerF.getCallingConv();
3706  bool CCMatch = CallerCC == CalleeCC;
3707
3708  // ***************************************************************************
3709  //  Look for obvious safe cases to perform tail call optimization that do not
3710  //  require ABI changes.
3711  // ***************************************************************************
3712
3713  // If this is a tail call via a function pointer, then don't do it!
3714  if (!isa<GlobalAddressSDNode>(Callee) &&
3715      !isa<ExternalSymbolSDNode>(Callee)) {
3716    return false;
3717  }
3718
3719  // Do not optimize if the calling conventions do not match and the conventions
3720  // used are not C or Fast.
3721  if (!CCMatch) {
3722    bool R = (CallerCC == CallingConv::C || CallerCC == CallingConv::Fast);
3723    bool E = (CalleeCC == CallingConv::C || CalleeCC == CallingConv::Fast);
3724    // If R & E, then ok.
3725    if (!R || !E)
3726      return false;
3727  }
3728
3729  // Do not tail call optimize vararg calls.
3730  if (IsVarArg)
3731    return false;
3732
3733  // Also avoid tail call optimization if either caller or callee uses struct
3734  // return semantics.
3735  if (IsCalleeStructRet || IsCallerStructRet)
3736    return false;
3737
3738  // In addition to the cases above, we also disable Tail Call Optimization if
3739  // the calling convention code that at least one outgoing argument needs to
3740  // go on the stack. We cannot check that here because at this point that
3741  // information is not available.
3742  return true;
3743}
3744
3745/// Returns the target specific optimal type for load and store operations as
3746/// a result of memset, memcpy, and memmove lowering.
3747///
3748/// If DstAlign is zero that means it's safe to destination alignment can
3749/// satisfy any constraint. Similarly if SrcAlign is zero it means there isn't
3750/// a need to check it against alignment requirement, probably because the
3751/// source does not need to be loaded. If 'IsMemset' is true, that means it's
3752/// expanding a memset. If 'ZeroMemset' is true, that means it's a memset of
3753/// zero. 'MemcpyStrSrc' indicates whether the memcpy source is constant so it
3754/// does not need to be loaded.  It returns EVT::Other if the type should be
3755/// determined using generic target-independent logic.
3756EVT HexagonTargetLowering::getOptimalMemOpType(
3757    const MemOp &Op, const AttributeList &FuncAttributes) const {
3758  if (Op.size() >= 8 && Op.isAligned(Align(8)))
3759    return MVT::i64;
3760  if (Op.size() >= 4 && Op.isAligned(Align(4)))
3761    return MVT::i32;
3762  if (Op.size() >= 2 && Op.isAligned(Align(2)))
3763    return MVT::i16;
3764  return MVT::Other;
3765}
3766
3767bool HexagonTargetLowering::allowsMemoryAccess(
3768    LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
3769    Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const {
3770  MVT SVT = VT.getSimpleVT();
3771  if (Subtarget.isHVXVectorType(SVT, true))
3772    return allowsHvxMemoryAccess(SVT, Flags, Fast);
3773  return TargetLoweringBase::allowsMemoryAccess(
3774              Context, DL, VT, AddrSpace, Alignment, Flags, Fast);
3775}
3776
3777bool HexagonTargetLowering::allowsMisalignedMemoryAccesses(
3778    EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
3779    unsigned *Fast) const {
3780  MVT SVT = VT.getSimpleVT();
3781  if (Subtarget.isHVXVectorType(SVT, true))
3782    return allowsHvxMisalignedMemoryAccesses(SVT, Flags, Fast);
3783  if (Fast)
3784    *Fast = 0;
3785  return false;
3786}
3787
3788std::pair<const TargetRegisterClass*, uint8_t>
3789HexagonTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
3790      MVT VT) const {
3791  if (Subtarget.isHVXVectorType(VT, true)) {
3792    unsigned BitWidth = VT.getSizeInBits();
3793    unsigned VecWidth = Subtarget.getVectorLength() * 8;
3794
3795    if (VT.getVectorElementType() == MVT::i1)
3796      return std::make_pair(&Hexagon::HvxQRRegClass, 1);
3797    if (BitWidth == VecWidth)
3798      return std::make_pair(&Hexagon::HvxVRRegClass, 1);
3799    assert(BitWidth == 2 * VecWidth);
3800    return std::make_pair(&Hexagon::HvxWRRegClass, 1);
3801  }
3802
3803  return TargetLowering::findRepresentativeClass(TRI, VT);
3804}
3805
3806bool HexagonTargetLowering::shouldReduceLoadWidth(SDNode *Load,
3807      ISD::LoadExtType ExtTy, EVT NewVT) const {
3808  // TODO: This may be worth removing. Check regression tests for diffs.
3809  if (!TargetLoweringBase::shouldReduceLoadWidth(Load, ExtTy, NewVT))
3810    return false;
3811
3812  auto *L = cast<LoadSDNode>(Load);
3813  std::pair<SDValue,int> BO = getBaseAndOffset(L->getBasePtr());
3814  // Small-data object, do not shrink.
3815  if (BO.first.getOpcode() == HexagonISD::CONST32_GP)
3816    return false;
3817  if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(BO.first)) {
3818    auto &HTM = static_cast<const HexagonTargetMachine&>(getTargetMachine());
3819    const auto *GO = dyn_cast_or_null<const GlobalObject>(GA->getGlobal());
3820    return !GO || !HTM.getObjFileLowering()->isGlobalInSmallSection(GO, HTM);
3821  }
3822  return true;
3823}
3824
3825void HexagonTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
3826      SDNode *Node) const {
3827  AdjustHvxInstrPostInstrSelection(MI, Node);
3828}
3829
3830Value *HexagonTargetLowering::emitLoadLinked(IRBuilderBase &Builder,
3831                                             Type *ValueTy, Value *Addr,
3832                                             AtomicOrdering Ord) const {
3833  BasicBlock *BB = Builder.GetInsertBlock();
3834  Module *M = BB->getParent()->getParent();
3835  unsigned SZ = ValueTy->getPrimitiveSizeInBits();
3836  assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic loads supported");
3837  Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_L2_loadw_locked
3838                                   : Intrinsic::hexagon_L4_loadd_locked;
3839  Function *Fn = Intrinsic::getDeclaration(M, IntID);
3840
3841  auto PtrTy = cast<PointerType>(Addr->getType());
3842  PointerType *NewPtrTy =
3843      Builder.getIntNTy(SZ)->getPointerTo(PtrTy->getAddressSpace());
3844  Addr = Builder.CreateBitCast(Addr, NewPtrTy);
3845
3846  Value *Call = Builder.CreateCall(Fn, Addr, "larx");
3847
3848  return Builder.CreateBitCast(Call, ValueTy);
3849}
3850
3851/// Perform a store-conditional operation to Addr. Return the status of the
3852/// store. This should be 0 if the store succeeded, non-zero otherwise.
3853Value *HexagonTargetLowering::emitStoreConditional(IRBuilderBase &Builder,
3854                                                   Value *Val, Value *Addr,
3855                                                   AtomicOrdering Ord) const {
3856  BasicBlock *BB = Builder.GetInsertBlock();
3857  Module *M = BB->getParent()->getParent();
3858  Type *Ty = Val->getType();
3859  unsigned SZ = Ty->getPrimitiveSizeInBits();
3860
3861  Type *CastTy = Builder.getIntNTy(SZ);
3862  assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic stores supported");
3863  Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_S2_storew_locked
3864                                   : Intrinsic::hexagon_S4_stored_locked;
3865  Function *Fn = Intrinsic::getDeclaration(M, IntID);
3866
3867  unsigned AS = Addr->getType()->getPointerAddressSpace();
3868  Addr = Builder.CreateBitCast(Addr, CastTy->getPointerTo(AS));
3869  Val = Builder.CreateBitCast(Val, CastTy);
3870
3871  Value *Call = Builder.CreateCall(Fn, {Addr, Val}, "stcx");
3872  Value *Cmp = Builder.CreateICmpEQ(Call, Builder.getInt32(0), "");
3873  Value *Ext = Builder.CreateZExt(Cmp, Type::getInt32Ty(M->getContext()));
3874  return Ext;
3875}
3876
3877TargetLowering::AtomicExpansionKind
3878HexagonTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
3879  // Do not expand loads and stores that don't exceed 64 bits.
3880  return LI->getType()->getPrimitiveSizeInBits() > 64
3881             ? AtomicExpansionKind::LLOnly
3882             : AtomicExpansionKind::None;
3883}
3884
3885TargetLowering::AtomicExpansionKind
3886HexagonTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
3887  // Do not expand loads and stores that don't exceed 64 bits.
3888  return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() > 64
3889             ? AtomicExpansionKind::Expand
3890             : AtomicExpansionKind::None;
3891}
3892
3893TargetLowering::AtomicExpansionKind
3894HexagonTargetLowering::shouldExpandAtomicCmpXchgInIR(
3895    AtomicCmpXchgInst *AI) const {
3896  return AtomicExpansionKind::LLSC;
3897}
3898