1//===- HexagonSubtarget.cpp - Hexagon Subtarget Information ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Hexagon specific subclass of TargetSubtarget.
10//
11//===----------------------------------------------------------------------===//
12
13#include "HexagonSubtarget.h"
14#include "Hexagon.h"
15#include "HexagonInstrInfo.h"
16#include "HexagonRegisterInfo.h"
17#include "MCTargetDesc/HexagonMCTargetDesc.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/SmallSet.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/CodeGen/MachineInstr.h"
23#include "llvm/CodeGen/MachineOperand.h"
24#include "llvm/CodeGen/MachineScheduler.h"
25#include "llvm/CodeGen/ScheduleDAG.h"
26#include "llvm/CodeGen/ScheduleDAGInstrs.h"
27#include "llvm/IR/IntrinsicsHexagon.h"
28#include "llvm/Support/CommandLine.h"
29#include "llvm/Support/ErrorHandling.h"
30#include "llvm/Target/TargetMachine.h"
31#include <algorithm>
32#include <cassert>
33#include <map>
34#include <optional>
35
36using namespace llvm;
37
38#define DEBUG_TYPE "hexagon-subtarget"
39
40#define GET_SUBTARGETINFO_CTOR
41#define GET_SUBTARGETINFO_TARGET_DESC
42#include "HexagonGenSubtargetInfo.inc"
43
44static cl::opt<bool> EnableBSBSched("enable-bsb-sched", cl::Hidden,
45                                    cl::init(true));
46
47static cl::opt<bool> EnableTCLatencySched("enable-tc-latency-sched", cl::Hidden,
48                                          cl::init(false));
49
50static cl::opt<bool>
51    EnableDotCurSched("enable-cur-sched", cl::Hidden, cl::init(true),
52                      cl::desc("Enable the scheduler to generate .cur"));
53
54static cl::opt<bool>
55    DisableHexagonMISched("disable-hexagon-misched", cl::Hidden,
56                          cl::desc("Disable Hexagon MI Scheduling"));
57
58static cl::opt<bool> EnableSubregLiveness(
59    "hexagon-subreg-liveness", cl::Hidden, cl::init(true),
60    cl::desc("Enable subregister liveness tracking for Hexagon"));
61
62static cl::opt<bool> OverrideLongCalls(
63    "hexagon-long-calls", cl::Hidden,
64    cl::desc("If present, forces/disables the use of long calls"));
65
66static cl::opt<bool>
67    EnablePredicatedCalls("hexagon-pred-calls", cl::Hidden,
68                          cl::desc("Consider calls to be predicable"));
69
70static cl::opt<bool> SchedPredsCloser("sched-preds-closer", cl::Hidden,
71                                      cl::init(true));
72
73static cl::opt<bool> SchedRetvalOptimization("sched-retval-optimization",
74                                             cl::Hidden, cl::init(true));
75
76static cl::opt<bool> EnableCheckBankConflict(
77    "hexagon-check-bank-conflict", cl::Hidden, cl::init(true),
78    cl::desc("Enable checking for cache bank conflicts"));
79
80HexagonSubtarget::HexagonSubtarget(const Triple &TT, StringRef CPU,
81                                   StringRef FS, const TargetMachine &TM)
82    : HexagonGenSubtargetInfo(TT, CPU, /*TuneCPU*/ CPU, FS),
83      OptLevel(TM.getOptLevel()),
84      CPUString(std::string(Hexagon_MC::selectHexagonCPU(CPU))),
85      TargetTriple(TT), InstrInfo(initializeSubtargetDependencies(CPU, FS)),
86      RegInfo(getHwMode()), TLInfo(TM, *this),
87      InstrItins(getInstrItineraryForCPU(CPUString)) {
88  Hexagon_MC::addArchSubtarget(this, FS);
89  // Beware of the default constructor of InstrItineraryData: it will
90  // reset all members to 0.
91  assert(InstrItins.Itineraries != nullptr && "InstrItins not initialized");
92}
93
94HexagonSubtarget &
95HexagonSubtarget::initializeSubtargetDependencies(StringRef CPU, StringRef FS) {
96  std::optional<Hexagon::ArchEnum> ArchVer = Hexagon::getCpu(CPUString);
97  if (ArchVer)
98    HexagonArchVersion = *ArchVer;
99  else
100    llvm_unreachable("Unrecognized Hexagon processor version");
101
102  UseHVX128BOps = false;
103  UseHVX64BOps = false;
104  UseAudioOps = false;
105  UseLongCalls = false;
106
107  SubtargetFeatures Features(FS);
108
109  // Turn on QFloat if the HVX version is v68+.
110  // The function ParseSubtargetFeatures will set feature bits and initialize
111  // subtarget's variables all in one, so there isn't a good way to preprocess
112  // the feature string, other than by tinkering with it directly.
113  auto IsQFloatFS = [](StringRef F) {
114    return F == "+hvx-qfloat" || F == "-hvx-qfloat";
115  };
116  if (!llvm::count_if(Features.getFeatures(), IsQFloatFS)) {
117    auto getHvxVersion = [&Features](StringRef FS) -> StringRef {
118      for (StringRef F : llvm::reverse(Features.getFeatures())) {
119        if (F.startswith("+hvxv"))
120          return F;
121      }
122      for (StringRef F : llvm::reverse(Features.getFeatures())) {
123        if (F == "-hvx")
124          return StringRef();
125        if (F.startswith("+hvx") || F == "-hvx")
126          return F.take_front(4);  // Return "+hvx" or "-hvx".
127      }
128      return StringRef();
129    };
130
131    bool AddQFloat = false;
132    StringRef HvxVer = getHvxVersion(FS);
133    if (HvxVer.startswith("+hvxv")) {
134      int Ver = 0;
135      if (!HvxVer.drop_front(5).consumeInteger(10, Ver) && Ver >= 68)
136        AddQFloat = true;
137    } else if (HvxVer == "+hvx") {
138      if (hasV68Ops())
139        AddQFloat = true;
140    }
141
142    if (AddQFloat)
143      Features.AddFeature("+hvx-qfloat");
144  }
145
146  std::string FeatureString = Features.getString();
147  ParseSubtargetFeatures(CPUString, /*TuneCPU*/ CPUString, FeatureString);
148
149  if (useHVXV68Ops())
150    UseHVXFloatingPoint = UseHVXIEEEFPOps || UseHVXQFloatOps;
151
152  if (UseHVXQFloatOps && UseHVXIEEEFPOps && UseHVXFloatingPoint)
153    LLVM_DEBUG(
154        dbgs() << "Behavior is undefined for simultaneous qfloat and ieee hvx codegen...");
155
156  if (OverrideLongCalls.getPosition())
157    UseLongCalls = OverrideLongCalls;
158
159  UseBSBScheduling = hasV60Ops() && EnableBSBSched;
160
161  if (isTinyCore()) {
162    // Tiny core has a single thread, so back-to-back scheduling is enabled by
163    // default.
164    if (!EnableBSBSched.getPosition())
165      UseBSBScheduling = false;
166  }
167
168  FeatureBitset FeatureBits = getFeatureBits();
169  if (HexagonDisableDuplex)
170    setFeatureBits(FeatureBits.reset(Hexagon::FeatureDuplex));
171  setFeatureBits(Hexagon_MC::completeHVXFeatures(FeatureBits));
172
173  return *this;
174}
175
176bool HexagonSubtarget::isHVXElementType(MVT Ty, bool IncludeBool) const {
177  if (!useHVXOps())
178    return false;
179  if (Ty.isVector())
180    Ty = Ty.getVectorElementType();
181  if (IncludeBool && Ty == MVT::i1)
182    return true;
183  ArrayRef<MVT> ElemTypes = getHVXElementTypes();
184  return llvm::is_contained(ElemTypes, Ty);
185}
186
187bool HexagonSubtarget::isHVXVectorType(EVT VecTy, bool IncludeBool) const {
188  if (!VecTy.isSimple())
189    return false;
190  if (!VecTy.isVector() || !useHVXOps() || VecTy.isScalableVector())
191    return false;
192  MVT ElemTy = VecTy.getSimpleVT().getVectorElementType();
193  if (!IncludeBool && ElemTy == MVT::i1)
194    return false;
195
196  unsigned HwLen = getVectorLength();
197  unsigned NumElems = VecTy.getVectorNumElements();
198  ArrayRef<MVT> ElemTypes = getHVXElementTypes();
199
200  if (IncludeBool && ElemTy == MVT::i1) {
201    // Boolean HVX vector types are formed from regular HVX vector types
202    // by replacing the element type with i1.
203    for (MVT T : ElemTypes)
204      if (NumElems * T.getSizeInBits() == 8 * HwLen)
205        return true;
206    return false;
207  }
208
209  unsigned VecWidth = VecTy.getSizeInBits();
210  if (VecWidth != 8 * HwLen && VecWidth != 16 * HwLen)
211    return false;
212  return llvm::is_contained(ElemTypes, ElemTy);
213}
214
215bool HexagonSubtarget::isTypeForHVX(Type *VecTy, bool IncludeBool) const {
216  if (!VecTy->isVectorTy() || isa<ScalableVectorType>(VecTy))
217    return false;
218  // Avoid types like <2 x i32*>.
219  Type *ScalTy = VecTy->getScalarType();
220  if (!ScalTy->isIntegerTy() &&
221      !(ScalTy->isFloatingPointTy() && useHVXFloatingPoint()))
222    return false;
223  // The given type may be something like <17 x i32>, which is not MVT,
224  // but can be represented as (non-simple) EVT.
225  EVT Ty = EVT::getEVT(VecTy, /*HandleUnknown*/false);
226  if (!Ty.getVectorElementType().isSimple())
227    return false;
228
229  auto isHvxTy = [this, IncludeBool](MVT SimpleTy) {
230    if (isHVXVectorType(SimpleTy, IncludeBool))
231      return true;
232    auto Action = getTargetLowering()->getPreferredVectorAction(SimpleTy);
233    return Action == TargetLoweringBase::TypeWidenVector;
234  };
235
236  // Round up EVT to have power-of-2 elements, and keep checking if it
237  // qualifies for HVX, dividing it in half after each step.
238  MVT ElemTy = Ty.getVectorElementType().getSimpleVT();
239  unsigned VecLen = PowerOf2Ceil(Ty.getVectorNumElements());
240  while (VecLen > 1) {
241    MVT SimpleTy = MVT::getVectorVT(ElemTy, VecLen);
242    if (SimpleTy.isValid() && isHvxTy(SimpleTy))
243      return true;
244    VecLen /= 2;
245  }
246
247  return false;
248}
249
250void HexagonSubtarget::UsrOverflowMutation::apply(ScheduleDAGInstrs *DAG) {
251  for (SUnit &SU : DAG->SUnits) {
252    if (!SU.isInstr())
253      continue;
254    SmallVector<SDep, 4> Erase;
255    for (auto &D : SU.Preds)
256      if (D.getKind() == SDep::Output && D.getReg() == Hexagon::USR_OVF)
257        Erase.push_back(D);
258    for (auto &E : Erase)
259      SU.removePred(E);
260  }
261}
262
263void HexagonSubtarget::HVXMemLatencyMutation::apply(ScheduleDAGInstrs *DAG) {
264  for (SUnit &SU : DAG->SUnits) {
265    // Update the latency of chain edges between v60 vector load or store
266    // instructions to be 1. These instruction cannot be scheduled in the
267    // same packet.
268    MachineInstr &MI1 = *SU.getInstr();
269    auto *QII = static_cast<const HexagonInstrInfo*>(DAG->TII);
270    bool IsStoreMI1 = MI1.mayStore();
271    bool IsLoadMI1 = MI1.mayLoad();
272    if (!QII->isHVXVec(MI1) || !(IsStoreMI1 || IsLoadMI1))
273      continue;
274    for (SDep &SI : SU.Succs) {
275      if (SI.getKind() != SDep::Order || SI.getLatency() != 0)
276        continue;
277      MachineInstr &MI2 = *SI.getSUnit()->getInstr();
278      if (!QII->isHVXVec(MI2))
279        continue;
280      if ((IsStoreMI1 && MI2.mayStore()) || (IsLoadMI1 && MI2.mayLoad())) {
281        SI.setLatency(1);
282        SU.setHeightDirty();
283        // Change the dependence in the opposite direction too.
284        for (SDep &PI : SI.getSUnit()->Preds) {
285          if (PI.getSUnit() != &SU || PI.getKind() != SDep::Order)
286            continue;
287          PI.setLatency(1);
288          SI.getSUnit()->setDepthDirty();
289        }
290      }
291    }
292  }
293}
294
295// Check if a call and subsequent A2_tfrpi instructions should maintain
296// scheduling affinity. We are looking for the TFRI to be consumed in
297// the next instruction. This should help reduce the instances of
298// double register pairs being allocated and scheduled before a call
299// when not used until after the call. This situation is exacerbated
300// by the fact that we allocate the pair from the callee saves list,
301// leading to excess spills and restores.
302bool HexagonSubtarget::CallMutation::shouldTFRICallBind(
303      const HexagonInstrInfo &HII, const SUnit &Inst1,
304      const SUnit &Inst2) const {
305  if (Inst1.getInstr()->getOpcode() != Hexagon::A2_tfrpi)
306    return false;
307
308  // TypeXTYPE are 64 bit operations.
309  unsigned Type = HII.getType(*Inst2.getInstr());
310  return Type == HexagonII::TypeS_2op || Type == HexagonII::TypeS_3op ||
311         Type == HexagonII::TypeALU64 || Type == HexagonII::TypeM;
312}
313
314void HexagonSubtarget::CallMutation::apply(ScheduleDAGInstrs *DAGInstrs) {
315  ScheduleDAGMI *DAG = static_cast<ScheduleDAGMI*>(DAGInstrs);
316  SUnit* LastSequentialCall = nullptr;
317  // Map from virtual register to physical register from the copy.
318  DenseMap<unsigned, unsigned> VRegHoldingReg;
319  // Map from the physical register to the instruction that uses virtual
320  // register. This is used to create the barrier edge.
321  DenseMap<unsigned, SUnit *> LastVRegUse;
322  auto &TRI = *DAG->MF.getSubtarget().getRegisterInfo();
323  auto &HII = *DAG->MF.getSubtarget<HexagonSubtarget>().getInstrInfo();
324
325  // Currently we only catch the situation when compare gets scheduled
326  // before preceding call.
327  for (unsigned su = 0, e = DAG->SUnits.size(); su != e; ++su) {
328    // Remember the call.
329    if (DAG->SUnits[su].getInstr()->isCall())
330      LastSequentialCall = &DAG->SUnits[su];
331    // Look for a compare that defines a predicate.
332    else if (DAG->SUnits[su].getInstr()->isCompare() && LastSequentialCall)
333      DAG->addEdge(&DAG->SUnits[su], SDep(LastSequentialCall, SDep::Barrier));
334    // Look for call and tfri* instructions.
335    else if (SchedPredsCloser && LastSequentialCall && su > 1 && su < e-1 &&
336             shouldTFRICallBind(HII, DAG->SUnits[su], DAG->SUnits[su+1]))
337      DAG->addEdge(&DAG->SUnits[su], SDep(&DAG->SUnits[su-1], SDep::Barrier));
338    // Prevent redundant register copies due to reads and writes of physical
339    // registers. The original motivation for this was the code generated
340    // between two calls, which are caused both the return value and the
341    // argument for the next call being in %r0.
342    // Example:
343    //   1: <call1>
344    //   2: %vreg = COPY %r0
345    //   3: <use of %vreg>
346    //   4: %r0 = ...
347    //   5: <call2>
348    // The scheduler would often swap 3 and 4, so an additional register is
349    // needed. This code inserts a Barrier dependence between 3 & 4 to prevent
350    // this.
351    // The code below checks for all the physical registers, not just R0/D0/V0.
352    else if (SchedRetvalOptimization) {
353      const MachineInstr *MI = DAG->SUnits[su].getInstr();
354      if (MI->isCopy() && MI->getOperand(1).getReg().isPhysical()) {
355        // %vregX = COPY %r0
356        VRegHoldingReg[MI->getOperand(0).getReg()] = MI->getOperand(1).getReg();
357        LastVRegUse.erase(MI->getOperand(1).getReg());
358      } else {
359        for (const MachineOperand &MO : MI->operands()) {
360          if (!MO.isReg())
361            continue;
362          if (MO.isUse() && !MI->isCopy() &&
363              VRegHoldingReg.count(MO.getReg())) {
364            // <use of %vregX>
365            LastVRegUse[VRegHoldingReg[MO.getReg()]] = &DAG->SUnits[su];
366          } else if (MO.isDef() && MO.getReg().isPhysical()) {
367            for (MCRegAliasIterator AI(MO.getReg(), &TRI, true); AI.isValid();
368                 ++AI) {
369              if (LastVRegUse.count(*AI) &&
370                  LastVRegUse[*AI] != &DAG->SUnits[su])
371                // %r0 = ...
372                DAG->addEdge(&DAG->SUnits[su], SDep(LastVRegUse[*AI], SDep::Barrier));
373              LastVRegUse.erase(*AI);
374            }
375          }
376        }
377      }
378    }
379  }
380}
381
382void HexagonSubtarget::BankConflictMutation::apply(ScheduleDAGInstrs *DAG) {
383  if (!EnableCheckBankConflict)
384    return;
385
386  const auto &HII = static_cast<const HexagonInstrInfo&>(*DAG->TII);
387
388  // Create artificial edges between loads that could likely cause a bank
389  // conflict. Since such loads would normally not have any dependency
390  // between them, we cannot rely on existing edges.
391  for (unsigned i = 0, e = DAG->SUnits.size(); i != e; ++i) {
392    SUnit &S0 = DAG->SUnits[i];
393    MachineInstr &L0 = *S0.getInstr();
394    if (!L0.mayLoad() || L0.mayStore() ||
395        HII.getAddrMode(L0) != HexagonII::BaseImmOffset)
396      continue;
397    int64_t Offset0;
398    unsigned Size0;
399    MachineOperand *BaseOp0 = HII.getBaseAndOffset(L0, Offset0, Size0);
400    // Is the access size is longer than the L1 cache line, skip the check.
401    if (BaseOp0 == nullptr || !BaseOp0->isReg() || Size0 >= 32)
402      continue;
403    // Scan only up to 32 instructions ahead (to avoid n^2 complexity).
404    for (unsigned j = i+1, m = std::min(i+32, e); j != m; ++j) {
405      SUnit &S1 = DAG->SUnits[j];
406      MachineInstr &L1 = *S1.getInstr();
407      if (!L1.mayLoad() || L1.mayStore() ||
408          HII.getAddrMode(L1) != HexagonII::BaseImmOffset)
409        continue;
410      int64_t Offset1;
411      unsigned Size1;
412      MachineOperand *BaseOp1 = HII.getBaseAndOffset(L1, Offset1, Size1);
413      if (BaseOp1 == nullptr || !BaseOp1->isReg() || Size1 >= 32 ||
414          BaseOp0->getReg() != BaseOp1->getReg())
415        continue;
416      // Check bits 3 and 4 of the offset: if they differ, a bank conflict
417      // is unlikely.
418      if (((Offset0 ^ Offset1) & 0x18) != 0)
419        continue;
420      // Bits 3 and 4 are the same, add an artificial edge and set extra
421      // latency.
422      SDep A(&S0, SDep::Artificial);
423      A.setLatency(1);
424      S1.addPred(A, true);
425    }
426  }
427}
428
429/// Enable use of alias analysis during code generation (during MI
430/// scheduling, DAGCombine, etc.).
431bool HexagonSubtarget::useAA() const {
432  if (OptLevel != CodeGenOpt::None)
433    return true;
434  return false;
435}
436
437/// Perform target specific adjustments to the latency of a schedule
438/// dependency.
439void HexagonSubtarget::adjustSchedDependency(SUnit *Src, int SrcOpIdx,
440                                             SUnit *Dst, int DstOpIdx,
441                                             SDep &Dep) const {
442  if (!Src->isInstr() || !Dst->isInstr())
443    return;
444
445  MachineInstr *SrcInst = Src->getInstr();
446  MachineInstr *DstInst = Dst->getInstr();
447  const HexagonInstrInfo *QII = getInstrInfo();
448
449  // Instructions with .new operands have zero latency.
450  SmallSet<SUnit *, 4> ExclSrc;
451  SmallSet<SUnit *, 4> ExclDst;
452  if (QII->canExecuteInBundle(*SrcInst, *DstInst) &&
453      isBestZeroLatency(Src, Dst, QII, ExclSrc, ExclDst)) {
454    Dep.setLatency(0);
455    return;
456  }
457
458  // Set the latency for a copy to zero since we hope that is will get
459  // removed.
460  if (DstInst->isCopy())
461    Dep.setLatency(0);
462
463  // If it's a REG_SEQUENCE/COPY, use its destination instruction to determine
464  // the correct latency.
465  // If there are multiple uses of the def of COPY/REG_SEQUENCE, set the latency
466  // only if the latencies on all the uses are equal, otherwise set it to
467  // default.
468  if ((DstInst->isRegSequence() || DstInst->isCopy())) {
469    Register DReg = DstInst->getOperand(0).getReg();
470    int DLatency = -1;
471    for (const auto &DDep : Dst->Succs) {
472      MachineInstr *DDst = DDep.getSUnit()->getInstr();
473      int UseIdx = -1;
474      for (unsigned OpNum = 0; OpNum < DDst->getNumOperands(); OpNum++) {
475        const MachineOperand &MO = DDst->getOperand(OpNum);
476        if (MO.isReg() && MO.getReg() && MO.isUse() && MO.getReg() == DReg) {
477          UseIdx = OpNum;
478          break;
479        }
480      }
481
482      if (UseIdx == -1)
483        continue;
484
485      int Latency = (InstrInfo.getOperandLatency(&InstrItins, *SrcInst, 0,
486                                                 *DDst, UseIdx));
487      // Set DLatency for the first time.
488      DLatency = (DLatency == -1) ? Latency : DLatency;
489
490      // For multiple uses, if the Latency is different across uses, reset
491      // DLatency.
492      if (DLatency != Latency) {
493        DLatency = -1;
494        break;
495      }
496    }
497
498    DLatency = std::max(DLatency, 0);
499    Dep.setLatency((unsigned)DLatency);
500  }
501
502  // Try to schedule uses near definitions to generate .cur.
503  ExclSrc.clear();
504  ExclDst.clear();
505  if (EnableDotCurSched && QII->isToBeScheduledASAP(*SrcInst, *DstInst) &&
506      isBestZeroLatency(Src, Dst, QII, ExclSrc, ExclDst)) {
507    Dep.setLatency(0);
508    return;
509  }
510  int Latency = Dep.getLatency();
511  bool IsArtificial = Dep.isArtificial();
512  Latency = updateLatency(*SrcInst, *DstInst, IsArtificial, Latency);
513  Dep.setLatency(Latency);
514}
515
516void HexagonSubtarget::getPostRAMutations(
517    std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations) const {
518  Mutations.push_back(std::make_unique<UsrOverflowMutation>());
519  Mutations.push_back(std::make_unique<HVXMemLatencyMutation>());
520  Mutations.push_back(std::make_unique<BankConflictMutation>());
521}
522
523void HexagonSubtarget::getSMSMutations(
524    std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations) const {
525  Mutations.push_back(std::make_unique<UsrOverflowMutation>());
526  Mutations.push_back(std::make_unique<HVXMemLatencyMutation>());
527}
528
529// Pin the vtable to this file.
530void HexagonSubtarget::anchor() {}
531
532bool HexagonSubtarget::enableMachineScheduler() const {
533  if (DisableHexagonMISched.getNumOccurrences())
534    return !DisableHexagonMISched;
535  return true;
536}
537
538bool HexagonSubtarget::usePredicatedCalls() const {
539  return EnablePredicatedCalls;
540}
541
542int HexagonSubtarget::updateLatency(MachineInstr &SrcInst,
543                                    MachineInstr &DstInst, bool IsArtificial,
544                                    int Latency) const {
545  if (IsArtificial)
546    return 1;
547  if (!hasV60Ops())
548    return Latency;
549
550  auto &QII = static_cast<const HexagonInstrInfo &>(*getInstrInfo());
551  // BSB scheduling.
552  if (QII.isHVXVec(SrcInst) || useBSBScheduling())
553    Latency = (Latency + 1) >> 1;
554  return Latency;
555}
556
557void HexagonSubtarget::restoreLatency(SUnit *Src, SUnit *Dst) const {
558  MachineInstr *SrcI = Src->getInstr();
559  for (auto &I : Src->Succs) {
560    if (!I.isAssignedRegDep() || I.getSUnit() != Dst)
561      continue;
562    Register DepR = I.getReg();
563    int DefIdx = -1;
564    for (unsigned OpNum = 0; OpNum < SrcI->getNumOperands(); OpNum++) {
565      const MachineOperand &MO = SrcI->getOperand(OpNum);
566      bool IsSameOrSubReg = false;
567      if (MO.isReg()) {
568        Register MOReg = MO.getReg();
569        if (DepR.isVirtual()) {
570          IsSameOrSubReg = (MOReg == DepR);
571        } else {
572          IsSameOrSubReg = getRegisterInfo()->isSubRegisterEq(DepR, MOReg);
573        }
574        if (MO.isDef() && IsSameOrSubReg)
575          DefIdx = OpNum;
576      }
577    }
578    assert(DefIdx >= 0 && "Def Reg not found in Src MI");
579    MachineInstr *DstI = Dst->getInstr();
580    SDep T = I;
581    for (unsigned OpNum = 0; OpNum < DstI->getNumOperands(); OpNum++) {
582      const MachineOperand &MO = DstI->getOperand(OpNum);
583      if (MO.isReg() && MO.isUse() && MO.getReg() == DepR) {
584        int Latency = (InstrInfo.getOperandLatency(&InstrItins, *SrcI,
585                                                   DefIdx, *DstI, OpNum));
586
587        // For some instructions (ex: COPY), we might end up with < 0 latency
588        // as they don't have any Itinerary class associated with them.
589        Latency = std::max(Latency, 0);
590        bool IsArtificial = I.isArtificial();
591        Latency = updateLatency(*SrcI, *DstI, IsArtificial, Latency);
592        I.setLatency(Latency);
593      }
594    }
595
596    // Update the latency of opposite edge too.
597    T.setSUnit(Src);
598    auto F = find(Dst->Preds, T);
599    assert(F != Dst->Preds.end());
600    F->setLatency(I.getLatency());
601  }
602}
603
604/// Change the latency between the two SUnits.
605void HexagonSubtarget::changeLatency(SUnit *Src, SUnit *Dst, unsigned Lat)
606      const {
607  for (auto &I : Src->Succs) {
608    if (!I.isAssignedRegDep() || I.getSUnit() != Dst)
609      continue;
610    SDep T = I;
611    I.setLatency(Lat);
612
613    // Update the latency of opposite edge too.
614    T.setSUnit(Src);
615    auto F = find(Dst->Preds, T);
616    assert(F != Dst->Preds.end());
617    F->setLatency(Lat);
618  }
619}
620
621/// If the SUnit has a zero latency edge, return the other SUnit.
622static SUnit *getZeroLatency(SUnit *N, SmallVector<SDep, 4> &Deps) {
623  for (auto &I : Deps)
624    if (I.isAssignedRegDep() && I.getLatency() == 0 &&
625        !I.getSUnit()->getInstr()->isPseudo())
626      return I.getSUnit();
627  return nullptr;
628}
629
630// Return true if these are the best two instructions to schedule
631// together with a zero latency. Only one dependence should have a zero
632// latency. If there are multiple choices, choose the best, and change
633// the others, if needed.
634bool HexagonSubtarget::isBestZeroLatency(SUnit *Src, SUnit *Dst,
635      const HexagonInstrInfo *TII, SmallSet<SUnit*, 4> &ExclSrc,
636      SmallSet<SUnit*, 4> &ExclDst) const {
637  MachineInstr &SrcInst = *Src->getInstr();
638  MachineInstr &DstInst = *Dst->getInstr();
639
640  // Ignore Boundary SU nodes as these have null instructions.
641  if (Dst->isBoundaryNode())
642    return false;
643
644  if (SrcInst.isPHI() || DstInst.isPHI())
645    return false;
646
647  if (!TII->isToBeScheduledASAP(SrcInst, DstInst) &&
648      !TII->canExecuteInBundle(SrcInst, DstInst))
649    return false;
650
651  // The architecture doesn't allow three dependent instructions in the same
652  // packet. So, if the destination has a zero latency successor, then it's
653  // not a candidate for a zero latency predecessor.
654  if (getZeroLatency(Dst, Dst->Succs) != nullptr)
655    return false;
656
657  // Check if the Dst instruction is the best candidate first.
658  SUnit *Best = nullptr;
659  SUnit *DstBest = nullptr;
660  SUnit *SrcBest = getZeroLatency(Dst, Dst->Preds);
661  if (SrcBest == nullptr || Src->NodeNum >= SrcBest->NodeNum) {
662    // Check that Src doesn't have a better candidate.
663    DstBest = getZeroLatency(Src, Src->Succs);
664    if (DstBest == nullptr || Dst->NodeNum <= DstBest->NodeNum)
665      Best = Dst;
666  }
667  if (Best != Dst)
668    return false;
669
670  // The caller frequently adds the same dependence twice. If so, then
671  // return true for this case too.
672  if ((Src == SrcBest && Dst == DstBest ) ||
673      (SrcBest == nullptr && Dst == DstBest) ||
674      (Src == SrcBest && Dst == nullptr))
675    return true;
676
677  // Reassign the latency for the previous bests, which requires setting
678  // the dependence edge in both directions.
679  if (SrcBest != nullptr) {
680    if (!hasV60Ops())
681      changeLatency(SrcBest, Dst, 1);
682    else
683      restoreLatency(SrcBest, Dst);
684  }
685  if (DstBest != nullptr) {
686    if (!hasV60Ops())
687      changeLatency(Src, DstBest, 1);
688    else
689      restoreLatency(Src, DstBest);
690  }
691
692  // Attempt to find another opprotunity for zero latency in a different
693  // dependence.
694  if (SrcBest && DstBest)
695    // If there is an edge from SrcBest to DstBst, then try to change that
696    // to 0 now.
697    changeLatency(SrcBest, DstBest, 0);
698  else if (DstBest) {
699    // Check if the previous best destination instruction has a new zero
700    // latency dependence opportunity.
701    ExclSrc.insert(Src);
702    for (auto &I : DstBest->Preds)
703      if (ExclSrc.count(I.getSUnit()) == 0 &&
704          isBestZeroLatency(I.getSUnit(), DstBest, TII, ExclSrc, ExclDst))
705        changeLatency(I.getSUnit(), DstBest, 0);
706  } else if (SrcBest) {
707    // Check if previous best source instruction has a new zero latency
708    // dependence opportunity.
709    ExclDst.insert(Dst);
710    for (auto &I : SrcBest->Succs)
711      if (ExclDst.count(I.getSUnit()) == 0 &&
712          isBestZeroLatency(SrcBest, I.getSUnit(), TII, ExclSrc, ExclDst))
713        changeLatency(SrcBest, I.getSUnit(), 0);
714  }
715
716  return true;
717}
718
719unsigned HexagonSubtarget::getL1CacheLineSize() const {
720  return 32;
721}
722
723unsigned HexagonSubtarget::getL1PrefetchDistance() const {
724  return 32;
725}
726
727bool HexagonSubtarget::enableSubRegLiveness() const {
728  return EnableSubregLiveness;
729}
730
731Intrinsic::ID HexagonSubtarget::getIntrinsicId(unsigned Opc) const {
732  struct Scalar {
733    unsigned Opcode;
734    Intrinsic::ID IntId;
735  };
736  struct Hvx {
737    unsigned Opcode;
738    Intrinsic::ID Int64Id, Int128Id;
739  };
740
741  static Scalar ScalarInts[] = {
742#define GET_SCALAR_INTRINSICS
743#include "HexagonDepInstrIntrinsics.inc"
744#undef GET_SCALAR_INTRINSICS
745  };
746
747  static Hvx HvxInts[] = {
748#define GET_HVX_INTRINSICS
749#include "HexagonDepInstrIntrinsics.inc"
750#undef GET_HVX_INTRINSICS
751  };
752
753  const auto CmpOpcode = [](auto A, auto B) { return A.Opcode < B.Opcode; };
754  [[maybe_unused]] static bool SortedScalar =
755      (llvm::sort(ScalarInts, CmpOpcode), true);
756  [[maybe_unused]] static bool SortedHvx =
757      (llvm::sort(HvxInts, CmpOpcode), true);
758
759  auto [BS, ES] = std::make_pair(std::begin(ScalarInts), std::end(ScalarInts));
760  auto [BH, EH] = std::make_pair(std::begin(HvxInts), std::end(HvxInts));
761
762  auto FoundScalar = std::lower_bound(BS, ES, Scalar{Opc, 0}, CmpOpcode);
763  if (FoundScalar != ES && FoundScalar->Opcode == Opc)
764    return FoundScalar->IntId;
765
766  auto FoundHvx = std::lower_bound(BH, EH, Hvx{Opc, 0, 0}, CmpOpcode);
767  if (FoundHvx != EH && FoundHvx->Opcode == Opc) {
768    unsigned HwLen = getVectorLength();
769    if (HwLen == 64)
770      return FoundHvx->Int64Id;
771    if (HwLen == 128)
772      return FoundHvx->Int128Id;
773  }
774
775  std::string error = "Invalid opcode (" + std::to_string(Opc) + ")";
776  llvm_unreachable(error.c_str());
777  return 0;
778}
779