1//===-- ARMHazardRecognizer.cpp - ARM postra hazard recognizer ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ARMHazardRecognizer.h"
10#include "ARMBaseInstrInfo.h"
11#include "ARMBaseRegisterInfo.h"
12#include "ARMSubtarget.h"
13#include "llvm/Analysis/ValueTracking.h"
14#include "llvm/CodeGen/MachineFrameInfo.h"
15#include "llvm/CodeGen/MachineFunctionPass.h"
16#include "llvm/CodeGen/MachineInstr.h"
17#include "llvm/CodeGen/ScheduleDAG.h"
18#include "llvm/CodeGen/TargetRegisterInfo.h"
19#include "llvm/Support/CommandLine.h"
20
21using namespace llvm;
22
23static cl::opt<int> DataBankMask("arm-data-bank-mask", cl::init(-1),
24                                 cl::Hidden);
25static cl::opt<bool> AssumeITCMConflict("arm-assume-itcm-bankconflict",
26                                        cl::init(false), cl::Hidden);
27
28static bool hasRAWHazard(MachineInstr *DefMI, MachineInstr *MI,
29                         const TargetRegisterInfo &TRI) {
30  // FIXME: Detect integer instructions properly.
31  const MCInstrDesc &MCID = MI->getDesc();
32  unsigned Domain = MCID.TSFlags & ARMII::DomainMask;
33  if (MI->mayStore())
34    return false;
35  unsigned Opcode = MCID.getOpcode();
36  if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
37    return false;
38  if ((Domain & ARMII::DomainVFP) || (Domain & ARMII::DomainNEON))
39    return MI->readsRegister(DefMI->getOperand(0).getReg(), &TRI);
40  return false;
41}
42
43ScheduleHazardRecognizer::HazardType
44ARMHazardRecognizerFPMLx::getHazardType(SUnit *SU, int Stalls) {
45  assert(Stalls == 0 && "ARM hazards don't support scoreboard lookahead");
46
47  MachineInstr *MI = SU->getInstr();
48
49  if (!MI->isDebugInstr()) {
50    // Look for special VMLA / VMLS hazards. A VMUL / VADD / VSUB following
51    // a VMLA / VMLS will cause 4 cycle stall.
52    const MCInstrDesc &MCID = MI->getDesc();
53    if (LastMI && (MCID.TSFlags & ARMII::DomainMask) != ARMII::DomainGeneral) {
54      MachineInstr *DefMI = LastMI;
55      const MCInstrDesc &LastMCID = LastMI->getDesc();
56      const MachineFunction *MF = MI->getParent()->getParent();
57      const ARMBaseInstrInfo &TII = *static_cast<const ARMBaseInstrInfo *>(
58                                        MF->getSubtarget().getInstrInfo());
59
60      // Skip over one non-VFP / NEON instruction.
61      if (!LastMI->isBarrier() &&
62          !(TII.getSubtarget().hasMuxedUnits() && LastMI->mayLoadOrStore()) &&
63          (LastMCID.TSFlags & ARMII::DomainMask) == ARMII::DomainGeneral) {
64        MachineBasicBlock::iterator I = LastMI;
65        if (I != LastMI->getParent()->begin()) {
66          I = std::prev(I);
67          DefMI = &*I;
68        }
69      }
70
71      if (TII.isFpMLxInstruction(DefMI->getOpcode()) &&
72          (TII.canCauseFpMLxStall(MI->getOpcode()) ||
73           hasRAWHazard(DefMI, MI, TII.getRegisterInfo()))) {
74        // Try to schedule another instruction for the next 4 cycles.
75        if (FpMLxStalls == 0)
76          FpMLxStalls = 4;
77        return Hazard;
78      }
79    }
80  }
81  return NoHazard;
82}
83
84void ARMHazardRecognizerFPMLx::Reset() {
85  LastMI = nullptr;
86  FpMLxStalls = 0;
87}
88
89void ARMHazardRecognizerFPMLx::EmitInstruction(SUnit *SU) {
90  MachineInstr *MI = SU->getInstr();
91  if (!MI->isDebugInstr()) {
92    LastMI = MI;
93    FpMLxStalls = 0;
94  }
95}
96
97void ARMHazardRecognizerFPMLx::AdvanceCycle() {
98  if (FpMLxStalls && --FpMLxStalls == 0)
99    // Stalled for 4 cycles but still can't schedule any other instructions.
100    LastMI = nullptr;
101}
102
103void ARMHazardRecognizerFPMLx::RecedeCycle() {
104  llvm_unreachable("reverse ARM hazard checking unsupported");
105}
106
107///////// Bank conflicts handled as hazards //////////////
108
109static bool getBaseOffset(const MachineInstr &MI, const MachineOperand *&BaseOp,
110                          int64_t &Offset) {
111
112  uint64_t TSFlags = MI.getDesc().TSFlags;
113  unsigned AddrMode = (TSFlags & ARMII::AddrModeMask);
114  unsigned IndexMode =
115      (TSFlags & ARMII::IndexModeMask) >> ARMII::IndexModeShift;
116
117  // Address mode tells us what we want to know about operands for T2
118  // instructions (but not size).  It tells us size (but not about operands)
119  // for T1 instructions.
120  switch (AddrMode) {
121  default:
122    return false;
123  case ARMII::AddrModeT2_i8:
124    // t2LDRBT, t2LDRB_POST, t2LDRB_PRE, t2LDRBi8,
125    // t2LDRHT, t2LDRH_POST, t2LDRH_PRE, t2LDRHi8,
126    // t2LDRSBT, t2LDRSB_POST, t2LDRSB_PRE, t2LDRSBi8,
127    // t2LDRSHT, t2LDRSH_POST, t2LDRSH_PRE, t2LDRSHi8,
128    // t2LDRT, t2LDR_POST, t2LDR_PRE, t2LDRi8
129    BaseOp = &MI.getOperand(1);
130    Offset = (IndexMode == ARMII::IndexModePost)
131                 ? 0
132                 : (IndexMode == ARMII::IndexModePre ||
133                    IndexMode == ARMII::IndexModeUpd)
134                       ? MI.getOperand(3).getImm()
135                       : MI.getOperand(2).getImm();
136    return true;
137  case ARMII::AddrModeT2_i12:
138    // t2LDRBi12, t2LDRHi12
139    // t2LDRSBi12, t2LDRSHi12
140    // t2LDRi12
141    BaseOp = &MI.getOperand(1);
142    Offset = MI.getOperand(2).getImm();
143    return true;
144  case ARMII::AddrModeT2_i8s4:
145    // t2LDRD_POST, t2LDRD_PRE, t2LDRDi8
146    BaseOp = &MI.getOperand(2);
147    Offset = (IndexMode == ARMII::IndexModePost)
148                 ? 0
149                 : (IndexMode == ARMII::IndexModePre ||
150                    IndexMode == ARMII::IndexModeUpd)
151                       ? MI.getOperand(4).getImm()
152                       : MI.getOperand(3).getImm();
153    return true;
154  case ARMII::AddrModeT1_1:
155    // tLDRBi, tLDRBr (watch out!), TLDRSB
156  case ARMII::AddrModeT1_2:
157    // tLDRHi, tLDRHr (watch out!), TLDRSH
158  case ARMII::AddrModeT1_4:
159    // tLDRi, tLDRr (watch out!)
160    BaseOp = &MI.getOperand(1);
161    Offset = MI.getOperand(2).isImm() ? MI.getOperand(2).getImm() : 0;
162    return MI.getOperand(2).isImm();
163  }
164  return false;
165}
166
167ARMBankConflictHazardRecognizer::ARMBankConflictHazardRecognizer(
168    const ScheduleDAG *DAG, int64_t CPUBankMask, bool CPUAssumeITCMConflict)
169    : MF(DAG->MF), DL(DAG->MF.getDataLayout()),
170      DataMask(DataBankMask.getNumOccurrences() ? int64_t(DataBankMask)
171                                                : CPUBankMask),
172      AssumeITCMBankConflict(AssumeITCMConflict.getNumOccurrences()
173                                 ? AssumeITCMConflict
174                                 : CPUAssumeITCMConflict) {
175  MaxLookAhead = 1;
176}
177
178ScheduleHazardRecognizer::HazardType
179ARMBankConflictHazardRecognizer::CheckOffsets(unsigned O0, unsigned O1) {
180  return (((O0 ^ O1) & DataMask) != 0) ? NoHazard : Hazard;
181}
182
183ScheduleHazardRecognizer::HazardType
184ARMBankConflictHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
185  MachineInstr &L0 = *SU->getInstr();
186  if (!L0.mayLoad() || L0.mayStore() || L0.getNumMemOperands() != 1)
187    return NoHazard;
188
189  auto MO0 = *L0.memoperands().begin();
190  auto BaseVal0 = MO0->getValue();
191  auto BasePseudoVal0 = MO0->getPseudoValue();
192  int64_t Offset0 = 0;
193
194  if (MO0->getSize() > 4)
195    return NoHazard;
196
197  bool SPvalid = false;
198  const MachineOperand *SP = nullptr;
199  int64_t SPOffset0 = 0;
200
201  for (auto L1 : Accesses) {
202    auto MO1 = *L1->memoperands().begin();
203    auto BaseVal1 = MO1->getValue();
204    auto BasePseudoVal1 = MO1->getPseudoValue();
205    int64_t Offset1 = 0;
206
207    // Pointers to the same object
208    if (BaseVal0 && BaseVal1) {
209      const Value *Ptr0, *Ptr1;
210      Ptr0 = GetPointerBaseWithConstantOffset(BaseVal0, Offset0, DL, true);
211      Ptr1 = GetPointerBaseWithConstantOffset(BaseVal1, Offset1, DL, true);
212      if (Ptr0 == Ptr1 && Ptr0)
213        return CheckOffsets(Offset0, Offset1);
214    }
215
216    if (BasePseudoVal0 && BasePseudoVal1 &&
217        BasePseudoVal0->kind() == BasePseudoVal1->kind() &&
218        BasePseudoVal0->kind() == PseudoSourceValue::FixedStack) {
219      // Spills/fills
220      auto FS0 = cast<FixedStackPseudoSourceValue>(BasePseudoVal0);
221      auto FS1 = cast<FixedStackPseudoSourceValue>(BasePseudoVal1);
222      Offset0 = MF.getFrameInfo().getObjectOffset(FS0->getFrameIndex());
223      Offset1 = MF.getFrameInfo().getObjectOffset(FS1->getFrameIndex());
224      return CheckOffsets(Offset0, Offset1);
225    }
226
227    // Constant pools (likely in ITCM)
228    if (BasePseudoVal0 && BasePseudoVal1 &&
229        BasePseudoVal0->kind() == BasePseudoVal1->kind() &&
230        BasePseudoVal0->isConstantPool() && AssumeITCMBankConflict)
231      return Hazard;
232
233    // Is this a stack pointer-relative access?  We could in general try to
234    // use "is this the same register and is it unchanged?", but the
235    // memory operand tracking is highly likely to have already found that.
236    // What we're after here is bank conflicts between different objects in
237    // the stack frame.
238    if (!SPvalid) { // set up SP
239      if (!getBaseOffset(L0, SP, SPOffset0) || SP->getReg().id() != ARM::SP)
240        SP = nullptr;
241      SPvalid = true;
242    }
243    if (SP) {
244      int64_t SPOffset1;
245      const MachineOperand *SP1;
246      if (getBaseOffset(*L1, SP1, SPOffset1) && SP1->getReg().id() == ARM::SP)
247        return CheckOffsets(SPOffset0, SPOffset1);
248    }
249  }
250
251  return NoHazard;
252}
253
254void ARMBankConflictHazardRecognizer::Reset() { Accesses.clear(); }
255
256void ARMBankConflictHazardRecognizer::EmitInstruction(SUnit *SU) {
257  MachineInstr &MI = *SU->getInstr();
258  if (!MI.mayLoad() || MI.mayStore() || MI.getNumMemOperands() != 1)
259    return;
260
261  auto MO = *MI.memoperands().begin();
262  uint64_t Size1 = MO->getSize();
263  if (Size1 > 4)
264    return;
265  Accesses.push_back(&MI);
266}
267
268void ARMBankConflictHazardRecognizer::AdvanceCycle() { Accesses.clear(); }
269
270void ARMBankConflictHazardRecognizer::RecedeCycle() { Accesses.clear(); }
271