1//===-- SystemZInstrInfo.cpp - SystemZ instruction information ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the SystemZ implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SystemZInstrInfo.h"
14#include "MCTargetDesc/SystemZMCTargetDesc.h"
15#include "SystemZ.h"
16#include "SystemZInstrBuilder.h"
17#include "SystemZSubtarget.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/CodeGen/LiveInterval.h"
20#include "llvm/CodeGen/LiveIntervals.h"
21#include "llvm/CodeGen/LiveVariables.h"
22#include "llvm/CodeGen/MachineBasicBlock.h"
23#include "llvm/CodeGen/MachineFrameInfo.h"
24#include "llvm/CodeGen/MachineFunction.h"
25#include "llvm/CodeGen/MachineInstr.h"
26#include "llvm/CodeGen/MachineMemOperand.h"
27#include "llvm/CodeGen/MachineOperand.h"
28#include "llvm/CodeGen/MachineRegisterInfo.h"
29#include "llvm/CodeGen/SlotIndexes.h"
30#include "llvm/CodeGen/TargetInstrInfo.h"
31#include "llvm/CodeGen/TargetSubtargetInfo.h"
32#include "llvm/MC/MCInstrDesc.h"
33#include "llvm/MC/MCRegisterInfo.h"
34#include "llvm/Support/BranchProbability.h"
35#include "llvm/Support/ErrorHandling.h"
36#include "llvm/Support/MathExtras.h"
37#include "llvm/Target/TargetMachine.h"
38#include <cassert>
39#include <cstdint>
40#include <iterator>
41
42using namespace llvm;
43
44#define GET_INSTRINFO_CTOR_DTOR
45#define GET_INSTRMAP_INFO
46#include "SystemZGenInstrInfo.inc"
47
48#define DEBUG_TYPE "systemz-II"
49
50// Return a mask with Count low bits set.
51static uint64_t allOnes(unsigned int Count) {
52  return Count == 0 ? 0 : (uint64_t(1) << (Count - 1) << 1) - 1;
53}
54
55// Pin the vtable to this file.
56void SystemZInstrInfo::anchor() {}
57
58SystemZInstrInfo::SystemZInstrInfo(SystemZSubtarget &sti)
59  : SystemZGenInstrInfo(SystemZ::ADJCALLSTACKDOWN, SystemZ::ADJCALLSTACKUP),
60    RI(), STI(sti) {
61}
62
63// MI is a 128-bit load or store.  Split it into two 64-bit loads or stores,
64// each having the opcode given by NewOpcode.
65void SystemZInstrInfo::splitMove(MachineBasicBlock::iterator MI,
66                                 unsigned NewOpcode) const {
67  MachineBasicBlock *MBB = MI->getParent();
68  MachineFunction &MF = *MBB->getParent();
69
70  // Get two load or store instructions.  Use the original instruction for one
71  // of them (arbitrarily the second here) and create a clone for the other.
72  MachineInstr *EarlierMI = MF.CloneMachineInstr(&*MI);
73  MBB->insert(MI, EarlierMI);
74
75  // Set up the two 64-bit registers and remember super reg and its flags.
76  MachineOperand &HighRegOp = EarlierMI->getOperand(0);
77  MachineOperand &LowRegOp = MI->getOperand(0);
78  Register Reg128 = LowRegOp.getReg();
79  unsigned Reg128Killed = getKillRegState(LowRegOp.isKill());
80  unsigned Reg128Undef  = getUndefRegState(LowRegOp.isUndef());
81  HighRegOp.setReg(RI.getSubReg(HighRegOp.getReg(), SystemZ::subreg_h64));
82  LowRegOp.setReg(RI.getSubReg(LowRegOp.getReg(), SystemZ::subreg_l64));
83
84  if (MI->mayStore()) {
85    // Add implicit uses of the super register in case one of the subregs is
86    // undefined. We could track liveness and skip storing an undefined
87    // subreg, but this is hopefully rare (discovered with llvm-stress).
88    // If Reg128 was killed, set kill flag on MI.
89    unsigned Reg128UndefImpl = (Reg128Undef | RegState::Implicit);
90    MachineInstrBuilder(MF, EarlierMI).addReg(Reg128, Reg128UndefImpl);
91    MachineInstrBuilder(MF, MI).addReg(Reg128, (Reg128UndefImpl | Reg128Killed));
92  }
93
94  // The address in the first (high) instruction is already correct.
95  // Adjust the offset in the second (low) instruction.
96  MachineOperand &HighOffsetOp = EarlierMI->getOperand(2);
97  MachineOperand &LowOffsetOp = MI->getOperand(2);
98  LowOffsetOp.setImm(LowOffsetOp.getImm() + 8);
99
100  // Clear the kill flags on the registers in the first instruction.
101  if (EarlierMI->getOperand(0).isReg() && EarlierMI->getOperand(0).isUse())
102    EarlierMI->getOperand(0).setIsKill(false);
103  EarlierMI->getOperand(1).setIsKill(false);
104  EarlierMI->getOperand(3).setIsKill(false);
105
106  // Set the opcodes.
107  unsigned HighOpcode = getOpcodeForOffset(NewOpcode, HighOffsetOp.getImm());
108  unsigned LowOpcode = getOpcodeForOffset(NewOpcode, LowOffsetOp.getImm());
109  assert(HighOpcode && LowOpcode && "Both offsets should be in range");
110
111  EarlierMI->setDesc(get(HighOpcode));
112  MI->setDesc(get(LowOpcode));
113}
114
115// Split ADJDYNALLOC instruction MI.
116void SystemZInstrInfo::splitAdjDynAlloc(MachineBasicBlock::iterator MI) const {
117  MachineBasicBlock *MBB = MI->getParent();
118  MachineFunction &MF = *MBB->getParent();
119  MachineFrameInfo &MFFrame = MF.getFrameInfo();
120  MachineOperand &OffsetMO = MI->getOperand(2);
121
122  uint64_t Offset = (MFFrame.getMaxCallFrameSize() +
123                     SystemZMC::CallFrameSize +
124                     OffsetMO.getImm());
125  unsigned NewOpcode = getOpcodeForOffset(SystemZ::LA, Offset);
126  assert(NewOpcode && "No support for huge argument lists yet");
127  MI->setDesc(get(NewOpcode));
128  OffsetMO.setImm(Offset);
129}
130
131// MI is an RI-style pseudo instruction.  Replace it with LowOpcode
132// if the first operand is a low GR32 and HighOpcode if the first operand
133// is a high GR32.  ConvertHigh is true if LowOpcode takes a signed operand
134// and HighOpcode takes an unsigned 32-bit operand.  In those cases,
135// MI has the same kind of operand as LowOpcode, so needs to be converted
136// if HighOpcode is used.
137void SystemZInstrInfo::expandRIPseudo(MachineInstr &MI, unsigned LowOpcode,
138                                      unsigned HighOpcode,
139                                      bool ConvertHigh) const {
140  Register Reg = MI.getOperand(0).getReg();
141  bool IsHigh = SystemZ::isHighReg(Reg);
142  MI.setDesc(get(IsHigh ? HighOpcode : LowOpcode));
143  if (IsHigh && ConvertHigh)
144    MI.getOperand(1).setImm(uint32_t(MI.getOperand(1).getImm()));
145}
146
147// MI is a three-operand RIE-style pseudo instruction.  Replace it with
148// LowOpcodeK if the registers are both low GR32s, otherwise use a move
149// followed by HighOpcode or LowOpcode, depending on whether the target
150// is a high or low GR32.
151void SystemZInstrInfo::expandRIEPseudo(MachineInstr &MI, unsigned LowOpcode,
152                                       unsigned LowOpcodeK,
153                                       unsigned HighOpcode) const {
154  Register DestReg = MI.getOperand(0).getReg();
155  Register SrcReg = MI.getOperand(1).getReg();
156  bool DestIsHigh = SystemZ::isHighReg(DestReg);
157  bool SrcIsHigh = SystemZ::isHighReg(SrcReg);
158  if (!DestIsHigh && !SrcIsHigh)
159    MI.setDesc(get(LowOpcodeK));
160  else {
161    if (DestReg != SrcReg) {
162      emitGRX32Move(*MI.getParent(), MI, MI.getDebugLoc(), DestReg, SrcReg,
163                    SystemZ::LR, 32, MI.getOperand(1).isKill(),
164                    MI.getOperand(1).isUndef());
165      MI.getOperand(1).setReg(DestReg);
166    }
167    MI.setDesc(get(DestIsHigh ? HighOpcode : LowOpcode));
168    MI.tieOperands(0, 1);
169  }
170}
171
172// MI is an RXY-style pseudo instruction.  Replace it with LowOpcode
173// if the first operand is a low GR32 and HighOpcode if the first operand
174// is a high GR32.
175void SystemZInstrInfo::expandRXYPseudo(MachineInstr &MI, unsigned LowOpcode,
176                                       unsigned HighOpcode) const {
177  Register Reg = MI.getOperand(0).getReg();
178  unsigned Opcode = getOpcodeForOffset(
179      SystemZ::isHighReg(Reg) ? HighOpcode : LowOpcode,
180      MI.getOperand(2).getImm());
181  MI.setDesc(get(Opcode));
182}
183
184// MI is a load-on-condition pseudo instruction with a single register
185// (source or destination) operand.  Replace it with LowOpcode if the
186// register is a low GR32 and HighOpcode if the register is a high GR32.
187void SystemZInstrInfo::expandLOCPseudo(MachineInstr &MI, unsigned LowOpcode,
188                                       unsigned HighOpcode) const {
189  Register Reg = MI.getOperand(0).getReg();
190  unsigned Opcode = SystemZ::isHighReg(Reg) ? HighOpcode : LowOpcode;
191  MI.setDesc(get(Opcode));
192}
193
194// MI is an RR-style pseudo instruction that zero-extends the low Size bits
195// of one GRX32 into another.  Replace it with LowOpcode if both operands
196// are low registers, otherwise use RISB[LH]G.
197void SystemZInstrInfo::expandZExtPseudo(MachineInstr &MI, unsigned LowOpcode,
198                                        unsigned Size) const {
199  MachineInstrBuilder MIB =
200    emitGRX32Move(*MI.getParent(), MI, MI.getDebugLoc(),
201               MI.getOperand(0).getReg(), MI.getOperand(1).getReg(), LowOpcode,
202               Size, MI.getOperand(1).isKill(), MI.getOperand(1).isUndef());
203
204  // Keep the remaining operands as-is.
205  for (unsigned I = 2; I < MI.getNumOperands(); ++I)
206    MIB.add(MI.getOperand(I));
207
208  MI.eraseFromParent();
209}
210
211void SystemZInstrInfo::expandLoadStackGuard(MachineInstr *MI) const {
212  MachineBasicBlock *MBB = MI->getParent();
213  MachineFunction &MF = *MBB->getParent();
214  const Register Reg64 = MI->getOperand(0).getReg();
215  const Register Reg32 = RI.getSubReg(Reg64, SystemZ::subreg_l32);
216
217  // EAR can only load the low subregister so us a shift for %a0 to produce
218  // the GR containing %a0 and %a1.
219
220  // ear <reg>, %a0
221  BuildMI(*MBB, MI, MI->getDebugLoc(), get(SystemZ::EAR), Reg32)
222    .addReg(SystemZ::A0)
223    .addReg(Reg64, RegState::ImplicitDefine);
224
225  // sllg <reg>, <reg>, 32
226  BuildMI(*MBB, MI, MI->getDebugLoc(), get(SystemZ::SLLG), Reg64)
227    .addReg(Reg64)
228    .addReg(0)
229    .addImm(32);
230
231  // ear <reg>, %a1
232  BuildMI(*MBB, MI, MI->getDebugLoc(), get(SystemZ::EAR), Reg32)
233    .addReg(SystemZ::A1);
234
235  // lg <reg>, 40(<reg>)
236  MI->setDesc(get(SystemZ::LG));
237  MachineInstrBuilder(MF, MI).addReg(Reg64).addImm(40).addReg(0);
238}
239
240// Emit a zero-extending move from 32-bit GPR SrcReg to 32-bit GPR
241// DestReg before MBBI in MBB.  Use LowLowOpcode when both DestReg and SrcReg
242// are low registers, otherwise use RISB[LH]G.  Size is the number of bits
243// taken from the low end of SrcReg (8 for LLCR, 16 for LLHR and 32 for LR).
244// KillSrc is true if this move is the last use of SrcReg.
245MachineInstrBuilder
246SystemZInstrInfo::emitGRX32Move(MachineBasicBlock &MBB,
247                                MachineBasicBlock::iterator MBBI,
248                                const DebugLoc &DL, unsigned DestReg,
249                                unsigned SrcReg, unsigned LowLowOpcode,
250                                unsigned Size, bool KillSrc,
251                                bool UndefSrc) const {
252  unsigned Opcode;
253  bool DestIsHigh = SystemZ::isHighReg(DestReg);
254  bool SrcIsHigh = SystemZ::isHighReg(SrcReg);
255  if (DestIsHigh && SrcIsHigh)
256    Opcode = SystemZ::RISBHH;
257  else if (DestIsHigh && !SrcIsHigh)
258    Opcode = SystemZ::RISBHL;
259  else if (!DestIsHigh && SrcIsHigh)
260    Opcode = SystemZ::RISBLH;
261  else {
262    return BuildMI(MBB, MBBI, DL, get(LowLowOpcode), DestReg)
263      .addReg(SrcReg, getKillRegState(KillSrc) | getUndefRegState(UndefSrc));
264  }
265  unsigned Rotate = (DestIsHigh != SrcIsHigh ? 32 : 0);
266  return BuildMI(MBB, MBBI, DL, get(Opcode), DestReg)
267    .addReg(DestReg, RegState::Undef)
268    .addReg(SrcReg, getKillRegState(KillSrc) | getUndefRegState(UndefSrc))
269    .addImm(32 - Size).addImm(128 + 31).addImm(Rotate);
270}
271
272MachineInstr *SystemZInstrInfo::commuteInstructionImpl(MachineInstr &MI,
273                                                       bool NewMI,
274                                                       unsigned OpIdx1,
275                                                       unsigned OpIdx2) const {
276  auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & {
277    if (NewMI)
278      return *MI.getParent()->getParent()->CloneMachineInstr(&MI);
279    return MI;
280  };
281
282  switch (MI.getOpcode()) {
283  case SystemZ::SELRMux:
284  case SystemZ::SELFHR:
285  case SystemZ::SELR:
286  case SystemZ::SELGR:
287  case SystemZ::LOCRMux:
288  case SystemZ::LOCFHR:
289  case SystemZ::LOCR:
290  case SystemZ::LOCGR: {
291    auto &WorkingMI = cloneIfNew(MI);
292    // Invert condition.
293    unsigned CCValid = WorkingMI.getOperand(3).getImm();
294    unsigned CCMask = WorkingMI.getOperand(4).getImm();
295    WorkingMI.getOperand(4).setImm(CCMask ^ CCValid);
296    return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
297                                                   OpIdx1, OpIdx2);
298  }
299  default:
300    return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
301  }
302}
303
304// If MI is a simple load or store for a frame object, return the register
305// it loads or stores and set FrameIndex to the index of the frame object.
306// Return 0 otherwise.
307//
308// Flag is SimpleBDXLoad for loads and SimpleBDXStore for stores.
309static int isSimpleMove(const MachineInstr &MI, int &FrameIndex,
310                        unsigned Flag) {
311  const MCInstrDesc &MCID = MI.getDesc();
312  if ((MCID.TSFlags & Flag) && MI.getOperand(1).isFI() &&
313      MI.getOperand(2).getImm() == 0 && MI.getOperand(3).getReg() == 0) {
314    FrameIndex = MI.getOperand(1).getIndex();
315    return MI.getOperand(0).getReg();
316  }
317  return 0;
318}
319
320unsigned SystemZInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
321                                               int &FrameIndex) const {
322  return isSimpleMove(MI, FrameIndex, SystemZII::SimpleBDXLoad);
323}
324
325unsigned SystemZInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
326                                              int &FrameIndex) const {
327  return isSimpleMove(MI, FrameIndex, SystemZII::SimpleBDXStore);
328}
329
330bool SystemZInstrInfo::isStackSlotCopy(const MachineInstr &MI,
331                                       int &DestFrameIndex,
332                                       int &SrcFrameIndex) const {
333  // Check for MVC 0(Length,FI1),0(FI2)
334  const MachineFrameInfo &MFI = MI.getParent()->getParent()->getFrameInfo();
335  if (MI.getOpcode() != SystemZ::MVC || !MI.getOperand(0).isFI() ||
336      MI.getOperand(1).getImm() != 0 || !MI.getOperand(3).isFI() ||
337      MI.getOperand(4).getImm() != 0)
338    return false;
339
340  // Check that Length covers the full slots.
341  int64_t Length = MI.getOperand(2).getImm();
342  unsigned FI1 = MI.getOperand(0).getIndex();
343  unsigned FI2 = MI.getOperand(3).getIndex();
344  if (MFI.getObjectSize(FI1) != Length ||
345      MFI.getObjectSize(FI2) != Length)
346    return false;
347
348  DestFrameIndex = FI1;
349  SrcFrameIndex = FI2;
350  return true;
351}
352
353bool SystemZInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
354                                     MachineBasicBlock *&TBB,
355                                     MachineBasicBlock *&FBB,
356                                     SmallVectorImpl<MachineOperand> &Cond,
357                                     bool AllowModify) const {
358  // Most of the code and comments here are boilerplate.
359
360  // Start from the bottom of the block and work up, examining the
361  // terminator instructions.
362  MachineBasicBlock::iterator I = MBB.end();
363  while (I != MBB.begin()) {
364    --I;
365    if (I->isDebugInstr())
366      continue;
367
368    // Working from the bottom, when we see a non-terminator instruction, we're
369    // done.
370    if (!isUnpredicatedTerminator(*I))
371      break;
372
373    // A terminator that isn't a branch can't easily be handled by this
374    // analysis.
375    if (!I->isBranch())
376      return true;
377
378    // Can't handle indirect branches.
379    SystemZII::Branch Branch(getBranchInfo(*I));
380    if (!Branch.hasMBBTarget())
381      return true;
382
383    // Punt on compound branches.
384    if (Branch.Type != SystemZII::BranchNormal)
385      return true;
386
387    if (Branch.CCMask == SystemZ::CCMASK_ANY) {
388      // Handle unconditional branches.
389      if (!AllowModify) {
390        TBB = Branch.getMBBTarget();
391        continue;
392      }
393
394      // If the block has any instructions after a JMP, delete them.
395      while (std::next(I) != MBB.end())
396        std::next(I)->eraseFromParent();
397
398      Cond.clear();
399      FBB = nullptr;
400
401      // Delete the JMP if it's equivalent to a fall-through.
402      if (MBB.isLayoutSuccessor(Branch.getMBBTarget())) {
403        TBB = nullptr;
404        I->eraseFromParent();
405        I = MBB.end();
406        continue;
407      }
408
409      // TBB is used to indicate the unconditinal destination.
410      TBB = Branch.getMBBTarget();
411      continue;
412    }
413
414    // Working from the bottom, handle the first conditional branch.
415    if (Cond.empty()) {
416      // FIXME: add X86-style branch swap
417      FBB = TBB;
418      TBB = Branch.getMBBTarget();
419      Cond.push_back(MachineOperand::CreateImm(Branch.CCValid));
420      Cond.push_back(MachineOperand::CreateImm(Branch.CCMask));
421      continue;
422    }
423
424    // Handle subsequent conditional branches.
425    assert(Cond.size() == 2 && TBB && "Should have seen a conditional branch");
426
427    // Only handle the case where all conditional branches branch to the same
428    // destination.
429    if (TBB != Branch.getMBBTarget())
430      return true;
431
432    // If the conditions are the same, we can leave them alone.
433    unsigned OldCCValid = Cond[0].getImm();
434    unsigned OldCCMask = Cond[1].getImm();
435    if (OldCCValid == Branch.CCValid && OldCCMask == Branch.CCMask)
436      continue;
437
438    // FIXME: Try combining conditions like X86 does.  Should be easy on Z!
439    return false;
440  }
441
442  return false;
443}
444
445unsigned SystemZInstrInfo::removeBranch(MachineBasicBlock &MBB,
446                                        int *BytesRemoved) const {
447  assert(!BytesRemoved && "code size not handled");
448
449  // Most of the code and comments here are boilerplate.
450  MachineBasicBlock::iterator I = MBB.end();
451  unsigned Count = 0;
452
453  while (I != MBB.begin()) {
454    --I;
455    if (I->isDebugInstr())
456      continue;
457    if (!I->isBranch())
458      break;
459    if (!getBranchInfo(*I).hasMBBTarget())
460      break;
461    // Remove the branch.
462    I->eraseFromParent();
463    I = MBB.end();
464    ++Count;
465  }
466
467  return Count;
468}
469
470bool SystemZInstrInfo::
471reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
472  assert(Cond.size() == 2 && "Invalid condition");
473  Cond[1].setImm(Cond[1].getImm() ^ Cond[0].getImm());
474  return false;
475}
476
477unsigned SystemZInstrInfo::insertBranch(MachineBasicBlock &MBB,
478                                        MachineBasicBlock *TBB,
479                                        MachineBasicBlock *FBB,
480                                        ArrayRef<MachineOperand> Cond,
481                                        const DebugLoc &DL,
482                                        int *BytesAdded) const {
483  // In this function we output 32-bit branches, which should always
484  // have enough range.  They can be shortened and relaxed by later code
485  // in the pipeline, if desired.
486
487  // Shouldn't be a fall through.
488  assert(TBB && "insertBranch must not be told to insert a fallthrough");
489  assert((Cond.size() == 2 || Cond.size() == 0) &&
490         "SystemZ branch conditions have one component!");
491  assert(!BytesAdded && "code size not handled");
492
493  if (Cond.empty()) {
494    // Unconditional branch?
495    assert(!FBB && "Unconditional branch with multiple successors!");
496    BuildMI(&MBB, DL, get(SystemZ::J)).addMBB(TBB);
497    return 1;
498  }
499
500  // Conditional branch.
501  unsigned Count = 0;
502  unsigned CCValid = Cond[0].getImm();
503  unsigned CCMask = Cond[1].getImm();
504  BuildMI(&MBB, DL, get(SystemZ::BRC))
505    .addImm(CCValid).addImm(CCMask).addMBB(TBB);
506  ++Count;
507
508  if (FBB) {
509    // Two-way Conditional branch. Insert the second branch.
510    BuildMI(&MBB, DL, get(SystemZ::J)).addMBB(FBB);
511    ++Count;
512  }
513  return Count;
514}
515
516bool SystemZInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
517                                      Register &SrcReg2, int &Mask,
518                                      int &Value) const {
519  assert(MI.isCompare() && "Caller should have checked for a comparison");
520
521  if (MI.getNumExplicitOperands() == 2 && MI.getOperand(0).isReg() &&
522      MI.getOperand(1).isImm()) {
523    SrcReg = MI.getOperand(0).getReg();
524    SrcReg2 = 0;
525    Value = MI.getOperand(1).getImm();
526    Mask = ~0;
527    return true;
528  }
529
530  return false;
531}
532
533bool SystemZInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
534                                       ArrayRef<MachineOperand> Pred,
535                                       Register DstReg, Register TrueReg,
536                                       Register FalseReg, int &CondCycles,
537                                       int &TrueCycles,
538                                       int &FalseCycles) const {
539  // Not all subtargets have LOCR instructions.
540  if (!STI.hasLoadStoreOnCond())
541    return false;
542  if (Pred.size() != 2)
543    return false;
544
545  // Check register classes.
546  const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
547  const TargetRegisterClass *RC =
548    RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
549  if (!RC)
550    return false;
551
552  // We have LOCR instructions for 32 and 64 bit general purpose registers.
553  if ((STI.hasLoadStoreOnCond2() &&
554       SystemZ::GRX32BitRegClass.hasSubClassEq(RC)) ||
555      SystemZ::GR32BitRegClass.hasSubClassEq(RC) ||
556      SystemZ::GR64BitRegClass.hasSubClassEq(RC)) {
557    CondCycles = 2;
558    TrueCycles = 2;
559    FalseCycles = 2;
560    return true;
561  }
562
563  // Can't do anything else.
564  return false;
565}
566
567void SystemZInstrInfo::insertSelect(MachineBasicBlock &MBB,
568                                    MachineBasicBlock::iterator I,
569                                    const DebugLoc &DL, Register DstReg,
570                                    ArrayRef<MachineOperand> Pred,
571                                    Register TrueReg,
572                                    Register FalseReg) const {
573  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
574  const TargetRegisterClass *RC = MRI.getRegClass(DstReg);
575
576  assert(Pred.size() == 2 && "Invalid condition");
577  unsigned CCValid = Pred[0].getImm();
578  unsigned CCMask = Pred[1].getImm();
579
580  unsigned Opc;
581  if (SystemZ::GRX32BitRegClass.hasSubClassEq(RC)) {
582    if (STI.hasMiscellaneousExtensions3())
583      Opc = SystemZ::SELRMux;
584    else if (STI.hasLoadStoreOnCond2())
585      Opc = SystemZ::LOCRMux;
586    else {
587      Opc = SystemZ::LOCR;
588      MRI.constrainRegClass(DstReg, &SystemZ::GR32BitRegClass);
589      Register TReg = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
590      Register FReg = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
591      BuildMI(MBB, I, DL, get(TargetOpcode::COPY), TReg).addReg(TrueReg);
592      BuildMI(MBB, I, DL, get(TargetOpcode::COPY), FReg).addReg(FalseReg);
593      TrueReg = TReg;
594      FalseReg = FReg;
595    }
596  } else if (SystemZ::GR64BitRegClass.hasSubClassEq(RC)) {
597    if (STI.hasMiscellaneousExtensions3())
598      Opc = SystemZ::SELGR;
599    else
600      Opc = SystemZ::LOCGR;
601  } else
602    llvm_unreachable("Invalid register class");
603
604  BuildMI(MBB, I, DL, get(Opc), DstReg)
605    .addReg(FalseReg).addReg(TrueReg)
606    .addImm(CCValid).addImm(CCMask);
607}
608
609bool SystemZInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
610                                     Register Reg,
611                                     MachineRegisterInfo *MRI) const {
612  unsigned DefOpc = DefMI.getOpcode();
613  if (DefOpc != SystemZ::LHIMux && DefOpc != SystemZ::LHI &&
614      DefOpc != SystemZ::LGHI)
615    return false;
616  if (DefMI.getOperand(0).getReg() != Reg)
617    return false;
618  int32_t ImmVal = (int32_t)DefMI.getOperand(1).getImm();
619
620  unsigned UseOpc = UseMI.getOpcode();
621  unsigned NewUseOpc;
622  unsigned UseIdx;
623  int CommuteIdx = -1;
624  bool TieOps = false;
625  switch (UseOpc) {
626  case SystemZ::SELRMux:
627    TieOps = true;
628    LLVM_FALLTHROUGH;
629  case SystemZ::LOCRMux:
630    if (!STI.hasLoadStoreOnCond2())
631      return false;
632    NewUseOpc = SystemZ::LOCHIMux;
633    if (UseMI.getOperand(2).getReg() == Reg)
634      UseIdx = 2;
635    else if (UseMI.getOperand(1).getReg() == Reg)
636      UseIdx = 2, CommuteIdx = 1;
637    else
638      return false;
639    break;
640  case SystemZ::SELGR:
641    TieOps = true;
642    LLVM_FALLTHROUGH;
643  case SystemZ::LOCGR:
644    if (!STI.hasLoadStoreOnCond2())
645      return false;
646    NewUseOpc = SystemZ::LOCGHI;
647    if (UseMI.getOperand(2).getReg() == Reg)
648      UseIdx = 2;
649    else if (UseMI.getOperand(1).getReg() == Reg)
650      UseIdx = 2, CommuteIdx = 1;
651    else
652      return false;
653    break;
654  default:
655    return false;
656  }
657
658  if (CommuteIdx != -1)
659    if (!commuteInstruction(UseMI, false, CommuteIdx, UseIdx))
660      return false;
661
662  bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
663  UseMI.setDesc(get(NewUseOpc));
664  if (TieOps)
665    UseMI.tieOperands(0, 1);
666  UseMI.getOperand(UseIdx).ChangeToImmediate(ImmVal);
667  if (DeleteDef)
668    DefMI.eraseFromParent();
669
670  return true;
671}
672
673bool SystemZInstrInfo::isPredicable(const MachineInstr &MI) const {
674  unsigned Opcode = MI.getOpcode();
675  if (Opcode == SystemZ::Return ||
676      Opcode == SystemZ::Trap ||
677      Opcode == SystemZ::CallJG ||
678      Opcode == SystemZ::CallBR)
679    return true;
680  return false;
681}
682
683bool SystemZInstrInfo::
684isProfitableToIfCvt(MachineBasicBlock &MBB,
685                    unsigned NumCycles, unsigned ExtraPredCycles,
686                    BranchProbability Probability) const {
687  // Avoid using conditional returns at the end of a loop (since then
688  // we'd need to emit an unconditional branch to the beginning anyway,
689  // making the loop body longer).  This doesn't apply for low-probability
690  // loops (eg. compare-and-swap retry), so just decide based on branch
691  // probability instead of looping structure.
692  // However, since Compare and Trap instructions cost the same as a regular
693  // Compare instruction, we should allow the if conversion to convert this
694  // into a Conditional Compare regardless of the branch probability.
695  if (MBB.getLastNonDebugInstr()->getOpcode() != SystemZ::Trap &&
696      MBB.succ_empty() && Probability < BranchProbability(1, 8))
697    return false;
698  // For now only convert single instructions.
699  return NumCycles == 1;
700}
701
702bool SystemZInstrInfo::
703isProfitableToIfCvt(MachineBasicBlock &TMBB,
704                    unsigned NumCyclesT, unsigned ExtraPredCyclesT,
705                    MachineBasicBlock &FMBB,
706                    unsigned NumCyclesF, unsigned ExtraPredCyclesF,
707                    BranchProbability Probability) const {
708  // For now avoid converting mutually-exclusive cases.
709  return false;
710}
711
712bool SystemZInstrInfo::
713isProfitableToDupForIfCvt(MachineBasicBlock &MBB, unsigned NumCycles,
714                          BranchProbability Probability) const {
715  // For now only duplicate single instructions.
716  return NumCycles == 1;
717}
718
719bool SystemZInstrInfo::PredicateInstruction(
720    MachineInstr &MI, ArrayRef<MachineOperand> Pred) const {
721  assert(Pred.size() == 2 && "Invalid condition");
722  unsigned CCValid = Pred[0].getImm();
723  unsigned CCMask = Pred[1].getImm();
724  assert(CCMask > 0 && CCMask < 15 && "Invalid predicate");
725  unsigned Opcode = MI.getOpcode();
726  if (Opcode == SystemZ::Trap) {
727    MI.setDesc(get(SystemZ::CondTrap));
728    MachineInstrBuilder(*MI.getParent()->getParent(), MI)
729      .addImm(CCValid).addImm(CCMask)
730      .addReg(SystemZ::CC, RegState::Implicit);
731    return true;
732  }
733  if (Opcode == SystemZ::Return) {
734    MI.setDesc(get(SystemZ::CondReturn));
735    MachineInstrBuilder(*MI.getParent()->getParent(), MI)
736      .addImm(CCValid).addImm(CCMask)
737      .addReg(SystemZ::CC, RegState::Implicit);
738    return true;
739  }
740  if (Opcode == SystemZ::CallJG) {
741    MachineOperand FirstOp = MI.getOperand(0);
742    const uint32_t *RegMask = MI.getOperand(1).getRegMask();
743    MI.RemoveOperand(1);
744    MI.RemoveOperand(0);
745    MI.setDesc(get(SystemZ::CallBRCL));
746    MachineInstrBuilder(*MI.getParent()->getParent(), MI)
747        .addImm(CCValid)
748        .addImm(CCMask)
749        .add(FirstOp)
750        .addRegMask(RegMask)
751        .addReg(SystemZ::CC, RegState::Implicit);
752    return true;
753  }
754  if (Opcode == SystemZ::CallBR) {
755    const uint32_t *RegMask = MI.getOperand(0).getRegMask();
756    MI.RemoveOperand(0);
757    MI.setDesc(get(SystemZ::CallBCR));
758    MachineInstrBuilder(*MI.getParent()->getParent(), MI)
759      .addImm(CCValid).addImm(CCMask)
760      .addRegMask(RegMask)
761      .addReg(SystemZ::CC, RegState::Implicit);
762    return true;
763  }
764  return false;
765}
766
767void SystemZInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
768                                   MachineBasicBlock::iterator MBBI,
769                                   const DebugLoc &DL, MCRegister DestReg,
770                                   MCRegister SrcReg, bool KillSrc) const {
771  // Split 128-bit GPR moves into two 64-bit moves. Add implicit uses of the
772  // super register in case one of the subregs is undefined.
773  // This handles ADDR128 too.
774  if (SystemZ::GR128BitRegClass.contains(DestReg, SrcReg)) {
775    copyPhysReg(MBB, MBBI, DL, RI.getSubReg(DestReg, SystemZ::subreg_h64),
776                RI.getSubReg(SrcReg, SystemZ::subreg_h64), KillSrc);
777    MachineInstrBuilder(*MBB.getParent(), std::prev(MBBI))
778      .addReg(SrcReg, RegState::Implicit);
779    copyPhysReg(MBB, MBBI, DL, RI.getSubReg(DestReg, SystemZ::subreg_l64),
780                RI.getSubReg(SrcReg, SystemZ::subreg_l64), KillSrc);
781    MachineInstrBuilder(*MBB.getParent(), std::prev(MBBI))
782      .addReg(SrcReg, (getKillRegState(KillSrc) | RegState::Implicit));
783    return;
784  }
785
786  if (SystemZ::GRX32BitRegClass.contains(DestReg, SrcReg)) {
787    emitGRX32Move(MBB, MBBI, DL, DestReg, SrcReg, SystemZ::LR, 32, KillSrc,
788                  false);
789    return;
790  }
791
792  // Move 128-bit floating-point values between VR128 and FP128.
793  if (SystemZ::VR128BitRegClass.contains(DestReg) &&
794      SystemZ::FP128BitRegClass.contains(SrcReg)) {
795    MCRegister SrcRegHi =
796        RI.getMatchingSuperReg(RI.getSubReg(SrcReg, SystemZ::subreg_h64),
797                               SystemZ::subreg_h64, &SystemZ::VR128BitRegClass);
798    MCRegister SrcRegLo =
799        RI.getMatchingSuperReg(RI.getSubReg(SrcReg, SystemZ::subreg_l64),
800                               SystemZ::subreg_h64, &SystemZ::VR128BitRegClass);
801
802    BuildMI(MBB, MBBI, DL, get(SystemZ::VMRHG), DestReg)
803      .addReg(SrcRegHi, getKillRegState(KillSrc))
804      .addReg(SrcRegLo, getKillRegState(KillSrc));
805    return;
806  }
807  if (SystemZ::FP128BitRegClass.contains(DestReg) &&
808      SystemZ::VR128BitRegClass.contains(SrcReg)) {
809    MCRegister DestRegHi =
810        RI.getMatchingSuperReg(RI.getSubReg(DestReg, SystemZ::subreg_h64),
811                               SystemZ::subreg_h64, &SystemZ::VR128BitRegClass);
812    MCRegister DestRegLo =
813        RI.getMatchingSuperReg(RI.getSubReg(DestReg, SystemZ::subreg_l64),
814                               SystemZ::subreg_h64, &SystemZ::VR128BitRegClass);
815
816    if (DestRegHi != SrcReg)
817      copyPhysReg(MBB, MBBI, DL, DestRegHi, SrcReg, false);
818    BuildMI(MBB, MBBI, DL, get(SystemZ::VREPG), DestRegLo)
819      .addReg(SrcReg, getKillRegState(KillSrc)).addImm(1);
820    return;
821  }
822
823  // Move CC value from a GR32.
824  if (DestReg == SystemZ::CC) {
825    unsigned Opcode =
826      SystemZ::GR32BitRegClass.contains(SrcReg) ? SystemZ::TMLH : SystemZ::TMHH;
827    BuildMI(MBB, MBBI, DL, get(Opcode))
828      .addReg(SrcReg, getKillRegState(KillSrc))
829      .addImm(3 << (SystemZ::IPM_CC - 16));
830    return;
831  }
832
833  // Everything else needs only one instruction.
834  unsigned Opcode;
835  if (SystemZ::GR64BitRegClass.contains(DestReg, SrcReg))
836    Opcode = SystemZ::LGR;
837  else if (SystemZ::FP32BitRegClass.contains(DestReg, SrcReg))
838    // For z13 we prefer LDR over LER to avoid partial register dependencies.
839    Opcode = STI.hasVector() ? SystemZ::LDR32 : SystemZ::LER;
840  else if (SystemZ::FP64BitRegClass.contains(DestReg, SrcReg))
841    Opcode = SystemZ::LDR;
842  else if (SystemZ::FP128BitRegClass.contains(DestReg, SrcReg))
843    Opcode = SystemZ::LXR;
844  else if (SystemZ::VR32BitRegClass.contains(DestReg, SrcReg))
845    Opcode = SystemZ::VLR32;
846  else if (SystemZ::VR64BitRegClass.contains(DestReg, SrcReg))
847    Opcode = SystemZ::VLR64;
848  else if (SystemZ::VR128BitRegClass.contains(DestReg, SrcReg))
849    Opcode = SystemZ::VLR;
850  else if (SystemZ::AR32BitRegClass.contains(DestReg, SrcReg))
851    Opcode = SystemZ::CPYA;
852  else
853    llvm_unreachable("Impossible reg-to-reg copy");
854
855  BuildMI(MBB, MBBI, DL, get(Opcode), DestReg)
856    .addReg(SrcReg, getKillRegState(KillSrc));
857}
858
859void SystemZInstrInfo::storeRegToStackSlot(
860    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg,
861    bool isKill, int FrameIdx, const TargetRegisterClass *RC,
862    const TargetRegisterInfo *TRI) const {
863  DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
864
865  // Callers may expect a single instruction, so keep 128-bit moves
866  // together for now and lower them after register allocation.
867  unsigned LoadOpcode, StoreOpcode;
868  getLoadStoreOpcodes(RC, LoadOpcode, StoreOpcode);
869  addFrameReference(BuildMI(MBB, MBBI, DL, get(StoreOpcode))
870                        .addReg(SrcReg, getKillRegState(isKill)),
871                    FrameIdx);
872}
873
874void SystemZInstrInfo::loadRegFromStackSlot(
875    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg,
876    int FrameIdx, const TargetRegisterClass *RC,
877    const TargetRegisterInfo *TRI) const {
878  DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
879
880  // Callers may expect a single instruction, so keep 128-bit moves
881  // together for now and lower them after register allocation.
882  unsigned LoadOpcode, StoreOpcode;
883  getLoadStoreOpcodes(RC, LoadOpcode, StoreOpcode);
884  addFrameReference(BuildMI(MBB, MBBI, DL, get(LoadOpcode), DestReg),
885                    FrameIdx);
886}
887
888// Return true if MI is a simple load or store with a 12-bit displacement
889// and no index.  Flag is SimpleBDXLoad for loads and SimpleBDXStore for stores.
890static bool isSimpleBD12Move(const MachineInstr *MI, unsigned Flag) {
891  const MCInstrDesc &MCID = MI->getDesc();
892  return ((MCID.TSFlags & Flag) &&
893          isUInt<12>(MI->getOperand(2).getImm()) &&
894          MI->getOperand(3).getReg() == 0);
895}
896
897namespace {
898
899struct LogicOp {
900  LogicOp() = default;
901  LogicOp(unsigned regSize, unsigned immLSB, unsigned immSize)
902    : RegSize(regSize), ImmLSB(immLSB), ImmSize(immSize) {}
903
904  explicit operator bool() const { return RegSize; }
905
906  unsigned RegSize = 0;
907  unsigned ImmLSB = 0;
908  unsigned ImmSize = 0;
909};
910
911} // end anonymous namespace
912
913static LogicOp interpretAndImmediate(unsigned Opcode) {
914  switch (Opcode) {
915  case SystemZ::NILMux: return LogicOp(32,  0, 16);
916  case SystemZ::NIHMux: return LogicOp(32, 16, 16);
917  case SystemZ::NILL64: return LogicOp(64,  0, 16);
918  case SystemZ::NILH64: return LogicOp(64, 16, 16);
919  case SystemZ::NIHL64: return LogicOp(64, 32, 16);
920  case SystemZ::NIHH64: return LogicOp(64, 48, 16);
921  case SystemZ::NIFMux: return LogicOp(32,  0, 32);
922  case SystemZ::NILF64: return LogicOp(64,  0, 32);
923  case SystemZ::NIHF64: return LogicOp(64, 32, 32);
924  default:              return LogicOp();
925  }
926}
927
928static void transferDeadCC(MachineInstr *OldMI, MachineInstr *NewMI) {
929  if (OldMI->registerDefIsDead(SystemZ::CC)) {
930    MachineOperand *CCDef = NewMI->findRegisterDefOperand(SystemZ::CC);
931    if (CCDef != nullptr)
932      CCDef->setIsDead(true);
933  }
934}
935
936static void transferMIFlag(MachineInstr *OldMI, MachineInstr *NewMI,
937                           MachineInstr::MIFlag Flag) {
938  if (OldMI->getFlag(Flag))
939    NewMI->setFlag(Flag);
940}
941
942MachineInstr *SystemZInstrInfo::convertToThreeAddress(
943    MachineFunction::iterator &MFI, MachineInstr &MI, LiveVariables *LV) const {
944  MachineBasicBlock *MBB = MI.getParent();
945
946  // Try to convert an AND into an RISBG-type instruction.
947  // TODO: It might be beneficial to select RISBG and shorten to AND instead.
948  if (LogicOp And = interpretAndImmediate(MI.getOpcode())) {
949    uint64_t Imm = MI.getOperand(2).getImm() << And.ImmLSB;
950    // AND IMMEDIATE leaves the other bits of the register unchanged.
951    Imm |= allOnes(And.RegSize) & ~(allOnes(And.ImmSize) << And.ImmLSB);
952    unsigned Start, End;
953    if (isRxSBGMask(Imm, And.RegSize, Start, End)) {
954      unsigned NewOpcode;
955      if (And.RegSize == 64) {
956        NewOpcode = SystemZ::RISBG;
957        // Prefer RISBGN if available, since it does not clobber CC.
958        if (STI.hasMiscellaneousExtensions())
959          NewOpcode = SystemZ::RISBGN;
960      } else {
961        NewOpcode = SystemZ::RISBMux;
962        Start &= 31;
963        End &= 31;
964      }
965      MachineOperand &Dest = MI.getOperand(0);
966      MachineOperand &Src = MI.getOperand(1);
967      MachineInstrBuilder MIB =
968          BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpcode))
969              .add(Dest)
970              .addReg(0)
971              .addReg(Src.getReg(), getKillRegState(Src.isKill()),
972                      Src.getSubReg())
973              .addImm(Start)
974              .addImm(End + 128)
975              .addImm(0);
976      if (LV) {
977        unsigned NumOps = MI.getNumOperands();
978        for (unsigned I = 1; I < NumOps; ++I) {
979          MachineOperand &Op = MI.getOperand(I);
980          if (Op.isReg() && Op.isKill())
981            LV->replaceKillInstruction(Op.getReg(), MI, *MIB);
982        }
983      }
984      transferDeadCC(&MI, MIB);
985      return MIB;
986    }
987  }
988  return nullptr;
989}
990
991MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl(
992    MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
993    MachineBasicBlock::iterator InsertPt, int FrameIndex,
994    LiveIntervals *LIS, VirtRegMap *VRM) const {
995  const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
996  MachineRegisterInfo &MRI = MF.getRegInfo();
997  const MachineFrameInfo &MFI = MF.getFrameInfo();
998  unsigned Size = MFI.getObjectSize(FrameIndex);
999  unsigned Opcode = MI.getOpcode();
1000
1001  // Check CC liveness if new instruction introduces a dead def of CC.
1002  MCRegUnitIterator CCUnit(SystemZ::CC, TRI);
1003  SlotIndex MISlot = SlotIndex();
1004  LiveRange *CCLiveRange = nullptr;
1005  bool CCLiveAtMI = true;
1006  if (LIS) {
1007    MISlot = LIS->getSlotIndexes()->getInstructionIndex(MI).getRegSlot();
1008    CCLiveRange = &LIS->getRegUnit(*CCUnit);
1009    CCLiveAtMI = CCLiveRange->liveAt(MISlot);
1010  }
1011  ++CCUnit;
1012  assert(!CCUnit.isValid() && "CC only has one reg unit.");
1013
1014  if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) {
1015    if (!CCLiveAtMI && (Opcode == SystemZ::LA || Opcode == SystemZ::LAY) &&
1016        isInt<8>(MI.getOperand(2).getImm()) && !MI.getOperand(3).getReg()) {
1017      // LA(Y) %reg, CONST(%reg) -> AGSI %mem, CONST
1018      MachineInstr *BuiltMI = BuildMI(*InsertPt->getParent(), InsertPt,
1019                                      MI.getDebugLoc(), get(SystemZ::AGSI))
1020        .addFrameIndex(FrameIndex)
1021        .addImm(0)
1022        .addImm(MI.getOperand(2).getImm());
1023      BuiltMI->findRegisterDefOperand(SystemZ::CC)->setIsDead(true);
1024      CCLiveRange->createDeadDef(MISlot, LIS->getVNInfoAllocator());
1025      return BuiltMI;
1026    }
1027    return nullptr;
1028  }
1029
1030  // All other cases require a single operand.
1031  if (Ops.size() != 1)
1032    return nullptr;
1033
1034  unsigned OpNum = Ops[0];
1035  assert(Size * 8 ==
1036           TRI->getRegSizeInBits(*MF.getRegInfo()
1037                               .getRegClass(MI.getOperand(OpNum).getReg())) &&
1038         "Invalid size combination");
1039
1040  if ((Opcode == SystemZ::AHI || Opcode == SystemZ::AGHI) && OpNum == 0 &&
1041      isInt<8>(MI.getOperand(2).getImm())) {
1042    // A(G)HI %reg, CONST -> A(G)SI %mem, CONST
1043    Opcode = (Opcode == SystemZ::AHI ? SystemZ::ASI : SystemZ::AGSI);
1044    MachineInstr *BuiltMI =
1045        BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), get(Opcode))
1046            .addFrameIndex(FrameIndex)
1047            .addImm(0)
1048            .addImm(MI.getOperand(2).getImm());
1049    transferDeadCC(&MI, BuiltMI);
1050    transferMIFlag(&MI, BuiltMI, MachineInstr::NoSWrap);
1051    return BuiltMI;
1052  }
1053
1054  if ((Opcode == SystemZ::ALFI && OpNum == 0 &&
1055       isInt<8>((int32_t)MI.getOperand(2).getImm())) ||
1056      (Opcode == SystemZ::ALGFI && OpNum == 0 &&
1057       isInt<8>((int64_t)MI.getOperand(2).getImm()))) {
1058    // AL(G)FI %reg, CONST -> AL(G)SI %mem, CONST
1059    Opcode = (Opcode == SystemZ::ALFI ? SystemZ::ALSI : SystemZ::ALGSI);
1060    MachineInstr *BuiltMI =
1061        BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), get(Opcode))
1062            .addFrameIndex(FrameIndex)
1063            .addImm(0)
1064            .addImm((int8_t)MI.getOperand(2).getImm());
1065    transferDeadCC(&MI, BuiltMI);
1066    return BuiltMI;
1067  }
1068
1069  if ((Opcode == SystemZ::SLFI && OpNum == 0 &&
1070       isInt<8>((int32_t)-MI.getOperand(2).getImm())) ||
1071      (Opcode == SystemZ::SLGFI && OpNum == 0 &&
1072       isInt<8>((int64_t)-MI.getOperand(2).getImm()))) {
1073    // SL(G)FI %reg, CONST -> AL(G)SI %mem, -CONST
1074    Opcode = (Opcode == SystemZ::SLFI ? SystemZ::ALSI : SystemZ::ALGSI);
1075    MachineInstr *BuiltMI =
1076        BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), get(Opcode))
1077            .addFrameIndex(FrameIndex)
1078            .addImm(0)
1079            .addImm((int8_t)-MI.getOperand(2).getImm());
1080    transferDeadCC(&MI, BuiltMI);
1081    return BuiltMI;
1082  }
1083
1084  unsigned MemImmOpc = 0;
1085  switch (Opcode) {
1086  case SystemZ::LHIMux:
1087  case SystemZ::LHI:    MemImmOpc = SystemZ::MVHI;  break;
1088  case SystemZ::LGHI:   MemImmOpc = SystemZ::MVGHI; break;
1089  case SystemZ::CHIMux:
1090  case SystemZ::CHI:    MemImmOpc = SystemZ::CHSI;  break;
1091  case SystemZ::CGHI:   MemImmOpc = SystemZ::CGHSI; break;
1092  case SystemZ::CLFIMux:
1093  case SystemZ::CLFI:
1094    if (isUInt<16>(MI.getOperand(1).getImm()))
1095      MemImmOpc = SystemZ::CLFHSI;
1096    break;
1097  case SystemZ::CLGFI:
1098    if (isUInt<16>(MI.getOperand(1).getImm()))
1099      MemImmOpc = SystemZ::CLGHSI;
1100    break;
1101  default: break;
1102  }
1103  if (MemImmOpc)
1104    return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(),
1105                   get(MemImmOpc))
1106               .addFrameIndex(FrameIndex)
1107               .addImm(0)
1108               .addImm(MI.getOperand(1).getImm());
1109
1110  if (Opcode == SystemZ::LGDR || Opcode == SystemZ::LDGR) {
1111    bool Op0IsGPR = (Opcode == SystemZ::LGDR);
1112    bool Op1IsGPR = (Opcode == SystemZ::LDGR);
1113    // If we're spilling the destination of an LDGR or LGDR, store the
1114    // source register instead.
1115    if (OpNum == 0) {
1116      unsigned StoreOpcode = Op1IsGPR ? SystemZ::STG : SystemZ::STD;
1117      return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(),
1118                     get(StoreOpcode))
1119          .add(MI.getOperand(1))
1120          .addFrameIndex(FrameIndex)
1121          .addImm(0)
1122          .addReg(0);
1123    }
1124    // If we're spilling the source of an LDGR or LGDR, load the
1125    // destination register instead.
1126    if (OpNum == 1) {
1127      unsigned LoadOpcode = Op0IsGPR ? SystemZ::LG : SystemZ::LD;
1128      return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(),
1129                     get(LoadOpcode))
1130        .add(MI.getOperand(0))
1131        .addFrameIndex(FrameIndex)
1132        .addImm(0)
1133        .addReg(0);
1134    }
1135  }
1136
1137  // Look for cases where the source of a simple store or the destination
1138  // of a simple load is being spilled.  Try to use MVC instead.
1139  //
1140  // Although MVC is in practice a fast choice in these cases, it is still
1141  // logically a bytewise copy.  This means that we cannot use it if the
1142  // load or store is volatile.  We also wouldn't be able to use MVC if
1143  // the two memories partially overlap, but that case cannot occur here,
1144  // because we know that one of the memories is a full frame index.
1145  //
1146  // For performance reasons, we also want to avoid using MVC if the addresses
1147  // might be equal.  We don't worry about that case here, because spill slot
1148  // coloring happens later, and because we have special code to remove
1149  // MVCs that turn out to be redundant.
1150  if (OpNum == 0 && MI.hasOneMemOperand()) {
1151    MachineMemOperand *MMO = *MI.memoperands_begin();
1152    if (MMO->getSize() == Size && !MMO->isVolatile() && !MMO->isAtomic()) {
1153      // Handle conversion of loads.
1154      if (isSimpleBD12Move(&MI, SystemZII::SimpleBDXLoad)) {
1155        return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(),
1156                       get(SystemZ::MVC))
1157            .addFrameIndex(FrameIndex)
1158            .addImm(0)
1159            .addImm(Size)
1160            .add(MI.getOperand(1))
1161            .addImm(MI.getOperand(2).getImm())
1162            .addMemOperand(MMO);
1163      }
1164      // Handle conversion of stores.
1165      if (isSimpleBD12Move(&MI, SystemZII::SimpleBDXStore)) {
1166        return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(),
1167                       get(SystemZ::MVC))
1168            .add(MI.getOperand(1))
1169            .addImm(MI.getOperand(2).getImm())
1170            .addImm(Size)
1171            .addFrameIndex(FrameIndex)
1172            .addImm(0)
1173            .addMemOperand(MMO);
1174      }
1175    }
1176  }
1177
1178  // If the spilled operand is the final one or the instruction is
1179  // commutable, try to change <INSN>R into <INSN>.  Don't introduce a def of
1180  // CC if it is live and MI does not define it.
1181  unsigned NumOps = MI.getNumExplicitOperands();
1182  int MemOpcode = SystemZ::getMemOpcode(Opcode);
1183  if (MemOpcode == -1 ||
1184      (CCLiveAtMI && !MI.definesRegister(SystemZ::CC) &&
1185       get(MemOpcode).hasImplicitDefOfPhysReg(SystemZ::CC)))
1186    return nullptr;
1187
1188  // Check if all other vregs have a usable allocation in the case of vector
1189  // to FP conversion.
1190  const MCInstrDesc &MCID = MI.getDesc();
1191  for (unsigned I = 0, E = MCID.getNumOperands(); I != E; ++I) {
1192    const MCOperandInfo &MCOI = MCID.OpInfo[I];
1193    if (MCOI.OperandType != MCOI::OPERAND_REGISTER || I == OpNum)
1194      continue;
1195    const TargetRegisterClass *RC = TRI->getRegClass(MCOI.RegClass);
1196    if (RC == &SystemZ::VR32BitRegClass || RC == &SystemZ::VR64BitRegClass) {
1197      Register Reg = MI.getOperand(I).getReg();
1198      Register PhysReg = Register::isVirtualRegister(Reg)
1199                             ? (VRM ? VRM->getPhys(Reg) : Register())
1200                             : Reg;
1201      if (!PhysReg ||
1202          !(SystemZ::FP32BitRegClass.contains(PhysReg) ||
1203            SystemZ::FP64BitRegClass.contains(PhysReg) ||
1204            SystemZ::VF128BitRegClass.contains(PhysReg)))
1205        return nullptr;
1206    }
1207  }
1208  // Fused multiply and add/sub need to have the same dst and accumulator reg.
1209  bool FusedFPOp = (Opcode == SystemZ::WFMADB || Opcode == SystemZ::WFMASB ||
1210                    Opcode == SystemZ::WFMSDB || Opcode == SystemZ::WFMSSB);
1211  if (FusedFPOp) {
1212    Register DstReg = VRM->getPhys(MI.getOperand(0).getReg());
1213    Register AccReg = VRM->getPhys(MI.getOperand(3).getReg());
1214    if (OpNum == 0 || OpNum == 3 || DstReg != AccReg)
1215      return nullptr;
1216  }
1217
1218  // Try to swap compare operands if possible.
1219  bool NeedsCommute = false;
1220  if ((MI.getOpcode() == SystemZ::CR || MI.getOpcode() == SystemZ::CGR ||
1221       MI.getOpcode() == SystemZ::CLR || MI.getOpcode() == SystemZ::CLGR ||
1222       MI.getOpcode() == SystemZ::WFCDB || MI.getOpcode() == SystemZ::WFCSB ||
1223       MI.getOpcode() == SystemZ::WFKDB || MI.getOpcode() == SystemZ::WFKSB) &&
1224      OpNum == 0 && prepareCompareSwapOperands(MI))
1225    NeedsCommute = true;
1226
1227  bool CCOperands = false;
1228  if (MI.getOpcode() == SystemZ::LOCRMux || MI.getOpcode() == SystemZ::LOCGR ||
1229      MI.getOpcode() == SystemZ::SELRMux || MI.getOpcode() == SystemZ::SELGR) {
1230    assert(MI.getNumOperands() == 6 && NumOps == 5 &&
1231           "LOCR/SELR instruction operands corrupt?");
1232    NumOps -= 2;
1233    CCOperands = true;
1234  }
1235
1236  // See if this is a 3-address instruction that is convertible to 2-address
1237  // and suitable for folding below.  Only try this with virtual registers
1238  // and a provided VRM (during regalloc).
1239  if (NumOps == 3 && SystemZ::getTargetMemOpcode(MemOpcode) != -1) {
1240    if (VRM == nullptr)
1241      return nullptr;
1242    else {
1243      Register DstReg = MI.getOperand(0).getReg();
1244      Register DstPhys =
1245          (Register::isVirtualRegister(DstReg) ? VRM->getPhys(DstReg) : DstReg);
1246      Register SrcReg = (OpNum == 2 ? MI.getOperand(1).getReg()
1247                                    : ((OpNum == 1 && MI.isCommutable())
1248                                           ? MI.getOperand(2).getReg()
1249                                           : Register()));
1250      if (DstPhys && !SystemZ::GRH32BitRegClass.contains(DstPhys) && SrcReg &&
1251          Register::isVirtualRegister(SrcReg) &&
1252          DstPhys == VRM->getPhys(SrcReg))
1253        NeedsCommute = (OpNum == 1);
1254      else
1255        return nullptr;
1256    }
1257  }
1258
1259  if ((OpNum == NumOps - 1) || NeedsCommute || FusedFPOp) {
1260    const MCInstrDesc &MemDesc = get(MemOpcode);
1261    uint64_t AccessBytes = SystemZII::getAccessSize(MemDesc.TSFlags);
1262    assert(AccessBytes != 0 && "Size of access should be known");
1263    assert(AccessBytes <= Size && "Access outside the frame index");
1264    uint64_t Offset = Size - AccessBytes;
1265    MachineInstrBuilder MIB = BuildMI(*InsertPt->getParent(), InsertPt,
1266                                      MI.getDebugLoc(), get(MemOpcode));
1267    if (MI.isCompare()) {
1268      assert(NumOps == 2 && "Expected 2 register operands for a compare.");
1269      MIB.add(MI.getOperand(NeedsCommute ? 1 : 0));
1270    }
1271    else if (FusedFPOp) {
1272      MIB.add(MI.getOperand(0));
1273      MIB.add(MI.getOperand(3));
1274      MIB.add(MI.getOperand(OpNum == 1 ? 2 : 1));
1275    }
1276    else {
1277      MIB.add(MI.getOperand(0));
1278      if (NeedsCommute)
1279        MIB.add(MI.getOperand(2));
1280      else
1281        for (unsigned I = 1; I < OpNum; ++I)
1282          MIB.add(MI.getOperand(I));
1283    }
1284    MIB.addFrameIndex(FrameIndex).addImm(Offset);
1285    if (MemDesc.TSFlags & SystemZII::HasIndex)
1286      MIB.addReg(0);
1287    if (CCOperands) {
1288      unsigned CCValid = MI.getOperand(NumOps).getImm();
1289      unsigned CCMask = MI.getOperand(NumOps + 1).getImm();
1290      MIB.addImm(CCValid);
1291      MIB.addImm(NeedsCommute ? CCMask ^ CCValid : CCMask);
1292    }
1293    if (MIB->definesRegister(SystemZ::CC) &&
1294        (!MI.definesRegister(SystemZ::CC) ||
1295         MI.registerDefIsDead(SystemZ::CC))) {
1296      MIB->addRegisterDead(SystemZ::CC, TRI);
1297      if (CCLiveRange)
1298        CCLiveRange->createDeadDef(MISlot, LIS->getVNInfoAllocator());
1299    }
1300    // Constrain the register classes if converted from a vector opcode. The
1301    // allocated regs are in an FP reg-class per previous check above.
1302    for (const MachineOperand &MO : MIB->operands())
1303      if (MO.isReg() && Register::isVirtualRegister(MO.getReg())) {
1304        unsigned Reg = MO.getReg();
1305        if (MRI.getRegClass(Reg) == &SystemZ::VR32BitRegClass)
1306          MRI.setRegClass(Reg, &SystemZ::FP32BitRegClass);
1307        else if (MRI.getRegClass(Reg) == &SystemZ::VR64BitRegClass)
1308          MRI.setRegClass(Reg, &SystemZ::FP64BitRegClass);
1309        else if (MRI.getRegClass(Reg) == &SystemZ::VR128BitRegClass)
1310          MRI.setRegClass(Reg, &SystemZ::VF128BitRegClass);
1311      }
1312
1313    transferDeadCC(&MI, MIB);
1314    transferMIFlag(&MI, MIB, MachineInstr::NoSWrap);
1315    transferMIFlag(&MI, MIB, MachineInstr::NoFPExcept);
1316    return MIB;
1317  }
1318
1319  return nullptr;
1320}
1321
1322MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl(
1323    MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
1324    MachineBasicBlock::iterator InsertPt, MachineInstr &LoadMI,
1325    LiveIntervals *LIS) const {
1326  return nullptr;
1327}
1328
1329bool SystemZInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
1330  switch (MI.getOpcode()) {
1331  case SystemZ::L128:
1332    splitMove(MI, SystemZ::LG);
1333    return true;
1334
1335  case SystemZ::ST128:
1336    splitMove(MI, SystemZ::STG);
1337    return true;
1338
1339  case SystemZ::LX:
1340    splitMove(MI, SystemZ::LD);
1341    return true;
1342
1343  case SystemZ::STX:
1344    splitMove(MI, SystemZ::STD);
1345    return true;
1346
1347  case SystemZ::LBMux:
1348    expandRXYPseudo(MI, SystemZ::LB, SystemZ::LBH);
1349    return true;
1350
1351  case SystemZ::LHMux:
1352    expandRXYPseudo(MI, SystemZ::LH, SystemZ::LHH);
1353    return true;
1354
1355  case SystemZ::LLCRMux:
1356    expandZExtPseudo(MI, SystemZ::LLCR, 8);
1357    return true;
1358
1359  case SystemZ::LLHRMux:
1360    expandZExtPseudo(MI, SystemZ::LLHR, 16);
1361    return true;
1362
1363  case SystemZ::LLCMux:
1364    expandRXYPseudo(MI, SystemZ::LLC, SystemZ::LLCH);
1365    return true;
1366
1367  case SystemZ::LLHMux:
1368    expandRXYPseudo(MI, SystemZ::LLH, SystemZ::LLHH);
1369    return true;
1370
1371  case SystemZ::LMux:
1372    expandRXYPseudo(MI, SystemZ::L, SystemZ::LFH);
1373    return true;
1374
1375  case SystemZ::LOCMux:
1376    expandLOCPseudo(MI, SystemZ::LOC, SystemZ::LOCFH);
1377    return true;
1378
1379  case SystemZ::LOCHIMux:
1380    expandLOCPseudo(MI, SystemZ::LOCHI, SystemZ::LOCHHI);
1381    return true;
1382
1383  case SystemZ::STCMux:
1384    expandRXYPseudo(MI, SystemZ::STC, SystemZ::STCH);
1385    return true;
1386
1387  case SystemZ::STHMux:
1388    expandRXYPseudo(MI, SystemZ::STH, SystemZ::STHH);
1389    return true;
1390
1391  case SystemZ::STMux:
1392    expandRXYPseudo(MI, SystemZ::ST, SystemZ::STFH);
1393    return true;
1394
1395  case SystemZ::STOCMux:
1396    expandLOCPseudo(MI, SystemZ::STOC, SystemZ::STOCFH);
1397    return true;
1398
1399  case SystemZ::LHIMux:
1400    expandRIPseudo(MI, SystemZ::LHI, SystemZ::IIHF, true);
1401    return true;
1402
1403  case SystemZ::IIFMux:
1404    expandRIPseudo(MI, SystemZ::IILF, SystemZ::IIHF, false);
1405    return true;
1406
1407  case SystemZ::IILMux:
1408    expandRIPseudo(MI, SystemZ::IILL, SystemZ::IIHL, false);
1409    return true;
1410
1411  case SystemZ::IIHMux:
1412    expandRIPseudo(MI, SystemZ::IILH, SystemZ::IIHH, false);
1413    return true;
1414
1415  case SystemZ::NIFMux:
1416    expandRIPseudo(MI, SystemZ::NILF, SystemZ::NIHF, false);
1417    return true;
1418
1419  case SystemZ::NILMux:
1420    expandRIPseudo(MI, SystemZ::NILL, SystemZ::NIHL, false);
1421    return true;
1422
1423  case SystemZ::NIHMux:
1424    expandRIPseudo(MI, SystemZ::NILH, SystemZ::NIHH, false);
1425    return true;
1426
1427  case SystemZ::OIFMux:
1428    expandRIPseudo(MI, SystemZ::OILF, SystemZ::OIHF, false);
1429    return true;
1430
1431  case SystemZ::OILMux:
1432    expandRIPseudo(MI, SystemZ::OILL, SystemZ::OIHL, false);
1433    return true;
1434
1435  case SystemZ::OIHMux:
1436    expandRIPseudo(MI, SystemZ::OILH, SystemZ::OIHH, false);
1437    return true;
1438
1439  case SystemZ::XIFMux:
1440    expandRIPseudo(MI, SystemZ::XILF, SystemZ::XIHF, false);
1441    return true;
1442
1443  case SystemZ::TMLMux:
1444    expandRIPseudo(MI, SystemZ::TMLL, SystemZ::TMHL, false);
1445    return true;
1446
1447  case SystemZ::TMHMux:
1448    expandRIPseudo(MI, SystemZ::TMLH, SystemZ::TMHH, false);
1449    return true;
1450
1451  case SystemZ::AHIMux:
1452    expandRIPseudo(MI, SystemZ::AHI, SystemZ::AIH, false);
1453    return true;
1454
1455  case SystemZ::AHIMuxK:
1456    expandRIEPseudo(MI, SystemZ::AHI, SystemZ::AHIK, SystemZ::AIH);
1457    return true;
1458
1459  case SystemZ::AFIMux:
1460    expandRIPseudo(MI, SystemZ::AFI, SystemZ::AIH, false);
1461    return true;
1462
1463  case SystemZ::CHIMux:
1464    expandRIPseudo(MI, SystemZ::CHI, SystemZ::CIH, false);
1465    return true;
1466
1467  case SystemZ::CFIMux:
1468    expandRIPseudo(MI, SystemZ::CFI, SystemZ::CIH, false);
1469    return true;
1470
1471  case SystemZ::CLFIMux:
1472    expandRIPseudo(MI, SystemZ::CLFI, SystemZ::CLIH, false);
1473    return true;
1474
1475  case SystemZ::CMux:
1476    expandRXYPseudo(MI, SystemZ::C, SystemZ::CHF);
1477    return true;
1478
1479  case SystemZ::CLMux:
1480    expandRXYPseudo(MI, SystemZ::CL, SystemZ::CLHF);
1481    return true;
1482
1483  case SystemZ::RISBMux: {
1484    bool DestIsHigh = SystemZ::isHighReg(MI.getOperand(0).getReg());
1485    bool SrcIsHigh = SystemZ::isHighReg(MI.getOperand(2).getReg());
1486    if (SrcIsHigh == DestIsHigh)
1487      MI.setDesc(get(DestIsHigh ? SystemZ::RISBHH : SystemZ::RISBLL));
1488    else {
1489      MI.setDesc(get(DestIsHigh ? SystemZ::RISBHL : SystemZ::RISBLH));
1490      MI.getOperand(5).setImm(MI.getOperand(5).getImm() ^ 32);
1491    }
1492    return true;
1493  }
1494
1495  case SystemZ::ADJDYNALLOC:
1496    splitAdjDynAlloc(MI);
1497    return true;
1498
1499  case TargetOpcode::LOAD_STACK_GUARD:
1500    expandLoadStackGuard(&MI);
1501    return true;
1502
1503  default:
1504    return false;
1505  }
1506}
1507
1508unsigned SystemZInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
1509  if (MI.isInlineAsm()) {
1510    const MachineFunction *MF = MI.getParent()->getParent();
1511    const char *AsmStr = MI.getOperand(0).getSymbolName();
1512    return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo());
1513  }
1514  return MI.getDesc().getSize();
1515}
1516
1517SystemZII::Branch
1518SystemZInstrInfo::getBranchInfo(const MachineInstr &MI) const {
1519  switch (MI.getOpcode()) {
1520  case SystemZ::BR:
1521  case SystemZ::BI:
1522  case SystemZ::J:
1523  case SystemZ::JG:
1524    return SystemZII::Branch(SystemZII::BranchNormal, SystemZ::CCMASK_ANY,
1525                             SystemZ::CCMASK_ANY, &MI.getOperand(0));
1526
1527  case SystemZ::BRC:
1528  case SystemZ::BRCL:
1529    return SystemZII::Branch(SystemZII::BranchNormal, MI.getOperand(0).getImm(),
1530                             MI.getOperand(1).getImm(), &MI.getOperand(2));
1531
1532  case SystemZ::BRCT:
1533  case SystemZ::BRCTH:
1534    return SystemZII::Branch(SystemZII::BranchCT, SystemZ::CCMASK_ICMP,
1535                             SystemZ::CCMASK_CMP_NE, &MI.getOperand(2));
1536
1537  case SystemZ::BRCTG:
1538    return SystemZII::Branch(SystemZII::BranchCTG, SystemZ::CCMASK_ICMP,
1539                             SystemZ::CCMASK_CMP_NE, &MI.getOperand(2));
1540
1541  case SystemZ::CIJ:
1542  case SystemZ::CRJ:
1543    return SystemZII::Branch(SystemZII::BranchC, SystemZ::CCMASK_ICMP,
1544                             MI.getOperand(2).getImm(), &MI.getOperand(3));
1545
1546  case SystemZ::CLIJ:
1547  case SystemZ::CLRJ:
1548    return SystemZII::Branch(SystemZII::BranchCL, SystemZ::CCMASK_ICMP,
1549                             MI.getOperand(2).getImm(), &MI.getOperand(3));
1550
1551  case SystemZ::CGIJ:
1552  case SystemZ::CGRJ:
1553    return SystemZII::Branch(SystemZII::BranchCG, SystemZ::CCMASK_ICMP,
1554                             MI.getOperand(2).getImm(), &MI.getOperand(3));
1555
1556  case SystemZ::CLGIJ:
1557  case SystemZ::CLGRJ:
1558    return SystemZII::Branch(SystemZII::BranchCLG, SystemZ::CCMASK_ICMP,
1559                             MI.getOperand(2).getImm(), &MI.getOperand(3));
1560
1561  case SystemZ::INLINEASM_BR:
1562    // Don't try to analyze asm goto, so pass nullptr as branch target argument.
1563    return SystemZII::Branch(SystemZII::AsmGoto, 0, 0, nullptr);
1564
1565  default:
1566    llvm_unreachable("Unrecognized branch opcode");
1567  }
1568}
1569
1570void SystemZInstrInfo::getLoadStoreOpcodes(const TargetRegisterClass *RC,
1571                                           unsigned &LoadOpcode,
1572                                           unsigned &StoreOpcode) const {
1573  if (RC == &SystemZ::GR32BitRegClass || RC == &SystemZ::ADDR32BitRegClass) {
1574    LoadOpcode = SystemZ::L;
1575    StoreOpcode = SystemZ::ST;
1576  } else if (RC == &SystemZ::GRH32BitRegClass) {
1577    LoadOpcode = SystemZ::LFH;
1578    StoreOpcode = SystemZ::STFH;
1579  } else if (RC == &SystemZ::GRX32BitRegClass) {
1580    LoadOpcode = SystemZ::LMux;
1581    StoreOpcode = SystemZ::STMux;
1582  } else if (RC == &SystemZ::GR64BitRegClass ||
1583             RC == &SystemZ::ADDR64BitRegClass) {
1584    LoadOpcode = SystemZ::LG;
1585    StoreOpcode = SystemZ::STG;
1586  } else if (RC == &SystemZ::GR128BitRegClass ||
1587             RC == &SystemZ::ADDR128BitRegClass) {
1588    LoadOpcode = SystemZ::L128;
1589    StoreOpcode = SystemZ::ST128;
1590  } else if (RC == &SystemZ::FP32BitRegClass) {
1591    LoadOpcode = SystemZ::LE;
1592    StoreOpcode = SystemZ::STE;
1593  } else if (RC == &SystemZ::FP64BitRegClass) {
1594    LoadOpcode = SystemZ::LD;
1595    StoreOpcode = SystemZ::STD;
1596  } else if (RC == &SystemZ::FP128BitRegClass) {
1597    LoadOpcode = SystemZ::LX;
1598    StoreOpcode = SystemZ::STX;
1599  } else if (RC == &SystemZ::VR32BitRegClass) {
1600    LoadOpcode = SystemZ::VL32;
1601    StoreOpcode = SystemZ::VST32;
1602  } else if (RC == &SystemZ::VR64BitRegClass) {
1603    LoadOpcode = SystemZ::VL64;
1604    StoreOpcode = SystemZ::VST64;
1605  } else if (RC == &SystemZ::VF128BitRegClass ||
1606             RC == &SystemZ::VR128BitRegClass) {
1607    LoadOpcode = SystemZ::VL;
1608    StoreOpcode = SystemZ::VST;
1609  } else
1610    llvm_unreachable("Unsupported regclass to load or store");
1611}
1612
1613unsigned SystemZInstrInfo::getOpcodeForOffset(unsigned Opcode,
1614                                              int64_t Offset) const {
1615  const MCInstrDesc &MCID = get(Opcode);
1616  int64_t Offset2 = (MCID.TSFlags & SystemZII::Is128Bit ? Offset + 8 : Offset);
1617  if (isUInt<12>(Offset) && isUInt<12>(Offset2)) {
1618    // Get the instruction to use for unsigned 12-bit displacements.
1619    int Disp12Opcode = SystemZ::getDisp12Opcode(Opcode);
1620    if (Disp12Opcode >= 0)
1621      return Disp12Opcode;
1622
1623    // All address-related instructions can use unsigned 12-bit
1624    // displacements.
1625    return Opcode;
1626  }
1627  if (isInt<20>(Offset) && isInt<20>(Offset2)) {
1628    // Get the instruction to use for signed 20-bit displacements.
1629    int Disp20Opcode = SystemZ::getDisp20Opcode(Opcode);
1630    if (Disp20Opcode >= 0)
1631      return Disp20Opcode;
1632
1633    // Check whether Opcode allows signed 20-bit displacements.
1634    if (MCID.TSFlags & SystemZII::Has20BitOffset)
1635      return Opcode;
1636  }
1637  return 0;
1638}
1639
1640unsigned SystemZInstrInfo::getLoadAndTest(unsigned Opcode) const {
1641  switch (Opcode) {
1642  case SystemZ::L:      return SystemZ::LT;
1643  case SystemZ::LY:     return SystemZ::LT;
1644  case SystemZ::LG:     return SystemZ::LTG;
1645  case SystemZ::LGF:    return SystemZ::LTGF;
1646  case SystemZ::LR:     return SystemZ::LTR;
1647  case SystemZ::LGFR:   return SystemZ::LTGFR;
1648  case SystemZ::LGR:    return SystemZ::LTGR;
1649  case SystemZ::LER:    return SystemZ::LTEBR;
1650  case SystemZ::LDR:    return SystemZ::LTDBR;
1651  case SystemZ::LXR:    return SystemZ::LTXBR;
1652  case SystemZ::LCDFR:  return SystemZ::LCDBR;
1653  case SystemZ::LPDFR:  return SystemZ::LPDBR;
1654  case SystemZ::LNDFR:  return SystemZ::LNDBR;
1655  case SystemZ::LCDFR_32:  return SystemZ::LCEBR;
1656  case SystemZ::LPDFR_32:  return SystemZ::LPEBR;
1657  case SystemZ::LNDFR_32:  return SystemZ::LNEBR;
1658  // On zEC12 we prefer to use RISBGN.  But if there is a chance to
1659  // actually use the condition code, we may turn it back into RISGB.
1660  // Note that RISBG is not really a "load-and-test" instruction,
1661  // but sets the same condition code values, so is OK to use here.
1662  case SystemZ::RISBGN: return SystemZ::RISBG;
1663  default:              return 0;
1664  }
1665}
1666
1667// Return true if Mask matches the regexp 0*1+0*, given that zero masks
1668// have already been filtered out.  Store the first set bit in LSB and
1669// the number of set bits in Length if so.
1670static bool isStringOfOnes(uint64_t Mask, unsigned &LSB, unsigned &Length) {
1671  unsigned First = findFirstSet(Mask);
1672  uint64_t Top = (Mask >> First) + 1;
1673  if ((Top & -Top) == Top) {
1674    LSB = First;
1675    Length = findFirstSet(Top);
1676    return true;
1677  }
1678  return false;
1679}
1680
1681bool SystemZInstrInfo::isRxSBGMask(uint64_t Mask, unsigned BitSize,
1682                                   unsigned &Start, unsigned &End) const {
1683  // Reject trivial all-zero masks.
1684  Mask &= allOnes(BitSize);
1685  if (Mask == 0)
1686    return false;
1687
1688  // Handle the 1+0+ or 0+1+0* cases.  Start then specifies the index of
1689  // the msb and End specifies the index of the lsb.
1690  unsigned LSB, Length;
1691  if (isStringOfOnes(Mask, LSB, Length)) {
1692    Start = 63 - (LSB + Length - 1);
1693    End = 63 - LSB;
1694    return true;
1695  }
1696
1697  // Handle the wrap-around 1+0+1+ cases.  Start then specifies the msb
1698  // of the low 1s and End specifies the lsb of the high 1s.
1699  if (isStringOfOnes(Mask ^ allOnes(BitSize), LSB, Length)) {
1700    assert(LSB > 0 && "Bottom bit must be set");
1701    assert(LSB + Length < BitSize && "Top bit must be set");
1702    Start = 63 - (LSB - 1);
1703    End = 63 - (LSB + Length);
1704    return true;
1705  }
1706
1707  return false;
1708}
1709
1710unsigned SystemZInstrInfo::getFusedCompare(unsigned Opcode,
1711                                           SystemZII::FusedCompareType Type,
1712                                           const MachineInstr *MI) const {
1713  switch (Opcode) {
1714  case SystemZ::CHI:
1715  case SystemZ::CGHI:
1716    if (!(MI && isInt<8>(MI->getOperand(1).getImm())))
1717      return 0;
1718    break;
1719  case SystemZ::CLFI:
1720  case SystemZ::CLGFI:
1721    if (!(MI && isUInt<8>(MI->getOperand(1).getImm())))
1722      return 0;
1723    break;
1724  case SystemZ::CL:
1725  case SystemZ::CLG:
1726    if (!STI.hasMiscellaneousExtensions())
1727      return 0;
1728    if (!(MI && MI->getOperand(3).getReg() == 0))
1729      return 0;
1730    break;
1731  }
1732  switch (Type) {
1733  case SystemZII::CompareAndBranch:
1734    switch (Opcode) {
1735    case SystemZ::CR:
1736      return SystemZ::CRJ;
1737    case SystemZ::CGR:
1738      return SystemZ::CGRJ;
1739    case SystemZ::CHI:
1740      return SystemZ::CIJ;
1741    case SystemZ::CGHI:
1742      return SystemZ::CGIJ;
1743    case SystemZ::CLR:
1744      return SystemZ::CLRJ;
1745    case SystemZ::CLGR:
1746      return SystemZ::CLGRJ;
1747    case SystemZ::CLFI:
1748      return SystemZ::CLIJ;
1749    case SystemZ::CLGFI:
1750      return SystemZ::CLGIJ;
1751    default:
1752      return 0;
1753    }
1754  case SystemZII::CompareAndReturn:
1755    switch (Opcode) {
1756    case SystemZ::CR:
1757      return SystemZ::CRBReturn;
1758    case SystemZ::CGR:
1759      return SystemZ::CGRBReturn;
1760    case SystemZ::CHI:
1761      return SystemZ::CIBReturn;
1762    case SystemZ::CGHI:
1763      return SystemZ::CGIBReturn;
1764    case SystemZ::CLR:
1765      return SystemZ::CLRBReturn;
1766    case SystemZ::CLGR:
1767      return SystemZ::CLGRBReturn;
1768    case SystemZ::CLFI:
1769      return SystemZ::CLIBReturn;
1770    case SystemZ::CLGFI:
1771      return SystemZ::CLGIBReturn;
1772    default:
1773      return 0;
1774    }
1775  case SystemZII::CompareAndSibcall:
1776    switch (Opcode) {
1777    case SystemZ::CR:
1778      return SystemZ::CRBCall;
1779    case SystemZ::CGR:
1780      return SystemZ::CGRBCall;
1781    case SystemZ::CHI:
1782      return SystemZ::CIBCall;
1783    case SystemZ::CGHI:
1784      return SystemZ::CGIBCall;
1785    case SystemZ::CLR:
1786      return SystemZ::CLRBCall;
1787    case SystemZ::CLGR:
1788      return SystemZ::CLGRBCall;
1789    case SystemZ::CLFI:
1790      return SystemZ::CLIBCall;
1791    case SystemZ::CLGFI:
1792      return SystemZ::CLGIBCall;
1793    default:
1794      return 0;
1795    }
1796  case SystemZII::CompareAndTrap:
1797    switch (Opcode) {
1798    case SystemZ::CR:
1799      return SystemZ::CRT;
1800    case SystemZ::CGR:
1801      return SystemZ::CGRT;
1802    case SystemZ::CHI:
1803      return SystemZ::CIT;
1804    case SystemZ::CGHI:
1805      return SystemZ::CGIT;
1806    case SystemZ::CLR:
1807      return SystemZ::CLRT;
1808    case SystemZ::CLGR:
1809      return SystemZ::CLGRT;
1810    case SystemZ::CLFI:
1811      return SystemZ::CLFIT;
1812    case SystemZ::CLGFI:
1813      return SystemZ::CLGIT;
1814    case SystemZ::CL:
1815      return SystemZ::CLT;
1816    case SystemZ::CLG:
1817      return SystemZ::CLGT;
1818    default:
1819      return 0;
1820    }
1821  }
1822  return 0;
1823}
1824
1825bool SystemZInstrInfo::
1826prepareCompareSwapOperands(MachineBasicBlock::iterator const MBBI) const {
1827  assert(MBBI->isCompare() && MBBI->getOperand(0).isReg() &&
1828         MBBI->getOperand(1).isReg() && !MBBI->mayLoad() &&
1829         "Not a compare reg/reg.");
1830
1831  MachineBasicBlock *MBB = MBBI->getParent();
1832  bool CCLive = true;
1833  SmallVector<MachineInstr *, 4> CCUsers;
1834  for (MachineBasicBlock::iterator Itr = std::next(MBBI);
1835       Itr != MBB->end(); ++Itr) {
1836    if (Itr->readsRegister(SystemZ::CC)) {
1837      unsigned Flags = Itr->getDesc().TSFlags;
1838      if ((Flags & SystemZII::CCMaskFirst) || (Flags & SystemZII::CCMaskLast))
1839        CCUsers.push_back(&*Itr);
1840      else
1841        return false;
1842    }
1843    if (Itr->definesRegister(SystemZ::CC)) {
1844      CCLive = false;
1845      break;
1846    }
1847  }
1848  if (CCLive) {
1849    LivePhysRegs LiveRegs(*MBB->getParent()->getSubtarget().getRegisterInfo());
1850    LiveRegs.addLiveOuts(*MBB);
1851    if (LiveRegs.contains(SystemZ::CC))
1852      return false;
1853  }
1854
1855  // Update all CC users.
1856  for (unsigned Idx = 0; Idx < CCUsers.size(); ++Idx) {
1857    unsigned Flags = CCUsers[Idx]->getDesc().TSFlags;
1858    unsigned FirstOpNum = ((Flags & SystemZII::CCMaskFirst) ?
1859                           0 : CCUsers[Idx]->getNumExplicitOperands() - 2);
1860    MachineOperand &CCMaskMO = CCUsers[Idx]->getOperand(FirstOpNum + 1);
1861    unsigned NewCCMask = SystemZ::reverseCCMask(CCMaskMO.getImm());
1862    CCMaskMO.setImm(NewCCMask);
1863  }
1864
1865  return true;
1866}
1867
1868unsigned SystemZ::reverseCCMask(unsigned CCMask) {
1869  return ((CCMask & SystemZ::CCMASK_CMP_EQ) |
1870          (CCMask & SystemZ::CCMASK_CMP_GT ? SystemZ::CCMASK_CMP_LT : 0) |
1871          (CCMask & SystemZ::CCMASK_CMP_LT ? SystemZ::CCMASK_CMP_GT : 0) |
1872          (CCMask & SystemZ::CCMASK_CMP_UO));
1873}
1874
1875MachineBasicBlock *SystemZ::emitBlockAfter(MachineBasicBlock *MBB) {
1876  MachineFunction &MF = *MBB->getParent();
1877  MachineBasicBlock *NewMBB = MF.CreateMachineBasicBlock(MBB->getBasicBlock());
1878  MF.insert(std::next(MachineFunction::iterator(MBB)), NewMBB);
1879  return NewMBB;
1880}
1881
1882MachineBasicBlock *SystemZ::splitBlockAfter(MachineBasicBlock::iterator MI,
1883                                            MachineBasicBlock *MBB) {
1884  MachineBasicBlock *NewMBB = emitBlockAfter(MBB);
1885  NewMBB->splice(NewMBB->begin(), MBB,
1886                 std::next(MachineBasicBlock::iterator(MI)), MBB->end());
1887  NewMBB->transferSuccessorsAndUpdatePHIs(MBB);
1888  return NewMBB;
1889}
1890
1891MachineBasicBlock *SystemZ::splitBlockBefore(MachineBasicBlock::iterator MI,
1892                                             MachineBasicBlock *MBB) {
1893  MachineBasicBlock *NewMBB = emitBlockAfter(MBB);
1894  NewMBB->splice(NewMBB->begin(), MBB, MI, MBB->end());
1895  NewMBB->transferSuccessorsAndUpdatePHIs(MBB);
1896  return NewMBB;
1897}
1898
1899unsigned SystemZInstrInfo::getLoadAndTrap(unsigned Opcode) const {
1900  if (!STI.hasLoadAndTrap())
1901    return 0;
1902  switch (Opcode) {
1903  case SystemZ::L:
1904  case SystemZ::LY:
1905    return SystemZ::LAT;
1906  case SystemZ::LG:
1907    return SystemZ::LGAT;
1908  case SystemZ::LFH:
1909    return SystemZ::LFHAT;
1910  case SystemZ::LLGF:
1911    return SystemZ::LLGFAT;
1912  case SystemZ::LLGT:
1913    return SystemZ::LLGTAT;
1914  }
1915  return 0;
1916}
1917
1918void SystemZInstrInfo::loadImmediate(MachineBasicBlock &MBB,
1919                                     MachineBasicBlock::iterator MBBI,
1920                                     unsigned Reg, uint64_t Value) const {
1921  DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
1922  unsigned Opcode;
1923  if (isInt<16>(Value))
1924    Opcode = SystemZ::LGHI;
1925  else if (SystemZ::isImmLL(Value))
1926    Opcode = SystemZ::LLILL;
1927  else if (SystemZ::isImmLH(Value)) {
1928    Opcode = SystemZ::LLILH;
1929    Value >>= 16;
1930  } else {
1931    assert(isInt<32>(Value) && "Huge values not handled yet");
1932    Opcode = SystemZ::LGFI;
1933  }
1934  BuildMI(MBB, MBBI, DL, get(Opcode), Reg).addImm(Value);
1935}
1936
1937bool SystemZInstrInfo::verifyInstruction(const MachineInstr &MI,
1938                                         StringRef &ErrInfo) const {
1939  const MCInstrDesc &MCID = MI.getDesc();
1940  for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
1941    if (I >= MCID.getNumOperands())
1942      break;
1943    const MachineOperand &Op = MI.getOperand(I);
1944    const MCOperandInfo &MCOI = MCID.OpInfo[I];
1945    // Addressing modes have register and immediate operands. Op should be a
1946    // register (or frame index) operand if MCOI.RegClass contains a valid
1947    // register class, or an immediate otherwise.
1948    if (MCOI.OperandType == MCOI::OPERAND_MEMORY &&
1949        ((MCOI.RegClass != -1 && !Op.isReg() && !Op.isFI()) ||
1950         (MCOI.RegClass == -1 && !Op.isImm()))) {
1951      ErrInfo = "Addressing mode operands corrupt!";
1952      return false;
1953    }
1954  }
1955
1956  return true;
1957}
1958
1959bool SystemZInstrInfo::
1960areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
1961                                const MachineInstr &MIb) const {
1962
1963  if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand())
1964    return false;
1965
1966  // If mem-operands show that the same address Value is used by both
1967  // instructions, check for non-overlapping offsets and widths. Not
1968  // sure if a register based analysis would be an improvement...
1969
1970  MachineMemOperand *MMOa = *MIa.memoperands_begin();
1971  MachineMemOperand *MMOb = *MIb.memoperands_begin();
1972  const Value *VALa = MMOa->getValue();
1973  const Value *VALb = MMOb->getValue();
1974  bool SameVal = (VALa && VALb && (VALa == VALb));
1975  if (!SameVal) {
1976    const PseudoSourceValue *PSVa = MMOa->getPseudoValue();
1977    const PseudoSourceValue *PSVb = MMOb->getPseudoValue();
1978    if (PSVa && PSVb && (PSVa == PSVb))
1979      SameVal = true;
1980  }
1981  if (SameVal) {
1982    int OffsetA = MMOa->getOffset(), OffsetB = MMOb->getOffset();
1983    int WidthA = MMOa->getSize(), WidthB = MMOb->getSize();
1984    int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
1985    int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
1986    int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
1987    if (LowOffset + LowWidth <= HighOffset)
1988      return true;
1989  }
1990
1991  return false;
1992}
1993