1193323Sed//===-- PPCISelDAGToDAG.cpp - PPC --pattern matching inst selector --------===// 2193323Sed// 3353358Sdim// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4353358Sdim// See https://llvm.org/LICENSE.txt for license information. 5353358Sdim// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6193323Sed// 7193323Sed//===----------------------------------------------------------------------===// 8193323Sed// 9193323Sed// This file defines a pattern matching instruction selector for PowerPC, 10193323Sed// converting from a legalized dag to a PPC dag. 11193323Sed// 12193323Sed//===----------------------------------------------------------------------===// 13193323Sed 14321369Sdim#include "MCTargetDesc/PPCMCTargetDesc.h" 15321369Sdim#include "MCTargetDesc/PPCPredicates.h" 16193323Sed#include "PPC.h" 17321369Sdim#include "PPCISelLowering.h" 18270147Srdivacky#include "PPCMachineFunctionInfo.h" 19321369Sdim#include "PPCSubtarget.h" 20193323Sed#include "PPCTargetMachine.h" 21321369Sdim#include "llvm/ADT/APInt.h" 22321369Sdim#include "llvm/ADT/DenseMap.h" 23321369Sdim#include "llvm/ADT/STLExtras.h" 24321369Sdim#include "llvm/ADT/SmallPtrSet.h" 25321369Sdim#include "llvm/ADT/SmallVector.h" 26321369Sdim#include "llvm/ADT/Statistic.h" 27296417Sdim#include "llvm/Analysis/BranchProbabilityInfo.h" 28296417Sdim#include "llvm/CodeGen/FunctionLoweringInfo.h" 29321369Sdim#include "llvm/CodeGen/ISDOpcodes.h" 30321369Sdim#include "llvm/CodeGen/MachineBasicBlock.h" 31249423Sdim#include "llvm/CodeGen/MachineFunction.h" 32193323Sed#include "llvm/CodeGen/MachineInstrBuilder.h" 33193323Sed#include "llvm/CodeGen/MachineRegisterInfo.h" 34193323Sed#include "llvm/CodeGen/SelectionDAG.h" 35193323Sed#include "llvm/CodeGen/SelectionDAGISel.h" 36321369Sdim#include "llvm/CodeGen/SelectionDAGNodes.h" 37327952Sdim#include "llvm/CodeGen/TargetInstrInfo.h" 38327952Sdim#include "llvm/CodeGen/TargetRegisterInfo.h" 39321369Sdim#include "llvm/CodeGen/ValueTypes.h" 40321369Sdim#include "llvm/IR/BasicBlock.h" 41321369Sdim#include "llvm/IR/DebugLoc.h" 42249423Sdim#include "llvm/IR/Function.h" 43249423Sdim#include "llvm/IR/GlobalValue.h" 44321369Sdim#include "llvm/IR/InlineAsm.h" 45321369Sdim#include "llvm/IR/InstrTypes.h" 46276479Sdim#include "llvm/IR/Module.h" 47321369Sdim#include "llvm/Support/Casting.h" 48321369Sdim#include "llvm/Support/CodeGen.h" 49276479Sdim#include "llvm/Support/CommandLine.h" 50321369Sdim#include "llvm/Support/Compiler.h" 51193323Sed#include "llvm/Support/Debug.h" 52249423Sdim#include "llvm/Support/ErrorHandling.h" 53321369Sdim#include "llvm/Support/KnownBits.h" 54341825Sdim#include "llvm/Support/MachineValueType.h" 55193323Sed#include "llvm/Support/MathExtras.h" 56198090Srdivacky#include "llvm/Support/raw_ostream.h" 57321369Sdim#include <algorithm> 58321369Sdim#include <cassert> 59321369Sdim#include <cstdint> 60321369Sdim#include <iterator> 61321369Sdim#include <limits> 62321369Sdim#include <memory> 63321369Sdim#include <new> 64321369Sdim#include <tuple> 65321369Sdim#include <utility> 66321369Sdim 67193323Sedusing namespace llvm; 68193323Sed 69276479Sdim#define DEBUG_TYPE "ppc-codegen" 70276479Sdim 71321369SdimSTATISTIC(NumSextSetcc, 72321369Sdim "Number of (sext(setcc)) nodes expanded into GPR sequence."); 73321369SdimSTATISTIC(NumZextSetcc, 74321369Sdim "Number of (zext(setcc)) nodes expanded into GPR sequence."); 75321369SdimSTATISTIC(SignExtensionsAdded, 76321369Sdim "Number of sign extensions for compare inputs added."); 77321369SdimSTATISTIC(ZeroExtensionsAdded, 78321369Sdim "Number of zero extensions for compare inputs added."); 79321369SdimSTATISTIC(NumLogicOpsOnComparison, 80321369Sdim "Number of logical ops on i1 values calculated in GPR."); 81321369SdimSTATISTIC(OmittedForNonExtendUses, 82321369Sdim "Number of compares not eliminated as they have non-extending uses."); 83344779SdimSTATISTIC(NumP9Setb, 84344779Sdim "Number of compares lowered to setb."); 85321369Sdim 86276479Sdim// FIXME: Remove this once the bug has been fixed! 87276479Sdimcl::opt<bool> ANDIGlueBug("expose-ppc-andi-glue-bug", 88276479Sdimcl::desc("expose the ANDI glue bug on PPC"), cl::Hidden); 89276479Sdim 90288943Sdimstatic cl::opt<bool> 91288943Sdim UseBitPermRewriter("ppc-use-bit-perm-rewriter", cl::init(true), 92288943Sdim cl::desc("use aggressive ppc isel for bit permutations"), 93288943Sdim cl::Hidden); 94288943Sdimstatic cl::opt<bool> BPermRewriterNoMasking( 95288943Sdim "ppc-bit-perm-rewriter-stress-rotates", 96288943Sdim cl::desc("stress rotate selection in aggressive ppc isel for " 97288943Sdim "bit permutations"), 98288943Sdim cl::Hidden); 99280031Sdim 100296417Sdimstatic cl::opt<bool> EnableBranchHint( 101296417Sdim "ppc-use-branch-hint", cl::init(true), 102296417Sdim cl::desc("Enable static hinting of branches on ppc"), 103296417Sdim cl::Hidden); 104296417Sdim 105341825Sdimstatic cl::opt<bool> EnableTLSOpt( 106341825Sdim "ppc-tls-opt", cl::init(true), 107341825Sdim cl::desc("Enable tls optimization peephole"), 108341825Sdim cl::Hidden); 109341825Sdim 110327952Sdimenum ICmpInGPRType { ICGPR_All, ICGPR_None, ICGPR_I32, ICGPR_I64, 111327952Sdim ICGPR_NonExtIn, ICGPR_Zext, ICGPR_Sext, ICGPR_ZextI32, 112327952Sdim ICGPR_SextI32, ICGPR_ZextI64, ICGPR_SextI64 }; 113327952Sdim 114327952Sdimstatic cl::opt<ICmpInGPRType> CmpInGPR( 115327952Sdim "ppc-gpr-icmps", cl::Hidden, cl::init(ICGPR_All), 116327952Sdim cl::desc("Specify the types of comparisons to emit GPR-only code for."), 117327952Sdim cl::values(clEnumValN(ICGPR_None, "none", "Do not modify integer comparisons."), 118327952Sdim clEnumValN(ICGPR_All, "all", "All possible int comparisons in GPRs."), 119327952Sdim clEnumValN(ICGPR_I32, "i32", "Only i32 comparisons in GPRs."), 120327952Sdim clEnumValN(ICGPR_I64, "i64", "Only i64 comparisons in GPRs."), 121327952Sdim clEnumValN(ICGPR_NonExtIn, "nonextin", 122327952Sdim "Only comparisons where inputs don't need [sz]ext."), 123327952Sdim clEnumValN(ICGPR_Zext, "zext", "Only comparisons with zext result."), 124327952Sdim clEnumValN(ICGPR_ZextI32, "zexti32", 125327952Sdim "Only i32 comparisons with zext result."), 126327952Sdim clEnumValN(ICGPR_ZextI64, "zexti64", 127327952Sdim "Only i64 comparisons with zext result."), 128327952Sdim clEnumValN(ICGPR_Sext, "sext", "Only comparisons with sext result."), 129327952Sdim clEnumValN(ICGPR_SextI32, "sexti32", 130327952Sdim "Only i32 comparisons with sext result."), 131327952Sdim clEnumValN(ICGPR_SextI64, "sexti64", 132327952Sdim "Only i64 comparisons with sext result."))); 133193323Sednamespace { 134321369Sdim 135193323Sed //===--------------------------------------------------------------------===// 136193323Sed /// PPCDAGToDAGISel - PPC specific code to select PPC machine 137193323Sed /// instructions for SelectionDAG operations. 138193323Sed /// 139198892Srdivacky class PPCDAGToDAGISel : public SelectionDAGISel { 140207618Srdivacky const PPCTargetMachine &TM; 141360784Sdim const PPCSubtarget *PPCSubTarget = nullptr; 142360784Sdim const PPCTargetLowering *PPCLowering = nullptr; 143360784Sdim unsigned GlobalBaseReg = 0; 144321369Sdim 145193323Sed public: 146321369Sdim explicit PPCDAGToDAGISel(PPCTargetMachine &tm, CodeGenOpt::Level OptLevel) 147321369Sdim : SelectionDAGISel(tm, OptLevel), TM(tm) {} 148218893Sdim 149276479Sdim bool runOnMachineFunction(MachineFunction &MF) override { 150193323Sed // Make sure we re-emit a set of the global base reg if necessary 151193323Sed GlobalBaseReg = 0; 152288943Sdim PPCSubTarget = &MF.getSubtarget<PPCSubtarget>(); 153288943Sdim PPCLowering = PPCSubTarget->getTargetLowering(); 154198090Srdivacky SelectionDAGISel::runOnMachineFunction(MF); 155218893Sdim 156276479Sdim if (!PPCSubTarget->isSVR4ABI()) 157243830Sdim InsertVRSaveCode(MF); 158243830Sdim 159193323Sed return true; 160193323Sed } 161218893Sdim 162280031Sdim void PreprocessISelDAG() override; 163276479Sdim void PostprocessISelDAG() override; 164249423Sdim 165327952Sdim /// getI16Imm - Return a target constant with the specified value, of type 166327952Sdim /// i16. 167327952Sdim inline SDValue getI16Imm(unsigned Imm, const SDLoc &dl) { 168327952Sdim return CurDAG->getTargetConstant(Imm, dl, MVT::i16); 169327952Sdim } 170327952Sdim 171193323Sed /// getI32Imm - Return a target constant with the specified value, of type 172193323Sed /// i32. 173309124Sdim inline SDValue getI32Imm(unsigned Imm, const SDLoc &dl) { 174288943Sdim return CurDAG->getTargetConstant(Imm, dl, MVT::i32); 175193323Sed } 176193323Sed 177193323Sed /// getI64Imm - Return a target constant with the specified value, of type 178193323Sed /// i64. 179309124Sdim inline SDValue getI64Imm(uint64_t Imm, const SDLoc &dl) { 180288943Sdim return CurDAG->getTargetConstant(Imm, dl, MVT::i64); 181193323Sed } 182218893Sdim 183193323Sed /// getSmallIPtrImm - Return a target constant of pointer type. 184309124Sdim inline SDValue getSmallIPtrImm(unsigned Imm, const SDLoc &dl) { 185288943Sdim return CurDAG->getTargetConstant( 186288943Sdim Imm, dl, PPCLowering->getPointerTy(CurDAG->getDataLayout())); 187193323Sed } 188218893Sdim 189193323Sed /// isRotateAndMask - Returns true if Mask and Shift can be folded into a 190193323Sed /// rotate and mask opcode and mask operation. 191199989Srdivacky static bool isRotateAndMask(SDNode *N, unsigned Mask, bool isShiftMask, 192193323Sed unsigned &SH, unsigned &MB, unsigned &ME); 193218893Sdim 194193323Sed /// getGlobalBaseReg - insert code into the entry mbb to materialize the PIC 195193323Sed /// base register. Return the virtual register that holds this value. 196193323Sed SDNode *getGlobalBaseReg(); 197218893Sdim 198309124Sdim void selectFrameIndex(SDNode *SN, SDNode *N, unsigned Offset = 0); 199280031Sdim 200193323Sed // Select - Convert the specified operand from a target-independent to a 201193323Sed // target-specific node if it hasn't already been changed. 202309124Sdim void Select(SDNode *N) override; 203218893Sdim 204309124Sdim bool tryBitfieldInsert(SDNode *N); 205309124Sdim bool tryBitPermutation(SDNode *N); 206327952Sdim bool tryIntCompareInGPR(SDNode *N); 207360784Sdim bool tryAndWithMask(SDNode *N); 208193323Sed 209341825Sdim // tryTLSXFormLoad - Convert an ISD::LOAD fed by a PPCISD::ADD_TLS into 210341825Sdim // an X-Form load instruction with the offset being a relocation coming from 211341825Sdim // the PPCISD::ADD_TLS. 212341825Sdim bool tryTLSXFormLoad(LoadSDNode *N); 213341825Sdim // tryTLSXFormStore - Convert an ISD::STORE fed by a PPCISD::ADD_TLS into 214341825Sdim // an X-Form store instruction with the offset being a relocation coming from 215341825Sdim // the PPCISD::ADD_TLS. 216341825Sdim bool tryTLSXFormStore(StoreSDNode *N); 217193323Sed /// SelectCC - Select a comparison of the specified values with the 218193323Sed /// specified condition code, returning the CR# of the expression. 219309124Sdim SDValue SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, 220309124Sdim const SDLoc &dl); 221193323Sed 222193323Sed /// SelectAddrImmOffs - Return true if the operand is valid for a preinc 223249423Sdim /// immediate field. Note that the operand at this point is already the 224249423Sdim /// result of a prior SelectAddressRegImm call. 225218893Sdim bool SelectAddrImmOffs(SDValue N, SDValue &Out) const { 226249423Sdim if (N.getOpcode() == ISD::TargetConstant || 227239462Sdim N.getOpcode() == ISD::TargetGlobalAddress) { 228239462Sdim Out = N; 229239462Sdim return true; 230239462Sdim } 231239462Sdim 232239462Sdim return false; 233239462Sdim } 234239462Sdim 235353358Sdim /// SelectAddrIdx - Given the specified address, check to see if it can be 236353358Sdim /// represented as an indexed [r+r] operation. 237353358Sdim /// This is for xform instructions whose associated displacement form is D. 238353358Sdim /// The last parameter \p 0 means associated D form has no requirment for 16 239353358Sdim /// bit signed displacement. 240353358Sdim /// Returns false if it can be represented by [r+imm], which are preferred. 241218893Sdim bool SelectAddrIdx(SDValue N, SDValue &Base, SDValue &Index) { 242353358Sdim return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG, 0); 243193323Sed } 244218893Sdim 245353358Sdim /// SelectAddrIdx4 - Given the specified address, check to see if it can be 246193323Sed /// represented as an indexed [r+r] operation. 247353358Sdim /// This is for xform instructions whose associated displacement form is DS. 248353358Sdim /// The last parameter \p 4 means associated DS form 16 bit signed 249353358Sdim /// displacement must be a multiple of 4. 250353358Sdim /// Returns false if it can be represented by [r+imm], which are preferred. 251353358Sdim bool SelectAddrIdxX4(SDValue N, SDValue &Base, SDValue &Index) { 252353358Sdim return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG, 4); 253353358Sdim } 254353358Sdim 255353358Sdim /// SelectAddrIdx16 - Given the specified address, check to see if it can be 256353358Sdim /// represented as an indexed [r+r] operation. 257353358Sdim /// This is for xform instructions whose associated displacement form is DQ. 258353358Sdim /// The last parameter \p 16 means associated DQ form 16 bit signed 259353358Sdim /// displacement must be a multiple of 16. 260353358Sdim /// Returns false if it can be represented by [r+imm], which are preferred. 261353358Sdim bool SelectAddrIdxX16(SDValue N, SDValue &Base, SDValue &Index) { 262353358Sdim return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG, 16); 263353358Sdim } 264353358Sdim 265353358Sdim /// SelectAddrIdxOnly - Given the specified address, force it to be 266353358Sdim /// represented as an indexed [r+r] operation. 267218893Sdim bool SelectAddrIdxOnly(SDValue N, SDValue &Base, SDValue &Index) { 268276479Sdim return PPCLowering->SelectAddressRegRegOnly(N, Base, Index, *CurDAG); 269193323Sed } 270353358Sdim 271353358Sdim /// SelectAddrImm - Returns true if the address N can be represented by 272353358Sdim /// a base register plus a signed 16-bit displacement [r+imm]. 273353358Sdim /// The last parameter \p 0 means D form has no requirment for 16 bit signed 274353358Sdim /// displacement. 275353358Sdim bool SelectAddrImm(SDValue N, SDValue &Disp, 276353358Sdim SDValue &Base) { 277353358Sdim return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, 0); 278353358Sdim } 279193323Sed 280261991Sdim /// SelectAddrImmX4 - Returns true if the address N can be represented by 281353358Sdim /// a base register plus a signed 16-bit displacement that is a multiple of 282353358Sdim /// 4 (last parameter). Suitable for use by STD and friends. 283261991Sdim bool SelectAddrImmX4(SDValue N, SDValue &Disp, SDValue &Base) { 284321369Sdim return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, 4); 285193323Sed } 286218893Sdim 287353358Sdim /// SelectAddrImmX16 - Returns true if the address N can be represented by 288353358Sdim /// a base register plus a signed 16-bit displacement that is a multiple of 289353358Sdim /// 16(last parameter). Suitable for use by STXV and friends. 290321369Sdim bool SelectAddrImmX16(SDValue N, SDValue &Disp, SDValue &Base) { 291321369Sdim return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, 16); 292321369Sdim } 293321369Sdim 294249423Sdim // Select an address into a single register. 295249423Sdim bool SelectAddr(SDValue N, SDValue &Base) { 296249423Sdim Base = N; 297249423Sdim return true; 298249423Sdim } 299249423Sdim 300193323Sed /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for 301198090Srdivacky /// inline asm expressions. It is always correct to compute the value into 302198090Srdivacky /// a register. The case of adding a (possibly relocatable) constant to a 303198090Srdivacky /// register can be improved, but it is wrong to substitute Reg+Reg for 304198090Srdivacky /// Reg in an asm, because the load or store opcode would have to change. 305277320Sdim bool SelectInlineAsmMemoryOperand(const SDValue &Op, 306288943Sdim unsigned ConstraintID, 307276479Sdim std::vector<SDValue> &OutOps) override { 308288943Sdim switch(ConstraintID) { 309288943Sdim default: 310288943Sdim errs() << "ConstraintID: " << ConstraintID << "\n"; 311288943Sdim llvm_unreachable("Unexpected asm memory constraint"); 312288943Sdim case InlineAsm::Constraint_es: 313288943Sdim case InlineAsm::Constraint_m: 314288943Sdim case InlineAsm::Constraint_o: 315288943Sdim case InlineAsm::Constraint_Q: 316288943Sdim case InlineAsm::Constraint_Z: 317288943Sdim case InlineAsm::Constraint_Zy: 318288943Sdim // We need to make sure that this one operand does not end up in r0 319288943Sdim // (because we might end up lowering this as 0(%op)). 320288943Sdim const TargetRegisterInfo *TRI = PPCSubTarget->getRegisterInfo(); 321288943Sdim const TargetRegisterClass *TRC = TRI->getPointerRegClass(*MF, /*Kind=*/1); 322288943Sdim SDLoc dl(Op); 323288943Sdim SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i32); 324288943Sdim SDValue NewOp = 325288943Sdim SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, 326288943Sdim dl, Op.getValueType(), 327288943Sdim Op, RC), 0); 328288943Sdim 329288943Sdim OutOps.push_back(NewOp); 330288943Sdim return false; 331288943Sdim } 332288943Sdim return true; 333193323Sed } 334218893Sdim 335198090Srdivacky void InsertVRSaveCode(MachineFunction &MF); 336193323Sed 337314564Sdim StringRef getPassName() const override { 338193323Sed return "PowerPC DAG->DAG Pattern Instruction Selection"; 339193323Sed } 340193323Sed 341193323Sed// Include the pieces autogenerated from the target description. 342193323Sed#include "PPCGenDAGISel.inc" 343218893Sdim 344193323Sedprivate: 345309124Sdim bool trySETCC(SDNode *N); 346276479Sdim 347276479Sdim void PeepholePPC64(); 348280031Sdim void PeepholePPC64ZExt(); 349276479Sdim void PeepholeCROps(); 350276479Sdim 351280031Sdim SDValue combineToCMPB(SDNode *N); 352280031Sdim void foldBoolExts(SDValue &Res, SDNode *&N); 353280031Sdim 354276479Sdim bool AllUsersSelectZero(SDNode *N); 355276479Sdim void SwapAllSelectUsers(SDNode *N); 356288943Sdim 357321369Sdim bool isOffsetMultipleOf(SDNode *N, unsigned Val) const; 358309124Sdim void transferMemOperands(SDNode *N, SDNode *Result); 359193323Sed }; 360193323Sed 361321369Sdim} // end anonymous namespace 362321369Sdim 363193323Sed/// InsertVRSaveCode - Once the entire function has been instruction selected, 364193323Sed/// all virtual registers are created and all machine instructions are built, 365193323Sed/// check to see if we need to save/restore VRSAVE. If so, do it. 366198090Srdivackyvoid PPCDAGToDAGISel::InsertVRSaveCode(MachineFunction &Fn) { 367193323Sed // Check to see if this function uses vector registers, which means we have to 368218893Sdim // save and restore the VRSAVE register and update it with the regs we use. 369193323Sed // 370203954Srdivacky // In this case, there will be virtual registers of vector type created 371193323Sed // by the scheduler. Detect them now. 372193323Sed bool HasVectorVReg = false; 373218893Sdim for (unsigned i = 0, e = RegInfo->getNumVirtRegs(); i != e; ++i) { 374360784Sdim unsigned Reg = Register::index2VirtReg(i); 375218893Sdim if (RegInfo->getRegClass(Reg) == &PPC::VRRCRegClass) { 376193323Sed HasVectorVReg = true; 377193323Sed break; 378193323Sed } 379218893Sdim } 380193323Sed if (!HasVectorVReg) return; // nothing to do. 381218893Sdim 382193323Sed // If we have a vector register, we want to emit code into the entry and exit 383193323Sed // blocks to save and restore the VRSAVE register. We do this here (instead 384193323Sed // of marking all vector instructions as clobbering VRSAVE) for two reasons: 385193323Sed // 386193323Sed // 1. This (trivially) reduces the load on the register allocator, by not 387193323Sed // having to represent the live range of the VRSAVE register. 388193323Sed // 2. This (more significantly) allows us to create a temporary virtual 389193323Sed // register to hold the saved VRSAVE value, allowing this temporary to be 390193323Sed // register allocated, instead of forcing it to be spilled to the stack. 391193323Sed 392193323Sed // Create two vregs - one to hold the VRSAVE register that is live-in to the 393193323Sed // function and one for the value after having bits or'd into it. 394360784Sdim Register InVRSAVE = RegInfo->createVirtualRegister(&PPC::GPRCRegClass); 395360784Sdim Register UpdatedVRSAVE = RegInfo->createVirtualRegister(&PPC::GPRCRegClass); 396218893Sdim 397288943Sdim const TargetInstrInfo &TII = *PPCSubTarget->getInstrInfo(); 398193323Sed MachineBasicBlock &EntryBB = *Fn.begin(); 399206124Srdivacky DebugLoc dl; 400193323Sed // Emit the following code into the entry block: 401193323Sed // InVRSAVE = MFVRSAVE 402193323Sed // UpdatedVRSAVE = UPDATE_VRSAVE InVRSAVE 403193323Sed // MTVRSAVE UpdatedVRSAVE 404193323Sed MachineBasicBlock::iterator IP = EntryBB.begin(); // Insert Point 405193323Sed BuildMI(EntryBB, IP, dl, TII.get(PPC::MFVRSAVE), InVRSAVE); 406193323Sed BuildMI(EntryBB, IP, dl, TII.get(PPC::UPDATE_VRSAVE), 407193323Sed UpdatedVRSAVE).addReg(InVRSAVE); 408193323Sed BuildMI(EntryBB, IP, dl, TII.get(PPC::MTVRSAVE)).addReg(UpdatedVRSAVE); 409218893Sdim 410193323Sed // Find all return blocks, outputting a restore in each epilog. 411193323Sed for (MachineFunction::iterator BB = Fn.begin(), E = Fn.end(); BB != E; ++BB) { 412296417Sdim if (BB->isReturnBlock()) { 413193323Sed IP = BB->end(); --IP; 414218893Sdim 415193323Sed // Skip over all terminator instructions, which are part of the return 416193323Sed // sequence. 417193323Sed MachineBasicBlock::iterator I2 = IP; 418234353Sdim while (I2 != BB->begin() && (--I2)->isTerminator()) 419193323Sed IP = I2; 420218893Sdim 421193323Sed // Emit: MTVRSAVE InVRSave 422193323Sed BuildMI(*BB, IP, dl, TII.get(PPC::MTVRSAVE)).addReg(InVRSAVE); 423218893Sdim } 424193323Sed } 425193323Sed} 426193323Sed 427193323Sed/// getGlobalBaseReg - Output the instructions required to put the 428193323Sed/// base address to use for accessing globals into a register. 429193323Sed/// 430193323SedSDNode *PPCDAGToDAGISel::getGlobalBaseReg() { 431193323Sed if (!GlobalBaseReg) { 432288943Sdim const TargetInstrInfo &TII = *PPCSubTarget->getInstrInfo(); 433193323Sed // Insert the set of GlobalBaseReg into the first MBB of the function 434198090Srdivacky MachineBasicBlock &FirstMBB = MF->front(); 435193323Sed MachineBasicBlock::iterator MBBI = FirstMBB.begin(); 436327952Sdim const Module *M = MF->getFunction().getParent(); 437206124Srdivacky DebugLoc dl; 438193323Sed 439288943Sdim if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) == MVT::i32) { 440276479Sdim if (PPCSubTarget->isTargetELF()) { 441270147Srdivacky GlobalBaseReg = PPC::R30; 442353358Sdim if (!PPCSubTarget->isSecurePlt() && 443353358Sdim M->getPICLevel() == PICLevel::SmallPIC) { 444276479Sdim BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MoveGOTtoLR)); 445276479Sdim BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg); 446280031Sdim MF->getInfo<PPCFunctionInfo>()->setUsesPICBase(true); 447276479Sdim } else { 448276479Sdim BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR)); 449276479Sdim BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg); 450360784Sdim Register TempReg = RegInfo->createVirtualRegister(&PPC::GPRCRegClass); 451276479Sdim BuildMI(FirstMBB, MBBI, dl, 452288943Sdim TII.get(PPC::UpdateGBR), GlobalBaseReg) 453276479Sdim .addReg(TempReg, RegState::Define).addReg(GlobalBaseReg); 454276479Sdim MF->getInfo<PPCFunctionInfo>()->setUsesPICBase(true); 455276479Sdim } 456276479Sdim } else { 457270147Srdivacky GlobalBaseReg = 458314564Sdim RegInfo->createVirtualRegister(&PPC::GPRC_and_GPRC_NOR0RegClass); 459276479Sdim BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR)); 460276479Sdim BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg); 461270147Srdivacky } 462193323Sed } else { 463341825Sdim // We must ensure that this sequence is dominated by the prologue. 464341825Sdim // FIXME: This is a bit of a big hammer since we don't get the benefits 465341825Sdim // of shrink-wrapping whenever we emit this instruction. Considering 466341825Sdim // this is used in any function where we emit a jump table, this may be 467341825Sdim // a significant limitation. We should consider inserting this in the 468341825Sdim // block where it is used and then commoning this sequence up if it 469341825Sdim // appears in multiple places. 470341825Sdim // Note: on ISA 3.0 cores, we can use lnia (addpcis) instead of 471341825Sdim // MovePCtoLR8. 472341825Sdim MF->getInfo<PPCFunctionInfo>()->setShrinkWrapDisabled(true); 473314564Sdim GlobalBaseReg = RegInfo->createVirtualRegister(&PPC::G8RC_and_G8RC_NOX0RegClass); 474223017Sdim BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR8)); 475193323Sed BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR8), GlobalBaseReg); 476193323Sed } 477193323Sed } 478193323Sed return CurDAG->getRegister(GlobalBaseReg, 479288943Sdim PPCLowering->getPointerTy(CurDAG->getDataLayout())) 480288943Sdim .getNode(); 481193323Sed} 482193323Sed 483193323Sed/// isInt32Immediate - This method tests to see if the node is a 32-bit constant 484193323Sed/// operand. If so Imm will receive the 32-bit value. 485193323Sedstatic bool isInt32Immediate(SDNode *N, unsigned &Imm) { 486193323Sed if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) { 487193323Sed Imm = cast<ConstantSDNode>(N)->getZExtValue(); 488193323Sed return true; 489193323Sed } 490193323Sed return false; 491193323Sed} 492193323Sed 493193323Sed/// isInt64Immediate - This method tests to see if the node is a 64-bit constant 494193323Sed/// operand. If so Imm will receive the 64-bit value. 495193323Sedstatic bool isInt64Immediate(SDNode *N, uint64_t &Imm) { 496193323Sed if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i64) { 497193323Sed Imm = cast<ConstantSDNode>(N)->getZExtValue(); 498193323Sed return true; 499193323Sed } 500193323Sed return false; 501193323Sed} 502193323Sed 503193323Sed// isInt32Immediate - This method tests to see if a constant operand. 504193323Sed// If so Imm will receive the 32 bit value. 505193323Sedstatic bool isInt32Immediate(SDValue N, unsigned &Imm) { 506193323Sed return isInt32Immediate(N.getNode(), Imm); 507193323Sed} 508193323Sed 509327952Sdim/// isInt64Immediate - This method tests to see if the value is a 64-bit 510327952Sdim/// constant operand. If so Imm will receive the 64-bit value. 511327952Sdimstatic bool isInt64Immediate(SDValue N, uint64_t &Imm) { 512327952Sdim return isInt64Immediate(N.getNode(), Imm); 513327952Sdim} 514327952Sdim 515360784Sdimstatic unsigned getBranchHint(unsigned PCC, 516360784Sdim const FunctionLoweringInfo &FuncInfo, 517296417Sdim const SDValue &DestMBB) { 518296417Sdim assert(isa<BasicBlockSDNode>(DestMBB)); 519193323Sed 520360784Sdim if (!FuncInfo.BPI) return PPC::BR_NO_HINT; 521296417Sdim 522360784Sdim const BasicBlock *BB = FuncInfo.MBB->getBasicBlock(); 523344779Sdim const Instruction *BBTerm = BB->getTerminator(); 524296417Sdim 525296417Sdim if (BBTerm->getNumSuccessors() != 2) return PPC::BR_NO_HINT; 526296417Sdim 527296417Sdim const BasicBlock *TBB = BBTerm->getSuccessor(0); 528296417Sdim const BasicBlock *FBB = BBTerm->getSuccessor(1); 529296417Sdim 530360784Sdim auto TProb = FuncInfo.BPI->getEdgeProbability(BB, TBB); 531360784Sdim auto FProb = FuncInfo.BPI->getEdgeProbability(BB, FBB); 532296417Sdim 533296417Sdim // We only want to handle cases which are easy to predict at static time, e.g. 534296417Sdim // C++ throw statement, that is very likely not taken, or calling never 535296417Sdim // returned function, e.g. stdlib exit(). So we set Threshold to filter 536296417Sdim // unwanted cases. 537296417Sdim // 538296417Sdim // Below is LLVM branch weight table, we only want to handle case 1, 2 539296417Sdim // 540296417Sdim // Case Taken:Nontaken Example 541296417Sdim // 1. Unreachable 1048575:1 C++ throw, stdlib exit(), 542296417Sdim // 2. Invoke-terminating 1:1048575 543296417Sdim // 3. Coldblock 4:64 __builtin_expect 544296417Sdim // 4. Loop Branch 124:4 For loop 545296417Sdim // 5. PH/ZH/FPH 20:12 546296417Sdim const uint32_t Threshold = 10000; 547296417Sdim 548296417Sdim if (std::max(TProb, FProb) / Threshold < std::min(TProb, FProb)) 549296417Sdim return PPC::BR_NO_HINT; 550296417Sdim 551360784Sdim LLVM_DEBUG(dbgs() << "Use branch hint for '" << FuncInfo.Fn->getName() 552341825Sdim << "::" << BB->getName() << "'\n" 553341825Sdim << " -> " << TBB->getName() << ": " << TProb << "\n" 554341825Sdim << " -> " << FBB->getName() << ": " << FProb << "\n"); 555296417Sdim 556296417Sdim const BasicBlockSDNode *BBDN = cast<BasicBlockSDNode>(DestMBB); 557296417Sdim 558296417Sdim // If Dest BasicBlock is False-BasicBlock (FBB), swap branch probabilities, 559296417Sdim // because we want 'TProb' stands for 'branch probability' to Dest BasicBlock 560296417Sdim if (BBDN->getBasicBlock()->getBasicBlock() != TBB) 561296417Sdim std::swap(TProb, FProb); 562296417Sdim 563296417Sdim return (TProb > FProb) ? PPC::BR_TAKEN_HINT : PPC::BR_NONTAKEN_HINT; 564296417Sdim} 565296417Sdim 566193323Sed// isOpcWithIntImmediate - This method tests to see if the node is a specific 567193323Sed// opcode and that it has a immediate integer right operand. 568193323Sed// If so Imm will receive the 32 bit value. 569193323Sedstatic bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) { 570193323Sed return N->getOpcode() == Opc 571193323Sed && isInt32Immediate(N->getOperand(1).getNode(), Imm); 572193323Sed} 573193323Sed 574309124Sdimvoid PPCDAGToDAGISel::selectFrameIndex(SDNode *SN, SDNode *N, unsigned Offset) { 575280031Sdim SDLoc dl(SN); 576280031Sdim int FI = cast<FrameIndexSDNode>(N)->getIndex(); 577280031Sdim SDValue TFI = CurDAG->getTargetFrameIndex(FI, N->getValueType(0)); 578280031Sdim unsigned Opc = N->getValueType(0) == MVT::i32 ? PPC::ADDI : PPC::ADDI8; 579280031Sdim if (SN->hasOneUse()) 580309124Sdim CurDAG->SelectNodeTo(SN, Opc, N->getValueType(0), TFI, 581309124Sdim getSmallIPtrImm(Offset, dl)); 582309124Sdim else 583309124Sdim ReplaceNode(SN, CurDAG->getMachineNode(Opc, dl, N->getValueType(0), TFI, 584309124Sdim getSmallIPtrImm(Offset, dl))); 585280031Sdim} 586280031Sdim 587218893Sdimbool PPCDAGToDAGISel::isRotateAndMask(SDNode *N, unsigned Mask, 588218893Sdim bool isShiftMask, unsigned &SH, 589193323Sed unsigned &MB, unsigned &ME) { 590193323Sed // Don't even go down this path for i64, since different logic will be 591193323Sed // necessary for rldicl/rldicr/rldimi. 592193323Sed if (N->getValueType(0) != MVT::i32) 593193323Sed return false; 594193323Sed 595193323Sed unsigned Shift = 32; 596193323Sed unsigned Indeterminant = ~0; // bit mask marking indeterminant results 597193323Sed unsigned Opcode = N->getOpcode(); 598193323Sed if (N->getNumOperands() != 2 || 599193323Sed !isInt32Immediate(N->getOperand(1).getNode(), Shift) || (Shift > 31)) 600193323Sed return false; 601218893Sdim 602193323Sed if (Opcode == ISD::SHL) { 603193323Sed // apply shift left to mask if it comes first 604199989Srdivacky if (isShiftMask) Mask = Mask << Shift; 605193323Sed // determine which bits are made indeterminant by shift 606193323Sed Indeterminant = ~(0xFFFFFFFFu << Shift); 607218893Sdim } else if (Opcode == ISD::SRL) { 608193323Sed // apply shift right to mask if it comes first 609199989Srdivacky if (isShiftMask) Mask = Mask >> Shift; 610193323Sed // determine which bits are made indeterminant by shift 611193323Sed Indeterminant = ~(0xFFFFFFFFu >> Shift); 612193323Sed // adjust for the left rotate 613193323Sed Shift = 32 - Shift; 614193323Sed } else if (Opcode == ISD::ROTL) { 615193323Sed Indeterminant = 0; 616193323Sed } else { 617193323Sed return false; 618193323Sed } 619218893Sdim 620193323Sed // if the mask doesn't intersect any Indeterminant bits 621193323Sed if (Mask && !(Mask & Indeterminant)) { 622193323Sed SH = Shift & 31; 623193323Sed // make sure the mask is still a mask (wrap arounds may not be) 624193323Sed return isRunOfOnes(Mask, MB, ME); 625193323Sed } 626193323Sed return false; 627193323Sed} 628193323Sed 629341825Sdimbool PPCDAGToDAGISel::tryTLSXFormStore(StoreSDNode *ST) { 630341825Sdim SDValue Base = ST->getBasePtr(); 631341825Sdim if (Base.getOpcode() != PPCISD::ADD_TLS) 632341825Sdim return false; 633341825Sdim SDValue Offset = ST->getOffset(); 634341825Sdim if (!Offset.isUndef()) 635341825Sdim return false; 636341825Sdim 637341825Sdim SDLoc dl(ST); 638341825Sdim EVT MemVT = ST->getMemoryVT(); 639341825Sdim EVT RegVT = ST->getValue().getValueType(); 640341825Sdim 641341825Sdim unsigned Opcode; 642341825Sdim switch (MemVT.getSimpleVT().SimpleTy) { 643341825Sdim default: 644341825Sdim return false; 645341825Sdim case MVT::i8: { 646341825Sdim Opcode = (RegVT == MVT::i32) ? PPC::STBXTLS_32 : PPC::STBXTLS; 647341825Sdim break; 648341825Sdim } 649341825Sdim case MVT::i16: { 650341825Sdim Opcode = (RegVT == MVT::i32) ? PPC::STHXTLS_32 : PPC::STHXTLS; 651341825Sdim break; 652341825Sdim } 653341825Sdim case MVT::i32: { 654341825Sdim Opcode = (RegVT == MVT::i32) ? PPC::STWXTLS_32 : PPC::STWXTLS; 655341825Sdim break; 656341825Sdim } 657341825Sdim case MVT::i64: { 658341825Sdim Opcode = PPC::STDXTLS; 659341825Sdim break; 660341825Sdim } 661341825Sdim } 662341825Sdim SDValue Chain = ST->getChain(); 663341825Sdim SDVTList VTs = ST->getVTList(); 664341825Sdim SDValue Ops[] = {ST->getValue(), Base.getOperand(0), Base.getOperand(1), 665341825Sdim Chain}; 666341825Sdim SDNode *MN = CurDAG->getMachineNode(Opcode, dl, VTs, Ops); 667341825Sdim transferMemOperands(ST, MN); 668341825Sdim ReplaceNode(ST, MN); 669341825Sdim return true; 670341825Sdim} 671341825Sdim 672341825Sdimbool PPCDAGToDAGISel::tryTLSXFormLoad(LoadSDNode *LD) { 673341825Sdim SDValue Base = LD->getBasePtr(); 674341825Sdim if (Base.getOpcode() != PPCISD::ADD_TLS) 675341825Sdim return false; 676341825Sdim SDValue Offset = LD->getOffset(); 677341825Sdim if (!Offset.isUndef()) 678341825Sdim return false; 679341825Sdim 680341825Sdim SDLoc dl(LD); 681341825Sdim EVT MemVT = LD->getMemoryVT(); 682341825Sdim EVT RegVT = LD->getValueType(0); 683341825Sdim unsigned Opcode; 684341825Sdim switch (MemVT.getSimpleVT().SimpleTy) { 685341825Sdim default: 686341825Sdim return false; 687341825Sdim case MVT::i8: { 688341825Sdim Opcode = (RegVT == MVT::i32) ? PPC::LBZXTLS_32 : PPC::LBZXTLS; 689341825Sdim break; 690341825Sdim } 691341825Sdim case MVT::i16: { 692341825Sdim Opcode = (RegVT == MVT::i32) ? PPC::LHZXTLS_32 : PPC::LHZXTLS; 693341825Sdim break; 694341825Sdim } 695341825Sdim case MVT::i32: { 696341825Sdim Opcode = (RegVT == MVT::i32) ? PPC::LWZXTLS_32 : PPC::LWZXTLS; 697341825Sdim break; 698341825Sdim } 699341825Sdim case MVT::i64: { 700341825Sdim Opcode = PPC::LDXTLS; 701341825Sdim break; 702341825Sdim } 703341825Sdim } 704341825Sdim SDValue Chain = LD->getChain(); 705341825Sdim SDVTList VTs = LD->getVTList(); 706341825Sdim SDValue Ops[] = {Base.getOperand(0), Base.getOperand(1), Chain}; 707341825Sdim SDNode *MN = CurDAG->getMachineNode(Opcode, dl, VTs, Ops); 708341825Sdim transferMemOperands(LD, MN); 709341825Sdim ReplaceNode(LD, MN); 710341825Sdim return true; 711341825Sdim} 712341825Sdim 713309124Sdim/// Turn an or of two masked values into the rotate left word immediate then 714309124Sdim/// mask insert (rlwimi) instruction. 715309124Sdimbool PPCDAGToDAGISel::tryBitfieldInsert(SDNode *N) { 716193323Sed SDValue Op0 = N->getOperand(0); 717193323Sed SDValue Op1 = N->getOperand(1); 718261991Sdim SDLoc dl(N); 719218893Sdim 720344779Sdim KnownBits LKnown = CurDAG->computeKnownBits(Op0); 721344779Sdim KnownBits RKnown = CurDAG->computeKnownBits(Op1); 722218893Sdim 723321369Sdim unsigned TargetMask = LKnown.Zero.getZExtValue(); 724321369Sdim unsigned InsertMask = RKnown.Zero.getZExtValue(); 725218893Sdim 726193323Sed if ((TargetMask | InsertMask) == 0xFFFFFFFF) { 727193323Sed unsigned Op0Opc = Op0.getOpcode(); 728193323Sed unsigned Op1Opc = Op1.getOpcode(); 729193323Sed unsigned Value, SH = 0; 730193323Sed TargetMask = ~TargetMask; 731193323Sed InsertMask = ~InsertMask; 732193323Sed 733193323Sed // If the LHS has a foldable shift and the RHS does not, then swap it to the 734193323Sed // RHS so that we can fold the shift into the insert. 735193323Sed if (Op0Opc == ISD::AND && Op1Opc == ISD::AND) { 736193323Sed if (Op0.getOperand(0).getOpcode() == ISD::SHL || 737193323Sed Op0.getOperand(0).getOpcode() == ISD::SRL) { 738193323Sed if (Op1.getOperand(0).getOpcode() != ISD::SHL && 739193323Sed Op1.getOperand(0).getOpcode() != ISD::SRL) { 740193323Sed std::swap(Op0, Op1); 741193323Sed std::swap(Op0Opc, Op1Opc); 742193323Sed std::swap(TargetMask, InsertMask); 743193323Sed } 744193323Sed } 745193323Sed } else if (Op0Opc == ISD::SHL || Op0Opc == ISD::SRL) { 746193323Sed if (Op1Opc == ISD::AND && Op1.getOperand(0).getOpcode() != ISD::SHL && 747193323Sed Op1.getOperand(0).getOpcode() != ISD::SRL) { 748193323Sed std::swap(Op0, Op1); 749193323Sed std::swap(Op0Opc, Op1Opc); 750193323Sed std::swap(TargetMask, InsertMask); 751193323Sed } 752193323Sed } 753218893Sdim 754193323Sed unsigned MB, ME; 755261991Sdim if (isRunOfOnes(InsertMask, MB, ME)) { 756193323Sed if ((Op1Opc == ISD::SHL || Op1Opc == ISD::SRL) && 757193323Sed isInt32Immediate(Op1.getOperand(1), Value)) { 758193323Sed Op1 = Op1.getOperand(0); 759193323Sed SH = (Op1Opc == ISD::SHL) ? Value : 32 - Value; 760193323Sed } 761193323Sed if (Op1Opc == ISD::AND) { 762276479Sdim // The AND mask might not be a constant, and we need to make sure that 763276479Sdim // if we're going to fold the masking with the insert, all bits not 764276479Sdim // know to be zero in the mask are known to be one. 765344779Sdim KnownBits MKnown = CurDAG->computeKnownBits(Op1.getOperand(1)); 766321369Sdim bool CanFoldMask = InsertMask == MKnown.One.getZExtValue(); 767276479Sdim 768193323Sed unsigned SHOpc = Op1.getOperand(0).getOpcode(); 769276479Sdim if ((SHOpc == ISD::SHL || SHOpc == ISD::SRL) && CanFoldMask && 770193323Sed isInt32Immediate(Op1.getOperand(0).getOperand(1), Value)) { 771276479Sdim // Note that Value must be in range here (less than 32) because 772276479Sdim // otherwise there would not be any bits set in InsertMask. 773193323Sed Op1 = Op1.getOperand(0).getOperand(0); 774193323Sed SH = (SHOpc == ISD::SHL) ? Value : 32 - Value; 775193323Sed } 776193323Sed } 777199989Srdivacky 778193323Sed SH &= 31; 779288943Sdim SDValue Ops[] = { Op0, Op1, getI32Imm(SH, dl), getI32Imm(MB, dl), 780288943Sdim getI32Imm(ME, dl) }; 781309124Sdim ReplaceNode(N, CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops)); 782309124Sdim return true; 783193323Sed } 784193323Sed } 785309124Sdim return false; 786193323Sed} 787193323Sed 788280031Sdim// Predict the number of instructions that would be generated by calling 789327952Sdim// selectI64Imm(N). 790327952Sdimstatic unsigned selectI64ImmInstrCountDirect(int64_t Imm) { 791280031Sdim // Assume no remaining bits. 792280031Sdim unsigned Remainder = 0; 793280031Sdim // Assume no shift required. 794280031Sdim unsigned Shift = 0; 795280031Sdim 796280031Sdim // If it can't be represented as a 32 bit value. 797280031Sdim if (!isInt<32>(Imm)) { 798280031Sdim Shift = countTrailingZeros<uint64_t>(Imm); 799280031Sdim int64_t ImmSh = static_cast<uint64_t>(Imm) >> Shift; 800280031Sdim 801280031Sdim // If the shifted value fits 32 bits. 802280031Sdim if (isInt<32>(ImmSh)) { 803280031Sdim // Go with the shifted value. 804280031Sdim Imm = ImmSh; 805280031Sdim } else { 806280031Sdim // Still stuck with a 64 bit value. 807280031Sdim Remainder = Imm; 808280031Sdim Shift = 32; 809280031Sdim Imm >>= 32; 810280031Sdim } 811280031Sdim } 812280031Sdim 813280031Sdim // Intermediate operand. 814280031Sdim unsigned Result = 0; 815280031Sdim 816280031Sdim // Handle first 32 bits. 817280031Sdim unsigned Lo = Imm & 0xFFFF; 818280031Sdim 819280031Sdim // Simple value. 820280031Sdim if (isInt<16>(Imm)) { 821280031Sdim // Just the Lo bits. 822280031Sdim ++Result; 823280031Sdim } else if (Lo) { 824280031Sdim // Handle the Hi bits and Lo bits. 825280031Sdim Result += 2; 826280031Sdim } else { 827280031Sdim // Just the Hi bits. 828280031Sdim ++Result; 829280031Sdim } 830280031Sdim 831280031Sdim // If no shift, we're done. 832280031Sdim if (!Shift) return Result; 833280031Sdim 834314564Sdim // If Hi word == Lo word, 835314564Sdim // we can use rldimi to insert the Lo word into Hi word. 836314564Sdim if ((unsigned)(Imm & 0xFFFFFFFF) == Remainder) { 837314564Sdim ++Result; 838314564Sdim return Result; 839314564Sdim } 840314564Sdim 841280031Sdim // Shift for next step if the upper 32-bits were not zero. 842280031Sdim if (Imm) 843280031Sdim ++Result; 844280031Sdim 845280031Sdim // Add in the last bits as required. 846296417Sdim if ((Remainder >> 16) & 0xFFFF) 847280031Sdim ++Result; 848296417Sdim if (Remainder & 0xFFFF) 849280031Sdim ++Result; 850280031Sdim 851280031Sdim return Result; 852280031Sdim} 853280031Sdim 854280031Sdimstatic uint64_t Rot64(uint64_t Imm, unsigned R) { 855280031Sdim return (Imm << R) | (Imm >> (64 - R)); 856280031Sdim} 857280031Sdim 858327952Sdimstatic unsigned selectI64ImmInstrCount(int64_t Imm) { 859327952Sdim unsigned Count = selectI64ImmInstrCountDirect(Imm); 860321369Sdim 861321369Sdim // If the instruction count is 1 or 2, we do not need further analysis 862321369Sdim // since rotate + load constant requires at least 2 instructions. 863321369Sdim if (Count <= 2) 864280031Sdim return Count; 865280031Sdim 866280031Sdim for (unsigned r = 1; r < 63; ++r) { 867280031Sdim uint64_t RImm = Rot64(Imm, r); 868327952Sdim unsigned RCount = selectI64ImmInstrCountDirect(RImm) + 1; 869280031Sdim Count = std::min(Count, RCount); 870280031Sdim 871327952Sdim // See comments in selectI64Imm for an explanation of the logic below. 872280031Sdim unsigned LS = findLastSet(RImm); 873280031Sdim if (LS != r-1) 874280031Sdim continue; 875280031Sdim 876280031Sdim uint64_t OnesMask = -(int64_t) (UINT64_C(1) << (LS+1)); 877280031Sdim uint64_t RImmWithOnes = RImm | OnesMask; 878280031Sdim 879327952Sdim RCount = selectI64ImmInstrCountDirect(RImmWithOnes) + 1; 880280031Sdim Count = std::min(Count, RCount); 881280031Sdim } 882280031Sdim 883280031Sdim return Count; 884280031Sdim} 885280031Sdim 886327952Sdim// Select a 64-bit constant. For cost-modeling purposes, selectI64ImmInstrCount 887280031Sdim// (above) needs to be kept in sync with this function. 888327952Sdimstatic SDNode *selectI64ImmDirect(SelectionDAG *CurDAG, const SDLoc &dl, 889327952Sdim int64_t Imm) { 890280031Sdim // Assume no remaining bits. 891280031Sdim unsigned Remainder = 0; 892280031Sdim // Assume no shift required. 893280031Sdim unsigned Shift = 0; 894280031Sdim 895280031Sdim // If it can't be represented as a 32 bit value. 896280031Sdim if (!isInt<32>(Imm)) { 897280031Sdim Shift = countTrailingZeros<uint64_t>(Imm); 898280031Sdim int64_t ImmSh = static_cast<uint64_t>(Imm) >> Shift; 899280031Sdim 900280031Sdim // If the shifted value fits 32 bits. 901280031Sdim if (isInt<32>(ImmSh)) { 902280031Sdim // Go with the shifted value. 903280031Sdim Imm = ImmSh; 904280031Sdim } else { 905280031Sdim // Still stuck with a 64 bit value. 906280031Sdim Remainder = Imm; 907280031Sdim Shift = 32; 908280031Sdim Imm >>= 32; 909280031Sdim } 910280031Sdim } 911280031Sdim 912280031Sdim // Intermediate operand. 913280031Sdim SDNode *Result; 914280031Sdim 915280031Sdim // Handle first 32 bits. 916280031Sdim unsigned Lo = Imm & 0xFFFF; 917280031Sdim unsigned Hi = (Imm >> 16) & 0xFFFF; 918280031Sdim 919288943Sdim auto getI32Imm = [CurDAG, dl](unsigned Imm) { 920288943Sdim return CurDAG->getTargetConstant(Imm, dl, MVT::i32); 921280031Sdim }; 922280031Sdim 923280031Sdim // Simple value. 924280031Sdim if (isInt<16>(Imm)) { 925327952Sdim uint64_t SextImm = SignExtend64(Lo, 16); 926327952Sdim SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i64); 927280031Sdim // Just the Lo bits. 928327952Sdim Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, SDImm); 929280031Sdim } else if (Lo) { 930280031Sdim // Handle the Hi bits. 931280031Sdim unsigned OpC = Hi ? PPC::LIS8 : PPC::LI8; 932280031Sdim Result = CurDAG->getMachineNode(OpC, dl, MVT::i64, getI32Imm(Hi)); 933280031Sdim // And Lo bits. 934280031Sdim Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, 935280031Sdim SDValue(Result, 0), getI32Imm(Lo)); 936280031Sdim } else { 937280031Sdim // Just the Hi bits. 938280031Sdim Result = CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, getI32Imm(Hi)); 939280031Sdim } 940280031Sdim 941280031Sdim // If no shift, we're done. 942280031Sdim if (!Shift) return Result; 943280031Sdim 944314564Sdim // If Hi word == Lo word, 945314564Sdim // we can use rldimi to insert the Lo word into Hi word. 946314564Sdim if ((unsigned)(Imm & 0xFFFFFFFF) == Remainder) { 947314564Sdim SDValue Ops[] = 948314564Sdim { SDValue(Result, 0), SDValue(Result, 0), getI32Imm(Shift), getI32Imm(0)}; 949314564Sdim return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops); 950314564Sdim } 951314564Sdim 952280031Sdim // Shift for next step if the upper 32-bits were not zero. 953280031Sdim if (Imm) { 954280031Sdim Result = CurDAG->getMachineNode(PPC::RLDICR, dl, MVT::i64, 955280031Sdim SDValue(Result, 0), 956280031Sdim getI32Imm(Shift), 957280031Sdim getI32Imm(63 - Shift)); 958280031Sdim } 959280031Sdim 960280031Sdim // Add in the last bits as required. 961280031Sdim if ((Hi = (Remainder >> 16) & 0xFFFF)) { 962280031Sdim Result = CurDAG->getMachineNode(PPC::ORIS8, dl, MVT::i64, 963280031Sdim SDValue(Result, 0), getI32Imm(Hi)); 964280031Sdim } 965280031Sdim if ((Lo = Remainder & 0xFFFF)) { 966280031Sdim Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, 967280031Sdim SDValue(Result, 0), getI32Imm(Lo)); 968280031Sdim } 969280031Sdim 970280031Sdim return Result; 971280031Sdim} 972280031Sdim 973327952Sdimstatic SDNode *selectI64Imm(SelectionDAG *CurDAG, const SDLoc &dl, 974327952Sdim int64_t Imm) { 975327952Sdim unsigned Count = selectI64ImmInstrCountDirect(Imm); 976321369Sdim 977321369Sdim // If the instruction count is 1 or 2, we do not need further analysis 978321369Sdim // since rotate + load constant requires at least 2 instructions. 979321369Sdim if (Count <= 2) 980327952Sdim return selectI64ImmDirect(CurDAG, dl, Imm); 981280031Sdim 982280031Sdim unsigned RMin = 0; 983280031Sdim 984280031Sdim int64_t MatImm; 985280031Sdim unsigned MaskEnd; 986280031Sdim 987280031Sdim for (unsigned r = 1; r < 63; ++r) { 988280031Sdim uint64_t RImm = Rot64(Imm, r); 989327952Sdim unsigned RCount = selectI64ImmInstrCountDirect(RImm) + 1; 990280031Sdim if (RCount < Count) { 991280031Sdim Count = RCount; 992280031Sdim RMin = r; 993280031Sdim MatImm = RImm; 994280031Sdim MaskEnd = 63; 995280031Sdim } 996280031Sdim 997280031Sdim // If the immediate to generate has many trailing zeros, it might be 998280031Sdim // worthwhile to generate a rotated value with too many leading ones 999280031Sdim // (because that's free with li/lis's sign-extension semantics), and then 1000280031Sdim // mask them off after rotation. 1001280031Sdim 1002280031Sdim unsigned LS = findLastSet(RImm); 1003280031Sdim // We're adding (63-LS) higher-order ones, and we expect to mask them off 1004280031Sdim // after performing the inverse rotation by (64-r). So we need that: 1005280031Sdim // 63-LS == 64-r => LS == r-1 1006280031Sdim if (LS != r-1) 1007280031Sdim continue; 1008280031Sdim 1009280031Sdim uint64_t OnesMask = -(int64_t) (UINT64_C(1) << (LS+1)); 1010280031Sdim uint64_t RImmWithOnes = RImm | OnesMask; 1011280031Sdim 1012327952Sdim RCount = selectI64ImmInstrCountDirect(RImmWithOnes) + 1; 1013280031Sdim if (RCount < Count) { 1014280031Sdim Count = RCount; 1015280031Sdim RMin = r; 1016280031Sdim MatImm = RImmWithOnes; 1017280031Sdim MaskEnd = LS; 1018280031Sdim } 1019280031Sdim } 1020280031Sdim 1021280031Sdim if (!RMin) 1022327952Sdim return selectI64ImmDirect(CurDAG, dl, Imm); 1023280031Sdim 1024288943Sdim auto getI32Imm = [CurDAG, dl](unsigned Imm) { 1025288943Sdim return CurDAG->getTargetConstant(Imm, dl, MVT::i32); 1026280031Sdim }; 1027280031Sdim 1028327952Sdim SDValue Val = SDValue(selectI64ImmDirect(CurDAG, dl, MatImm), 0); 1029280031Sdim return CurDAG->getMachineNode(PPC::RLDICR, dl, MVT::i64, Val, 1030280031Sdim getI32Imm(64 - RMin), getI32Imm(MaskEnd)); 1031280031Sdim} 1032280031Sdim 1033327952Sdimstatic unsigned allUsesTruncate(SelectionDAG *CurDAG, SDNode *N) { 1034327952Sdim unsigned MaxTruncation = 0; 1035327952Sdim // Cannot use range-based for loop here as we need the actual use (i.e. we 1036327952Sdim // need the operand number corresponding to the use). A range-based for 1037327952Sdim // will unbox the use and provide an SDNode*. 1038327952Sdim for (SDNode::use_iterator Use = N->use_begin(), UseEnd = N->use_end(); 1039327952Sdim Use != UseEnd; ++Use) { 1040327952Sdim unsigned Opc = 1041327952Sdim Use->isMachineOpcode() ? Use->getMachineOpcode() : Use->getOpcode(); 1042327952Sdim switch (Opc) { 1043327952Sdim default: return 0; 1044327952Sdim case ISD::TRUNCATE: 1045327952Sdim if (Use->isMachineOpcode()) 1046327952Sdim return 0; 1047327952Sdim MaxTruncation = 1048360784Sdim std::max(MaxTruncation, (unsigned)Use->getValueType(0).getSizeInBits()); 1049327952Sdim continue; 1050327952Sdim case ISD::STORE: { 1051327952Sdim if (Use->isMachineOpcode()) 1052327952Sdim return 0; 1053327952Sdim StoreSDNode *STN = cast<StoreSDNode>(*Use); 1054327952Sdim unsigned MemVTSize = STN->getMemoryVT().getSizeInBits(); 1055327952Sdim if (MemVTSize == 64 || Use.getOperandNo() != 0) 1056327952Sdim return 0; 1057327952Sdim MaxTruncation = std::max(MaxTruncation, MemVTSize); 1058327952Sdim continue; 1059327952Sdim } 1060327952Sdim case PPC::STW8: 1061327952Sdim case PPC::STWX8: 1062327952Sdim case PPC::STWU8: 1063327952Sdim case PPC::STWUX8: 1064327952Sdim if (Use.getOperandNo() != 0) 1065327952Sdim return 0; 1066327952Sdim MaxTruncation = std::max(MaxTruncation, 32u); 1067327952Sdim continue; 1068327952Sdim case PPC::STH8: 1069327952Sdim case PPC::STHX8: 1070327952Sdim case PPC::STHU8: 1071327952Sdim case PPC::STHUX8: 1072327952Sdim if (Use.getOperandNo() != 0) 1073327952Sdim return 0; 1074327952Sdim MaxTruncation = std::max(MaxTruncation, 16u); 1075327952Sdim continue; 1076327952Sdim case PPC::STB8: 1077327952Sdim case PPC::STBX8: 1078327952Sdim case PPC::STBU8: 1079327952Sdim case PPC::STBUX8: 1080327952Sdim if (Use.getOperandNo() != 0) 1081327952Sdim return 0; 1082327952Sdim MaxTruncation = std::max(MaxTruncation, 8u); 1083327952Sdim continue; 1084327952Sdim } 1085327952Sdim } 1086327952Sdim return MaxTruncation; 1087327952Sdim} 1088327952Sdim 1089280031Sdim// Select a 64-bit constant. 1090327952Sdimstatic SDNode *selectI64Imm(SelectionDAG *CurDAG, SDNode *N) { 1091280031Sdim SDLoc dl(N); 1092280031Sdim 1093280031Sdim // Get 64 bit value. 1094280031Sdim int64_t Imm = cast<ConstantSDNode>(N)->getZExtValue(); 1095327952Sdim if (unsigned MinSize = allUsesTruncate(CurDAG, N)) { 1096327952Sdim uint64_t SextImm = SignExtend64(Imm, MinSize); 1097327952Sdim SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i64); 1098327952Sdim if (isInt<16>(SextImm)) 1099327952Sdim return CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, SDImm); 1100327952Sdim } 1101327952Sdim return selectI64Imm(CurDAG, dl, Imm); 1102280031Sdim} 1103280031Sdim 1104280031Sdimnamespace { 1105321369Sdim 1106280031Sdimclass BitPermutationSelector { 1107280031Sdim struct ValueBit { 1108280031Sdim SDValue V; 1109280031Sdim 1110280031Sdim // The bit number in the value, using a convention where bit 0 is the 1111280031Sdim // lowest-order bit. 1112280031Sdim unsigned Idx; 1113280031Sdim 1114344779Sdim // ConstZero means a bit we need to mask off. 1115344779Sdim // Variable is a bit comes from an input variable. 1116344779Sdim // VariableKnownToBeZero is also a bit comes from an input variable, 1117344779Sdim // but it is known to be already zero. So we do not need to mask them. 1118280031Sdim enum Kind { 1119280031Sdim ConstZero, 1120344779Sdim Variable, 1121344779Sdim VariableKnownToBeZero 1122280031Sdim } K; 1123280031Sdim 1124280031Sdim ValueBit(SDValue V, unsigned I, Kind K = Variable) 1125280031Sdim : V(V), Idx(I), K(K) {} 1126280031Sdim ValueBit(Kind K = Variable) 1127280031Sdim : V(SDValue(nullptr, 0)), Idx(UINT32_MAX), K(K) {} 1128280031Sdim 1129280031Sdim bool isZero() const { 1130344779Sdim return K == ConstZero || K == VariableKnownToBeZero; 1131280031Sdim } 1132280031Sdim 1133280031Sdim bool hasValue() const { 1134344779Sdim return K == Variable || K == VariableKnownToBeZero; 1135280031Sdim } 1136280031Sdim 1137280031Sdim SDValue getValue() const { 1138280031Sdim assert(hasValue() && "Cannot get the value of a constant bit"); 1139280031Sdim return V; 1140280031Sdim } 1141280031Sdim 1142280031Sdim unsigned getValueBitIndex() const { 1143280031Sdim assert(hasValue() && "Cannot get the value bit index of a constant bit"); 1144280031Sdim return Idx; 1145280031Sdim } 1146280031Sdim }; 1147280031Sdim 1148280031Sdim // A bit group has the same underlying value and the same rotate factor. 1149280031Sdim struct BitGroup { 1150280031Sdim SDValue V; 1151280031Sdim unsigned RLAmt; 1152280031Sdim unsigned StartIdx, EndIdx; 1153280031Sdim 1154280031Sdim // This rotation amount assumes that the lower 32 bits of the quantity are 1155280031Sdim // replicated in the high 32 bits by the rotation operator (which is done 1156280031Sdim // by rlwinm and friends in 64-bit mode). 1157280031Sdim bool Repl32; 1158280031Sdim // Did converting to Repl32 == true change the rotation factor? If it did, 1159280031Sdim // it decreased it by 32. 1160280031Sdim bool Repl32CR; 1161280031Sdim // Was this group coalesced after setting Repl32 to true? 1162280031Sdim bool Repl32Coalesced; 1163280031Sdim 1164280031Sdim BitGroup(SDValue V, unsigned R, unsigned S, unsigned E) 1165280031Sdim : V(V), RLAmt(R), StartIdx(S), EndIdx(E), Repl32(false), Repl32CR(false), 1166280031Sdim Repl32Coalesced(false) { 1167341825Sdim LLVM_DEBUG(dbgs() << "\tbit group for " << V.getNode() << " RLAmt = " << R 1168341825Sdim << " [" << S << ", " << E << "]\n"); 1169280031Sdim } 1170280031Sdim }; 1171280031Sdim 1172280031Sdim // Information on each (Value, RLAmt) pair (like the number of groups 1173280031Sdim // associated with each) used to choose the lowering method. 1174280031Sdim struct ValueRotInfo { 1175280031Sdim SDValue V; 1176321369Sdim unsigned RLAmt = std::numeric_limits<unsigned>::max(); 1177321369Sdim unsigned NumGroups = 0; 1178321369Sdim unsigned FirstGroupStartIdx = std::numeric_limits<unsigned>::max(); 1179321369Sdim bool Repl32 = false; 1180280031Sdim 1181321369Sdim ValueRotInfo() = default; 1182280031Sdim 1183280031Sdim // For sorting (in reverse order) by NumGroups, and then by 1184280031Sdim // FirstGroupStartIdx. 1185280031Sdim bool operator < (const ValueRotInfo &Other) const { 1186280031Sdim // We need to sort so that the non-Repl32 come first because, when we're 1187280031Sdim // doing masking, the Repl32 bit groups might be subsumed into the 64-bit 1188280031Sdim // masking operation. 1189280031Sdim if (Repl32 < Other.Repl32) 1190280031Sdim return true; 1191280031Sdim else if (Repl32 > Other.Repl32) 1192280031Sdim return false; 1193280031Sdim else if (NumGroups > Other.NumGroups) 1194280031Sdim return true; 1195280031Sdim else if (NumGroups < Other.NumGroups) 1196280031Sdim return false; 1197341825Sdim else if (RLAmt == 0 && Other.RLAmt != 0) 1198341825Sdim return true; 1199341825Sdim else if (RLAmt != 0 && Other.RLAmt == 0) 1200341825Sdim return false; 1201280031Sdim else if (FirstGroupStartIdx < Other.FirstGroupStartIdx) 1202280031Sdim return true; 1203280031Sdim return false; 1204280031Sdim } 1205280031Sdim }; 1206280031Sdim 1207314564Sdim using ValueBitsMemoizedValue = std::pair<bool, SmallVector<ValueBit, 64>>; 1208314564Sdim using ValueBitsMemoizer = 1209314564Sdim DenseMap<SDValue, std::unique_ptr<ValueBitsMemoizedValue>>; 1210314564Sdim ValueBitsMemoizer Memoizer; 1211314564Sdim 1212314564Sdim // Return a pair of bool and a SmallVector pointer to a memoization entry. 1213314564Sdim // The bool is true if something interesting was deduced, otherwise if we're 1214280031Sdim // providing only a generic representation of V (or something else likewise 1215314564Sdim // uninteresting for instruction selection) through the SmallVector. 1216314564Sdim std::pair<bool, SmallVector<ValueBit, 64> *> getValueBits(SDValue V, 1217314564Sdim unsigned NumBits) { 1218314564Sdim auto &ValueEntry = Memoizer[V]; 1219314564Sdim if (ValueEntry) 1220314564Sdim return std::make_pair(ValueEntry->first, &ValueEntry->second); 1221314564Sdim ValueEntry.reset(new ValueBitsMemoizedValue()); 1222314564Sdim bool &Interesting = ValueEntry->first; 1223314564Sdim SmallVector<ValueBit, 64> &Bits = ValueEntry->second; 1224314564Sdim Bits.resize(NumBits); 1225314564Sdim 1226280031Sdim switch (V.getOpcode()) { 1227280031Sdim default: break; 1228280031Sdim case ISD::ROTL: 1229280031Sdim if (isa<ConstantSDNode>(V.getOperand(1))) { 1230280031Sdim unsigned RotAmt = V.getConstantOperandVal(1); 1231280031Sdim 1232314564Sdim const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second; 1233280031Sdim 1234314564Sdim for (unsigned i = 0; i < NumBits; ++i) 1235314564Sdim Bits[i] = LHSBits[i < RotAmt ? i + (NumBits - RotAmt) : i - RotAmt]; 1236280031Sdim 1237314564Sdim return std::make_pair(Interesting = true, &Bits); 1238280031Sdim } 1239280031Sdim break; 1240280031Sdim case ISD::SHL: 1241280031Sdim if (isa<ConstantSDNode>(V.getOperand(1))) { 1242280031Sdim unsigned ShiftAmt = V.getConstantOperandVal(1); 1243280031Sdim 1244314564Sdim const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second; 1245280031Sdim 1246314564Sdim for (unsigned i = ShiftAmt; i < NumBits; ++i) 1247280031Sdim Bits[i] = LHSBits[i - ShiftAmt]; 1248280031Sdim 1249280031Sdim for (unsigned i = 0; i < ShiftAmt; ++i) 1250280031Sdim Bits[i] = ValueBit(ValueBit::ConstZero); 1251280031Sdim 1252314564Sdim return std::make_pair(Interesting = true, &Bits); 1253280031Sdim } 1254280031Sdim break; 1255280031Sdim case ISD::SRL: 1256280031Sdim if (isa<ConstantSDNode>(V.getOperand(1))) { 1257280031Sdim unsigned ShiftAmt = V.getConstantOperandVal(1); 1258280031Sdim 1259314564Sdim const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second; 1260280031Sdim 1261314564Sdim for (unsigned i = 0; i < NumBits - ShiftAmt; ++i) 1262280031Sdim Bits[i] = LHSBits[i + ShiftAmt]; 1263280031Sdim 1264314564Sdim for (unsigned i = NumBits - ShiftAmt; i < NumBits; ++i) 1265280031Sdim Bits[i] = ValueBit(ValueBit::ConstZero); 1266280031Sdim 1267314564Sdim return std::make_pair(Interesting = true, &Bits); 1268280031Sdim } 1269280031Sdim break; 1270280031Sdim case ISD::AND: 1271280031Sdim if (isa<ConstantSDNode>(V.getOperand(1))) { 1272280031Sdim uint64_t Mask = V.getConstantOperandVal(1); 1273280031Sdim 1274314564Sdim const SmallVector<ValueBit, 64> *LHSBits; 1275314564Sdim // Mark this as interesting, only if the LHS was also interesting. This 1276314564Sdim // prevents the overall procedure from matching a single immediate 'and' 1277314564Sdim // (which is non-optimal because such an and might be folded with other 1278314564Sdim // things if we don't select it here). 1279314564Sdim std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0), NumBits); 1280280031Sdim 1281314564Sdim for (unsigned i = 0; i < NumBits; ++i) 1282280031Sdim if (((Mask >> i) & 1) == 1) 1283314564Sdim Bits[i] = (*LHSBits)[i]; 1284344779Sdim else { 1285344779Sdim // AND instruction masks this bit. If the input is already zero, 1286344779Sdim // we have nothing to do here. Otherwise, make the bit ConstZero. 1287344779Sdim if ((*LHSBits)[i].isZero()) 1288344779Sdim Bits[i] = (*LHSBits)[i]; 1289344779Sdim else 1290344779Sdim Bits[i] = ValueBit(ValueBit::ConstZero); 1291344779Sdim } 1292280031Sdim 1293314564Sdim return std::make_pair(Interesting, &Bits); 1294280031Sdim } 1295280031Sdim break; 1296280031Sdim case ISD::OR: { 1297314564Sdim const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second; 1298314564Sdim const auto &RHSBits = *getValueBits(V.getOperand(1), NumBits).second; 1299280031Sdim 1300280031Sdim bool AllDisjoint = true; 1301344779Sdim SDValue LastVal = SDValue(); 1302344779Sdim unsigned LastIdx = 0; 1303344779Sdim for (unsigned i = 0; i < NumBits; ++i) { 1304344779Sdim if (LHSBits[i].isZero() && RHSBits[i].isZero()) { 1305344779Sdim // If both inputs are known to be zero and one is ConstZero and 1306344779Sdim // another is VariableKnownToBeZero, we can select whichever 1307344779Sdim // we like. To minimize the number of bit groups, we select 1308344779Sdim // VariableKnownToBeZero if this bit is the next bit of the same 1309344779Sdim // input variable from the previous bit. Otherwise, we select 1310344779Sdim // ConstZero. 1311344779Sdim if (LHSBits[i].hasValue() && LHSBits[i].getValue() == LastVal && 1312344779Sdim LHSBits[i].getValueBitIndex() == LastIdx + 1) 1313344779Sdim Bits[i] = LHSBits[i]; 1314344779Sdim else if (RHSBits[i].hasValue() && RHSBits[i].getValue() == LastVal && 1315344779Sdim RHSBits[i].getValueBitIndex() == LastIdx + 1) 1316344779Sdim Bits[i] = RHSBits[i]; 1317344779Sdim else 1318344779Sdim Bits[i] = ValueBit(ValueBit::ConstZero); 1319344779Sdim } 1320344779Sdim else if (LHSBits[i].isZero()) 1321280031Sdim Bits[i] = RHSBits[i]; 1322280031Sdim else if (RHSBits[i].isZero()) 1323280031Sdim Bits[i] = LHSBits[i]; 1324280031Sdim else { 1325280031Sdim AllDisjoint = false; 1326280031Sdim break; 1327280031Sdim } 1328344779Sdim // We remember the value and bit index of this bit. 1329344779Sdim if (Bits[i].hasValue()) { 1330344779Sdim LastVal = Bits[i].getValue(); 1331344779Sdim LastIdx = Bits[i].getValueBitIndex(); 1332344779Sdim } 1333344779Sdim else { 1334344779Sdim if (LastVal) LastVal = SDValue(); 1335344779Sdim LastIdx = 0; 1336344779Sdim } 1337344779Sdim } 1338280031Sdim 1339280031Sdim if (!AllDisjoint) 1340280031Sdim break; 1341280031Sdim 1342314564Sdim return std::make_pair(Interesting = true, &Bits); 1343280031Sdim } 1344327952Sdim case ISD::ZERO_EXTEND: { 1345327952Sdim // We support only the case with zero extension from i32 to i64 so far. 1346327952Sdim if (V.getValueType() != MVT::i64 || 1347327952Sdim V.getOperand(0).getValueType() != MVT::i32) 1348327952Sdim break; 1349327952Sdim 1350327952Sdim const SmallVector<ValueBit, 64> *LHSBits; 1351327952Sdim const unsigned NumOperandBits = 32; 1352327952Sdim std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0), 1353327952Sdim NumOperandBits); 1354327952Sdim 1355327952Sdim for (unsigned i = 0; i < NumOperandBits; ++i) 1356327952Sdim Bits[i] = (*LHSBits)[i]; 1357327952Sdim 1358327952Sdim for (unsigned i = NumOperandBits; i < NumBits; ++i) 1359327952Sdim Bits[i] = ValueBit(ValueBit::ConstZero); 1360327952Sdim 1361327952Sdim return std::make_pair(Interesting, &Bits); 1362280031Sdim } 1363344779Sdim case ISD::TRUNCATE: { 1364344779Sdim EVT FromType = V.getOperand(0).getValueType(); 1365344779Sdim EVT ToType = V.getValueType(); 1366344779Sdim // We support only the case with truncate from i64 to i32. 1367344779Sdim if (FromType != MVT::i64 || ToType != MVT::i32) 1368344779Sdim break; 1369344779Sdim const unsigned NumAllBits = FromType.getSizeInBits(); 1370344779Sdim SmallVector<ValueBit, 64> *InBits; 1371344779Sdim std::tie(Interesting, InBits) = getValueBits(V.getOperand(0), 1372344779Sdim NumAllBits); 1373344779Sdim const unsigned NumValidBits = ToType.getSizeInBits(); 1374344779Sdim 1375344779Sdim // A 32-bit instruction cannot touch upper 32-bit part of 64-bit value. 1376344779Sdim // So, we cannot include this truncate. 1377344779Sdim bool UseUpper32bit = false; 1378344779Sdim for (unsigned i = 0; i < NumValidBits; ++i) 1379344779Sdim if ((*InBits)[i].hasValue() && (*InBits)[i].getValueBitIndex() >= 32) { 1380344779Sdim UseUpper32bit = true; 1381344779Sdim break; 1382344779Sdim } 1383344779Sdim if (UseUpper32bit) 1384344779Sdim break; 1385344779Sdim 1386344779Sdim for (unsigned i = 0; i < NumValidBits; ++i) 1387344779Sdim Bits[i] = (*InBits)[i]; 1388344779Sdim 1389344779Sdim return std::make_pair(Interesting, &Bits); 1390341825Sdim } 1391344779Sdim case ISD::AssertZext: { 1392344779Sdim // For AssertZext, we look through the operand and 1393344779Sdim // mark the bits known to be zero. 1394344779Sdim const SmallVector<ValueBit, 64> *LHSBits; 1395344779Sdim std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0), 1396344779Sdim NumBits); 1397280031Sdim 1398344779Sdim EVT FromType = cast<VTSDNode>(V.getOperand(1))->getVT(); 1399344779Sdim const unsigned NumValidBits = FromType.getSizeInBits(); 1400344779Sdim for (unsigned i = 0; i < NumValidBits; ++i) 1401344779Sdim Bits[i] = (*LHSBits)[i]; 1402344779Sdim 1403360784Sdim // These bits are known to be zero but the AssertZext may be from a value 1404360784Sdim // that already has some constant zero bits (i.e. from a masking and). 1405344779Sdim for (unsigned i = NumValidBits; i < NumBits; ++i) 1406360784Sdim Bits[i] = (*LHSBits)[i].hasValue() 1407360784Sdim ? ValueBit((*LHSBits)[i].getValue(), 1408360784Sdim (*LHSBits)[i].getValueBitIndex(), 1409360784Sdim ValueBit::VariableKnownToBeZero) 1410360784Sdim : ValueBit(ValueBit::ConstZero); 1411344779Sdim 1412344779Sdim return std::make_pair(Interesting, &Bits); 1413344779Sdim } 1414344779Sdim case ISD::LOAD: 1415344779Sdim LoadSDNode *LD = cast<LoadSDNode>(V); 1416344779Sdim if (ISD::isZEXTLoad(V.getNode()) && V.getResNo() == 0) { 1417344779Sdim EVT VT = LD->getMemoryVT(); 1418344779Sdim const unsigned NumValidBits = VT.getSizeInBits(); 1419344779Sdim 1420344779Sdim for (unsigned i = 0; i < NumValidBits; ++i) 1421344779Sdim Bits[i] = ValueBit(V, i); 1422344779Sdim 1423344779Sdim // These bits are known to be zero. 1424344779Sdim for (unsigned i = NumValidBits; i < NumBits; ++i) 1425344779Sdim Bits[i] = ValueBit(V, i, ValueBit::VariableKnownToBeZero); 1426344779Sdim 1427344779Sdim // Zero-extending load itself cannot be optimized. So, it is not 1428344779Sdim // interesting by itself though it gives useful information. 1429344779Sdim return std::make_pair(Interesting = false, &Bits); 1430344779Sdim } 1431344779Sdim break; 1432344779Sdim } 1433344779Sdim 1434314564Sdim for (unsigned i = 0; i < NumBits; ++i) 1435280031Sdim Bits[i] = ValueBit(V, i); 1436280031Sdim 1437314564Sdim return std::make_pair(Interesting = false, &Bits); 1438280031Sdim } 1439280031Sdim 1440280031Sdim // For each value (except the constant ones), compute the left-rotate amount 1441280031Sdim // to get it from its original to final position. 1442280031Sdim void computeRotationAmounts() { 1443344779Sdim NeedMask = false; 1444280031Sdim RLAmt.resize(Bits.size()); 1445280031Sdim for (unsigned i = 0; i < Bits.size(); ++i) 1446280031Sdim if (Bits[i].hasValue()) { 1447280031Sdim unsigned VBI = Bits[i].getValueBitIndex(); 1448280031Sdim if (i >= VBI) 1449280031Sdim RLAmt[i] = i - VBI; 1450280031Sdim else 1451280031Sdim RLAmt[i] = Bits.size() - (VBI - i); 1452280031Sdim } else if (Bits[i].isZero()) { 1453344779Sdim NeedMask = true; 1454280031Sdim RLAmt[i] = UINT32_MAX; 1455280031Sdim } else { 1456280031Sdim llvm_unreachable("Unknown value bit type"); 1457280031Sdim } 1458280031Sdim } 1459280031Sdim 1460280031Sdim // Collect groups of consecutive bits with the same underlying value and 1461280031Sdim // rotation factor. If we're doing late masking, we ignore zeros, otherwise 1462280031Sdim // they break up groups. 1463280031Sdim void collectBitGroups(bool LateMask) { 1464280031Sdim BitGroups.clear(); 1465280031Sdim 1466280031Sdim unsigned LastRLAmt = RLAmt[0]; 1467280031Sdim SDValue LastValue = Bits[0].hasValue() ? Bits[0].getValue() : SDValue(); 1468280031Sdim unsigned LastGroupStartIdx = 0; 1469344779Sdim bool IsGroupOfZeros = !Bits[LastGroupStartIdx].hasValue(); 1470280031Sdim for (unsigned i = 1; i < Bits.size(); ++i) { 1471280031Sdim unsigned ThisRLAmt = RLAmt[i]; 1472280031Sdim SDValue ThisValue = Bits[i].hasValue() ? Bits[i].getValue() : SDValue(); 1473280031Sdim if (LateMask && !ThisValue) { 1474280031Sdim ThisValue = LastValue; 1475280031Sdim ThisRLAmt = LastRLAmt; 1476280031Sdim // If we're doing late masking, then the first bit group always starts 1477280031Sdim // at zero (even if the first bits were zero). 1478280031Sdim if (BitGroups.empty()) 1479280031Sdim LastGroupStartIdx = 0; 1480280031Sdim } 1481280031Sdim 1482344779Sdim // If this bit is known to be zero and the current group is a bit group 1483344779Sdim // of zeros, we do not need to terminate the current bit group even the 1484344779Sdim // Value or RLAmt does not match here. Instead, we terminate this group 1485344779Sdim // when the first non-zero bit appears later. 1486344779Sdim if (IsGroupOfZeros && Bits[i].isZero()) 1487344779Sdim continue; 1488344779Sdim 1489280031Sdim // If this bit has the same underlying value and the same rotate factor as 1490280031Sdim // the last one, then they're part of the same group. 1491280031Sdim if (ThisRLAmt == LastRLAmt && ThisValue == LastValue) 1492344779Sdim // We cannot continue the current group if this bits is not known to 1493344779Sdim // be zero in a bit group of zeros. 1494344779Sdim if (!(IsGroupOfZeros && ThisValue && !Bits[i].isZero())) 1495344779Sdim continue; 1496280031Sdim 1497280031Sdim if (LastValue.getNode()) 1498280031Sdim BitGroups.push_back(BitGroup(LastValue, LastRLAmt, LastGroupStartIdx, 1499280031Sdim i-1)); 1500280031Sdim LastRLAmt = ThisRLAmt; 1501280031Sdim LastValue = ThisValue; 1502280031Sdim LastGroupStartIdx = i; 1503344779Sdim IsGroupOfZeros = !Bits[LastGroupStartIdx].hasValue(); 1504280031Sdim } 1505280031Sdim if (LastValue.getNode()) 1506280031Sdim BitGroups.push_back(BitGroup(LastValue, LastRLAmt, LastGroupStartIdx, 1507280031Sdim Bits.size()-1)); 1508280031Sdim 1509280031Sdim if (BitGroups.empty()) 1510280031Sdim return; 1511280031Sdim 1512280031Sdim // We might be able to combine the first and last groups. 1513280031Sdim if (BitGroups.size() > 1) { 1514280031Sdim // If the first and last groups are the same, then remove the first group 1515280031Sdim // in favor of the last group, making the ending index of the last group 1516280031Sdim // equal to the ending index of the to-be-removed first group. 1517280031Sdim if (BitGroups[0].StartIdx == 0 && 1518280031Sdim BitGroups[BitGroups.size()-1].EndIdx == Bits.size()-1 && 1519280031Sdim BitGroups[0].V == BitGroups[BitGroups.size()-1].V && 1520280031Sdim BitGroups[0].RLAmt == BitGroups[BitGroups.size()-1].RLAmt) { 1521341825Sdim LLVM_DEBUG(dbgs() << "\tcombining final bit group with initial one\n"); 1522280031Sdim BitGroups[BitGroups.size()-1].EndIdx = BitGroups[0].EndIdx; 1523280031Sdim BitGroups.erase(BitGroups.begin()); 1524280031Sdim } 1525280031Sdim } 1526280031Sdim } 1527280031Sdim 1528280031Sdim // Take all (SDValue, RLAmt) pairs and sort them by the number of groups 1529341825Sdim // associated with each. If the number of groups are same, we prefer a group 1530341825Sdim // which does not require rotate, i.e. RLAmt is 0, to avoid the first rotate 1531341825Sdim // instruction. If there is a degeneracy, pick the one that occurs 1532280031Sdim // first (in the final value). 1533280031Sdim void collectValueRotInfo() { 1534280031Sdim ValueRots.clear(); 1535280031Sdim 1536280031Sdim for (auto &BG : BitGroups) { 1537280031Sdim unsigned RLAmtKey = BG.RLAmt + (BG.Repl32 ? 64 : 0); 1538280031Sdim ValueRotInfo &VRI = ValueRots[std::make_pair(BG.V, RLAmtKey)]; 1539280031Sdim VRI.V = BG.V; 1540280031Sdim VRI.RLAmt = BG.RLAmt; 1541280031Sdim VRI.Repl32 = BG.Repl32; 1542280031Sdim VRI.NumGroups += 1; 1543280031Sdim VRI.FirstGroupStartIdx = std::min(VRI.FirstGroupStartIdx, BG.StartIdx); 1544280031Sdim } 1545280031Sdim 1546280031Sdim // Now that we've collected the various ValueRotInfo instances, we need to 1547280031Sdim // sort them. 1548280031Sdim ValueRotsVec.clear(); 1549280031Sdim for (auto &I : ValueRots) { 1550280031Sdim ValueRotsVec.push_back(I.second); 1551280031Sdim } 1552344779Sdim llvm::sort(ValueRotsVec); 1553280031Sdim } 1554280031Sdim 1555280031Sdim // In 64-bit mode, rlwinm and friends have a rotation operator that 1556280031Sdim // replicates the low-order 32 bits into the high-order 32-bits. The mask 1557280031Sdim // indices of these instructions can only be in the lower 32 bits, so they 1558280031Sdim // can only represent some 64-bit bit groups. However, when they can be used, 1559280031Sdim // the 32-bit replication can be used to represent, as a single bit group, 1560280031Sdim // otherwise separate bit groups. We'll convert to replicated-32-bit bit 1561280031Sdim // groups when possible. Returns true if any of the bit groups were 1562280031Sdim // converted. 1563280031Sdim void assignRepl32BitGroups() { 1564280031Sdim // If we have bits like this: 1565280031Sdim // 1566280031Sdim // Indices: 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0 1567280031Sdim // V bits: ... 7 6 5 4 3 2 1 0 31 30 29 28 27 26 25 24 1568280031Sdim // Groups: | RLAmt = 8 | RLAmt = 40 | 1569280031Sdim // 1570280031Sdim // But, making use of a 32-bit operation that replicates the low-order 32 1571280031Sdim // bits into the high-order 32 bits, this can be one bit group with a RLAmt 1572280031Sdim // of 8. 1573280031Sdim 1574280031Sdim auto IsAllLow32 = [this](BitGroup & BG) { 1575280031Sdim if (BG.StartIdx <= BG.EndIdx) { 1576280031Sdim for (unsigned i = BG.StartIdx; i <= BG.EndIdx; ++i) { 1577280031Sdim if (!Bits[i].hasValue()) 1578280031Sdim continue; 1579280031Sdim if (Bits[i].getValueBitIndex() >= 32) 1580280031Sdim return false; 1581280031Sdim } 1582280031Sdim } else { 1583280031Sdim for (unsigned i = BG.StartIdx; i < Bits.size(); ++i) { 1584280031Sdim if (!Bits[i].hasValue()) 1585280031Sdim continue; 1586280031Sdim if (Bits[i].getValueBitIndex() >= 32) 1587280031Sdim return false; 1588280031Sdim } 1589280031Sdim for (unsigned i = 0; i <= BG.EndIdx; ++i) { 1590280031Sdim if (!Bits[i].hasValue()) 1591280031Sdim continue; 1592280031Sdim if (Bits[i].getValueBitIndex() >= 32) 1593280031Sdim return false; 1594280031Sdim } 1595280031Sdim } 1596280031Sdim 1597280031Sdim return true; 1598280031Sdim }; 1599280031Sdim 1600280031Sdim for (auto &BG : BitGroups) { 1601341825Sdim // If this bit group has RLAmt of 0 and will not be merged with 1602341825Sdim // another bit group, we don't benefit from Repl32. We don't mark 1603341825Sdim // such group to give more freedom for later instruction selection. 1604341825Sdim if (BG.RLAmt == 0) { 1605341825Sdim auto PotentiallyMerged = [this](BitGroup & BG) { 1606341825Sdim for (auto &BG2 : BitGroups) 1607341825Sdim if (&BG != &BG2 && BG.V == BG2.V && 1608341825Sdim (BG2.RLAmt == 0 || BG2.RLAmt == 32)) 1609341825Sdim return true; 1610341825Sdim return false; 1611341825Sdim }; 1612341825Sdim if (!PotentiallyMerged(BG)) 1613341825Sdim continue; 1614341825Sdim } 1615280031Sdim if (BG.StartIdx < 32 && BG.EndIdx < 32) { 1616280031Sdim if (IsAllLow32(BG)) { 1617280031Sdim if (BG.RLAmt >= 32) { 1618280031Sdim BG.RLAmt -= 32; 1619280031Sdim BG.Repl32CR = true; 1620280031Sdim } 1621280031Sdim 1622280031Sdim BG.Repl32 = true; 1623280031Sdim 1624341825Sdim LLVM_DEBUG(dbgs() << "\t32-bit replicated bit group for " 1625341825Sdim << BG.V.getNode() << " RLAmt = " << BG.RLAmt << " [" 1626341825Sdim << BG.StartIdx << ", " << BG.EndIdx << "]\n"); 1627280031Sdim } 1628280031Sdim } 1629280031Sdim } 1630280031Sdim 1631280031Sdim // Now walk through the bit groups, consolidating where possible. 1632280031Sdim for (auto I = BitGroups.begin(); I != BitGroups.end();) { 1633280031Sdim // We might want to remove this bit group by merging it with the previous 1634280031Sdim // group (which might be the ending group). 1635280031Sdim auto IP = (I == BitGroups.begin()) ? 1636280031Sdim std::prev(BitGroups.end()) : std::prev(I); 1637280031Sdim if (I->Repl32 && IP->Repl32 && I->V == IP->V && I->RLAmt == IP->RLAmt && 1638280031Sdim I->StartIdx == (IP->EndIdx + 1) % 64 && I != IP) { 1639280031Sdim 1640341825Sdim LLVM_DEBUG(dbgs() << "\tcombining 32-bit replicated bit group for " 1641341825Sdim << I->V.getNode() << " RLAmt = " << I->RLAmt << " [" 1642341825Sdim << I->StartIdx << ", " << I->EndIdx 1643341825Sdim << "] with group with range [" << IP->StartIdx << ", " 1644341825Sdim << IP->EndIdx << "]\n"); 1645280031Sdim 1646280031Sdim IP->EndIdx = I->EndIdx; 1647280031Sdim IP->Repl32CR = IP->Repl32CR || I->Repl32CR; 1648280031Sdim IP->Repl32Coalesced = true; 1649280031Sdim I = BitGroups.erase(I); 1650280031Sdim continue; 1651280031Sdim } else { 1652280031Sdim // There is a special case worth handling: If there is a single group 1653280031Sdim // covering the entire upper 32 bits, and it can be merged with both 1654280031Sdim // the next and previous groups (which might be the same group), then 1655280031Sdim // do so. If it is the same group (so there will be only one group in 1656280031Sdim // total), then we need to reverse the order of the range so that it 1657280031Sdim // covers the entire 64 bits. 1658280031Sdim if (I->StartIdx == 32 && I->EndIdx == 63) { 1659280031Sdim assert(std::next(I) == BitGroups.end() && 1660280031Sdim "bit group ends at index 63 but there is another?"); 1661280031Sdim auto IN = BitGroups.begin(); 1662280031Sdim 1663309124Sdim if (IP->Repl32 && IN->Repl32 && I->V == IP->V && I->V == IN->V && 1664280031Sdim (I->RLAmt % 32) == IP->RLAmt && (I->RLAmt % 32) == IN->RLAmt && 1665280031Sdim IP->EndIdx == 31 && IN->StartIdx == 0 && I != IP && 1666280031Sdim IsAllLow32(*I)) { 1667280031Sdim 1668341825Sdim LLVM_DEBUG(dbgs() << "\tcombining bit group for " << I->V.getNode() 1669341825Sdim << " RLAmt = " << I->RLAmt << " [" << I->StartIdx 1670341825Sdim << ", " << I->EndIdx 1671341825Sdim << "] with 32-bit replicated groups with ranges [" 1672341825Sdim << IP->StartIdx << ", " << IP->EndIdx << "] and [" 1673341825Sdim << IN->StartIdx << ", " << IN->EndIdx << "]\n"); 1674280031Sdim 1675280031Sdim if (IP == IN) { 1676280031Sdim // There is only one other group; change it to cover the whole 1677280031Sdim // range (backward, so that it can still be Repl32 but cover the 1678280031Sdim // whole 64-bit range). 1679280031Sdim IP->StartIdx = 31; 1680280031Sdim IP->EndIdx = 30; 1681280031Sdim IP->Repl32CR = IP->Repl32CR || I->RLAmt >= 32; 1682280031Sdim IP->Repl32Coalesced = true; 1683280031Sdim I = BitGroups.erase(I); 1684280031Sdim } else { 1685280031Sdim // There are two separate groups, one before this group and one 1686280031Sdim // after us (at the beginning). We're going to remove this group, 1687280031Sdim // but also the group at the very beginning. 1688280031Sdim IP->EndIdx = IN->EndIdx; 1689280031Sdim IP->Repl32CR = IP->Repl32CR || IN->Repl32CR || I->RLAmt >= 32; 1690280031Sdim IP->Repl32Coalesced = true; 1691280031Sdim I = BitGroups.erase(I); 1692280031Sdim BitGroups.erase(BitGroups.begin()); 1693280031Sdim } 1694280031Sdim 1695280031Sdim // This must be the last group in the vector (and we might have 1696280031Sdim // just invalidated the iterator above), so break here. 1697280031Sdim break; 1698280031Sdim } 1699280031Sdim } 1700280031Sdim } 1701280031Sdim 1702280031Sdim ++I; 1703280031Sdim } 1704280031Sdim } 1705280031Sdim 1706309124Sdim SDValue getI32Imm(unsigned Imm, const SDLoc &dl) { 1707288943Sdim return CurDAG->getTargetConstant(Imm, dl, MVT::i32); 1708280031Sdim } 1709280031Sdim 1710280031Sdim uint64_t getZerosMask() { 1711280031Sdim uint64_t Mask = 0; 1712280031Sdim for (unsigned i = 0; i < Bits.size(); ++i) { 1713280031Sdim if (Bits[i].hasValue()) 1714280031Sdim continue; 1715280031Sdim Mask |= (UINT64_C(1) << i); 1716280031Sdim } 1717280031Sdim 1718280031Sdim return ~Mask; 1719280031Sdim } 1720280031Sdim 1721327952Sdim // This method extends an input value to 64 bit if input is 32-bit integer. 1722327952Sdim // While selecting instructions in BitPermutationSelector in 64-bit mode, 1723327952Sdim // an input value can be a 32-bit integer if a ZERO_EXTEND node is included. 1724327952Sdim // In such case, we extend it to 64 bit to be consistent with other values. 1725327952Sdim SDValue ExtendToInt64(SDValue V, const SDLoc &dl) { 1726327952Sdim if (V.getValueSizeInBits() == 64) 1727327952Sdim return V; 1728327952Sdim 1729327952Sdim assert(V.getValueSizeInBits() == 32); 1730327952Sdim SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32); 1731327952Sdim SDValue ImDef = SDValue(CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, 1732327952Sdim MVT::i64), 0); 1733327952Sdim SDValue ExtVal = SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl, 1734327952Sdim MVT::i64, ImDef, V, 1735327952Sdim SubRegIdx), 0); 1736327952Sdim return ExtVal; 1737327952Sdim } 1738327952Sdim 1739344779Sdim SDValue TruncateToInt32(SDValue V, const SDLoc &dl) { 1740344779Sdim if (V.getValueSizeInBits() == 32) 1741344779Sdim return V; 1742344779Sdim 1743344779Sdim assert(V.getValueSizeInBits() == 64); 1744344779Sdim SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32); 1745344779Sdim SDValue SubVal = SDValue(CurDAG->getMachineNode(PPC::EXTRACT_SUBREG, dl, 1746344779Sdim MVT::i32, V, SubRegIdx), 0); 1747344779Sdim return SubVal; 1748344779Sdim } 1749344779Sdim 1750280031Sdim // Depending on the number of groups for a particular value, it might be 1751280031Sdim // better to rotate, mask explicitly (using andi/andis), and then or the 1752280031Sdim // result. Select this part of the result first. 1753309124Sdim void SelectAndParts32(const SDLoc &dl, SDValue &Res, unsigned *InstCnt) { 1754280031Sdim if (BPermRewriterNoMasking) 1755280031Sdim return; 1756280031Sdim 1757280031Sdim for (ValueRotInfo &VRI : ValueRotsVec) { 1758280031Sdim unsigned Mask = 0; 1759280031Sdim for (unsigned i = 0; i < Bits.size(); ++i) { 1760280031Sdim if (!Bits[i].hasValue() || Bits[i].getValue() != VRI.V) 1761280031Sdim continue; 1762280031Sdim if (RLAmt[i] != VRI.RLAmt) 1763280031Sdim continue; 1764280031Sdim Mask |= (1u << i); 1765280031Sdim } 1766280031Sdim 1767280031Sdim // Compute the masks for andi/andis that would be necessary. 1768280031Sdim unsigned ANDIMask = (Mask & UINT16_MAX), ANDISMask = Mask >> 16; 1769280031Sdim assert((ANDIMask != 0 || ANDISMask != 0) && 1770280031Sdim "No set bits in mask for value bit groups"); 1771280031Sdim bool NeedsRotate = VRI.RLAmt != 0; 1772280031Sdim 1773280031Sdim // We're trying to minimize the number of instructions. If we have one 1774280031Sdim // group, using one of andi/andis can break even. If we have three 1775280031Sdim // groups, we can use both andi and andis and break even (to use both 1776280031Sdim // andi and andis we also need to or the results together). We need four 1777280031Sdim // groups if we also need to rotate. To use andi/andis we need to do more 1778280031Sdim // than break even because rotate-and-mask instructions tend to be easier 1779280031Sdim // to schedule. 1780280031Sdim 1781280031Sdim // FIXME: We've biased here against using andi/andis, which is right for 1782280031Sdim // POWER cores, but not optimal everywhere. For example, on the A2, 1783280031Sdim // andi/andis have single-cycle latency whereas the rotate-and-mask 1784280031Sdim // instructions take two cycles, and it would be better to bias toward 1785280031Sdim // andi/andis in break-even cases. 1786280031Sdim 1787280031Sdim unsigned NumAndInsts = (unsigned) NeedsRotate + 1788280031Sdim (unsigned) (ANDIMask != 0) + 1789280031Sdim (unsigned) (ANDISMask != 0) + 1790280031Sdim (unsigned) (ANDIMask != 0 && ANDISMask != 0) + 1791280031Sdim (unsigned) (bool) Res; 1792280031Sdim 1793341825Sdim LLVM_DEBUG(dbgs() << "\t\trotation groups for " << VRI.V.getNode() 1794341825Sdim << " RL: " << VRI.RLAmt << ":" 1795341825Sdim << "\n\t\t\tisel using masking: " << NumAndInsts 1796341825Sdim << " using rotates: " << VRI.NumGroups << "\n"); 1797280031Sdim 1798280031Sdim if (NumAndInsts >= VRI.NumGroups) 1799280031Sdim continue; 1800280031Sdim 1801341825Sdim LLVM_DEBUG(dbgs() << "\t\t\t\tusing masking\n"); 1802280031Sdim 1803280031Sdim if (InstCnt) *InstCnt += NumAndInsts; 1804280031Sdim 1805280031Sdim SDValue VRot; 1806280031Sdim if (VRI.RLAmt) { 1807280031Sdim SDValue Ops[] = 1808344779Sdim { TruncateToInt32(VRI.V, dl), getI32Imm(VRI.RLAmt, dl), 1809344779Sdim getI32Imm(0, dl), getI32Imm(31, dl) }; 1810280031Sdim VRot = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, 1811280031Sdim Ops), 0); 1812280031Sdim } else { 1813344779Sdim VRot = TruncateToInt32(VRI.V, dl); 1814280031Sdim } 1815280031Sdim 1816280031Sdim SDValue ANDIVal, ANDISVal; 1817280031Sdim if (ANDIMask != 0) 1818360784Sdim ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI_rec, dl, MVT::i32, 1819360784Sdim VRot, getI32Imm(ANDIMask, dl)), 1820360784Sdim 0); 1821280031Sdim if (ANDISMask != 0) 1822360784Sdim ANDISVal = 1823360784Sdim SDValue(CurDAG->getMachineNode(PPC::ANDIS_rec, dl, MVT::i32, VRot, 1824360784Sdim getI32Imm(ANDISMask, dl)), 1825360784Sdim 0); 1826280031Sdim 1827280031Sdim SDValue TotalVal; 1828280031Sdim if (!ANDIVal) 1829280031Sdim TotalVal = ANDISVal; 1830280031Sdim else if (!ANDISVal) 1831280031Sdim TotalVal = ANDIVal; 1832280031Sdim else 1833280031Sdim TotalVal = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32, 1834280031Sdim ANDIVal, ANDISVal), 0); 1835280031Sdim 1836280031Sdim if (!Res) 1837280031Sdim Res = TotalVal; 1838280031Sdim else 1839280031Sdim Res = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32, 1840280031Sdim Res, TotalVal), 0); 1841280031Sdim 1842280031Sdim // Now, remove all groups with this underlying value and rotation 1843280031Sdim // factor. 1844288943Sdim eraseMatchingBitGroups([VRI](const BitGroup &BG) { 1845288943Sdim return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt; 1846288943Sdim }); 1847280031Sdim } 1848280031Sdim } 1849280031Sdim 1850280031Sdim // Instruction selection for the 32-bit case. 1851280031Sdim SDNode *Select32(SDNode *N, bool LateMask, unsigned *InstCnt) { 1852280031Sdim SDLoc dl(N); 1853280031Sdim SDValue Res; 1854280031Sdim 1855280031Sdim if (InstCnt) *InstCnt = 0; 1856280031Sdim 1857280031Sdim // Take care of cases that should use andi/andis first. 1858280031Sdim SelectAndParts32(dl, Res, InstCnt); 1859280031Sdim 1860280031Sdim // If we've not yet selected a 'starting' instruction, and we have no zeros 1861280031Sdim // to fill in, select the (Value, RLAmt) with the highest priority (largest 1862280031Sdim // number of groups), and start with this rotated value. 1863344779Sdim if ((!NeedMask || LateMask) && !Res) { 1864280031Sdim ValueRotInfo &VRI = ValueRotsVec[0]; 1865280031Sdim if (VRI.RLAmt) { 1866280031Sdim if (InstCnt) *InstCnt += 1; 1867280031Sdim SDValue Ops[] = 1868344779Sdim { TruncateToInt32(VRI.V, dl), getI32Imm(VRI.RLAmt, dl), 1869344779Sdim getI32Imm(0, dl), getI32Imm(31, dl) }; 1870288943Sdim Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 1871288943Sdim 0); 1872280031Sdim } else { 1873344779Sdim Res = TruncateToInt32(VRI.V, dl); 1874280031Sdim } 1875280031Sdim 1876280031Sdim // Now, remove all groups with this underlying value and rotation factor. 1877288943Sdim eraseMatchingBitGroups([VRI](const BitGroup &BG) { 1878288943Sdim return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt; 1879288943Sdim }); 1880280031Sdim } 1881280031Sdim 1882280031Sdim if (InstCnt) *InstCnt += BitGroups.size(); 1883280031Sdim 1884280031Sdim // Insert the other groups (one at a time). 1885280031Sdim for (auto &BG : BitGroups) { 1886280031Sdim if (!Res) { 1887280031Sdim SDValue Ops[] = 1888344779Sdim { TruncateToInt32(BG.V, dl), getI32Imm(BG.RLAmt, dl), 1889288943Sdim getI32Imm(Bits.size() - BG.EndIdx - 1, dl), 1890288943Sdim getI32Imm(Bits.size() - BG.StartIdx - 1, dl) }; 1891280031Sdim Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); 1892280031Sdim } else { 1893280031Sdim SDValue Ops[] = 1894344779Sdim { Res, TruncateToInt32(BG.V, dl), getI32Imm(BG.RLAmt, dl), 1895288943Sdim getI32Imm(Bits.size() - BG.EndIdx - 1, dl), 1896288943Sdim getI32Imm(Bits.size() - BG.StartIdx - 1, dl) }; 1897280031Sdim Res = SDValue(CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops), 0); 1898280031Sdim } 1899280031Sdim } 1900280031Sdim 1901280031Sdim if (LateMask) { 1902280031Sdim unsigned Mask = (unsigned) getZerosMask(); 1903280031Sdim 1904280031Sdim unsigned ANDIMask = (Mask & UINT16_MAX), ANDISMask = Mask >> 16; 1905280031Sdim assert((ANDIMask != 0 || ANDISMask != 0) && 1906280031Sdim "No set bits in zeros mask?"); 1907280031Sdim 1908280031Sdim if (InstCnt) *InstCnt += (unsigned) (ANDIMask != 0) + 1909280031Sdim (unsigned) (ANDISMask != 0) + 1910280031Sdim (unsigned) (ANDIMask != 0 && ANDISMask != 0); 1911280031Sdim 1912280031Sdim SDValue ANDIVal, ANDISVal; 1913280031Sdim if (ANDIMask != 0) 1914360784Sdim ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI_rec, dl, MVT::i32, 1915360784Sdim Res, getI32Imm(ANDIMask, dl)), 1916360784Sdim 0); 1917280031Sdim if (ANDISMask != 0) 1918360784Sdim ANDISVal = 1919360784Sdim SDValue(CurDAG->getMachineNode(PPC::ANDIS_rec, dl, MVT::i32, Res, 1920360784Sdim getI32Imm(ANDISMask, dl)), 1921360784Sdim 0); 1922280031Sdim 1923280031Sdim if (!ANDIVal) 1924280031Sdim Res = ANDISVal; 1925280031Sdim else if (!ANDISVal) 1926280031Sdim Res = ANDIVal; 1927280031Sdim else 1928280031Sdim Res = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32, 1929280031Sdim ANDIVal, ANDISVal), 0); 1930280031Sdim } 1931280031Sdim 1932280031Sdim return Res.getNode(); 1933280031Sdim } 1934280031Sdim 1935280031Sdim unsigned SelectRotMask64Count(unsigned RLAmt, bool Repl32, 1936280031Sdim unsigned MaskStart, unsigned MaskEnd, 1937280031Sdim bool IsIns) { 1938280031Sdim // In the notation used by the instructions, 'start' and 'end' are reversed 1939280031Sdim // because bits are counted from high to low order. 1940280031Sdim unsigned InstMaskStart = 64 - MaskEnd - 1, 1941280031Sdim InstMaskEnd = 64 - MaskStart - 1; 1942280031Sdim 1943280031Sdim if (Repl32) 1944280031Sdim return 1; 1945280031Sdim 1946280031Sdim if ((!IsIns && (InstMaskEnd == 63 || InstMaskStart == 0)) || 1947280031Sdim InstMaskEnd == 63 - RLAmt) 1948280031Sdim return 1; 1949280031Sdim 1950280031Sdim return 2; 1951280031Sdim } 1952280031Sdim 1953280031Sdim // For 64-bit values, not all combinations of rotates and masks are 1954280031Sdim // available. Produce one if it is available. 1955309124Sdim SDValue SelectRotMask64(SDValue V, const SDLoc &dl, unsigned RLAmt, 1956309124Sdim bool Repl32, unsigned MaskStart, unsigned MaskEnd, 1957280031Sdim unsigned *InstCnt = nullptr) { 1958280031Sdim // In the notation used by the instructions, 'start' and 'end' are reversed 1959280031Sdim // because bits are counted from high to low order. 1960280031Sdim unsigned InstMaskStart = 64 - MaskEnd - 1, 1961280031Sdim InstMaskEnd = 64 - MaskStart - 1; 1962280031Sdim 1963280031Sdim if (InstCnt) *InstCnt += 1; 1964280031Sdim 1965280031Sdim if (Repl32) { 1966280031Sdim // This rotation amount assumes that the lower 32 bits of the quantity 1967280031Sdim // are replicated in the high 32 bits by the rotation operator (which is 1968280031Sdim // done by rlwinm and friends). 1969280031Sdim assert(InstMaskStart >= 32 && "Mask cannot start out of range"); 1970280031Sdim assert(InstMaskEnd >= 32 && "Mask cannot end out of range"); 1971280031Sdim SDValue Ops[] = 1972327952Sdim { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl), 1973327952Sdim getI32Imm(InstMaskStart - 32, dl), getI32Imm(InstMaskEnd - 32, dl) }; 1974280031Sdim return SDValue(CurDAG->getMachineNode(PPC::RLWINM8, dl, MVT::i64, 1975280031Sdim Ops), 0); 1976280031Sdim } 1977280031Sdim 1978280031Sdim if (InstMaskEnd == 63) { 1979280031Sdim SDValue Ops[] = 1980327952Sdim { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl), 1981327952Sdim getI32Imm(InstMaskStart, dl) }; 1982280031Sdim return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Ops), 0); 1983280031Sdim } 1984280031Sdim 1985280031Sdim if (InstMaskStart == 0) { 1986280031Sdim SDValue Ops[] = 1987327952Sdim { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl), 1988327952Sdim getI32Imm(InstMaskEnd, dl) }; 1989280031Sdim return SDValue(CurDAG->getMachineNode(PPC::RLDICR, dl, MVT::i64, Ops), 0); 1990280031Sdim } 1991280031Sdim 1992280031Sdim if (InstMaskEnd == 63 - RLAmt) { 1993280031Sdim SDValue Ops[] = 1994327952Sdim { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl), 1995327952Sdim getI32Imm(InstMaskStart, dl) }; 1996280031Sdim return SDValue(CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, Ops), 0); 1997280031Sdim } 1998280031Sdim 1999280031Sdim // We cannot do this with a single instruction, so we'll use two. The 2000280031Sdim // problem is that we're not free to choose both a rotation amount and mask 2001280031Sdim // start and end independently. We can choose an arbitrary mask start and 2002280031Sdim // end, but then the rotation amount is fixed. Rotation, however, can be 2003280031Sdim // inverted, and so by applying an "inverse" rotation first, we can get the 2004280031Sdim // desired result. 2005280031Sdim if (InstCnt) *InstCnt += 1; 2006280031Sdim 2007280031Sdim // The rotation mask for the second instruction must be MaskStart. 2008280031Sdim unsigned RLAmt2 = MaskStart; 2009280031Sdim // The first instruction must rotate V so that the overall rotation amount 2010280031Sdim // is RLAmt. 2011280031Sdim unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64; 2012280031Sdim if (RLAmt1) 2013280031Sdim V = SelectRotMask64(V, dl, RLAmt1, false, 0, 63); 2014280031Sdim return SelectRotMask64(V, dl, RLAmt2, false, MaskStart, MaskEnd); 2015280031Sdim } 2016280031Sdim 2017280031Sdim // For 64-bit values, not all combinations of rotates and masks are 2018280031Sdim // available. Produce a rotate-mask-and-insert if one is available. 2019309124Sdim SDValue SelectRotMaskIns64(SDValue Base, SDValue V, const SDLoc &dl, 2020309124Sdim unsigned RLAmt, bool Repl32, unsigned MaskStart, 2021280031Sdim unsigned MaskEnd, unsigned *InstCnt = nullptr) { 2022280031Sdim // In the notation used by the instructions, 'start' and 'end' are reversed 2023280031Sdim // because bits are counted from high to low order. 2024280031Sdim unsigned InstMaskStart = 64 - MaskEnd - 1, 2025280031Sdim InstMaskEnd = 64 - MaskStart - 1; 2026280031Sdim 2027280031Sdim if (InstCnt) *InstCnt += 1; 2028280031Sdim 2029280031Sdim if (Repl32) { 2030280031Sdim // This rotation amount assumes that the lower 32 bits of the quantity 2031280031Sdim // are replicated in the high 32 bits by the rotation operator (which is 2032280031Sdim // done by rlwinm and friends). 2033280031Sdim assert(InstMaskStart >= 32 && "Mask cannot start out of range"); 2034280031Sdim assert(InstMaskEnd >= 32 && "Mask cannot end out of range"); 2035280031Sdim SDValue Ops[] = 2036327952Sdim { ExtendToInt64(Base, dl), ExtendToInt64(V, dl), getI32Imm(RLAmt, dl), 2037327952Sdim getI32Imm(InstMaskStart - 32, dl), getI32Imm(InstMaskEnd - 32, dl) }; 2038280031Sdim return SDValue(CurDAG->getMachineNode(PPC::RLWIMI8, dl, MVT::i64, 2039280031Sdim Ops), 0); 2040280031Sdim } 2041280031Sdim 2042280031Sdim if (InstMaskEnd == 63 - RLAmt) { 2043280031Sdim SDValue Ops[] = 2044327952Sdim { ExtendToInt64(Base, dl), ExtendToInt64(V, dl), getI32Imm(RLAmt, dl), 2045327952Sdim getI32Imm(InstMaskStart, dl) }; 2046280031Sdim return SDValue(CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops), 0); 2047280031Sdim } 2048280031Sdim 2049280031Sdim // We cannot do this with a single instruction, so we'll use two. The 2050280031Sdim // problem is that we're not free to choose both a rotation amount and mask 2051280031Sdim // start and end independently. We can choose an arbitrary mask start and 2052280031Sdim // end, but then the rotation amount is fixed. Rotation, however, can be 2053280031Sdim // inverted, and so by applying an "inverse" rotation first, we can get the 2054280031Sdim // desired result. 2055280031Sdim if (InstCnt) *InstCnt += 1; 2056280031Sdim 2057280031Sdim // The rotation mask for the second instruction must be MaskStart. 2058280031Sdim unsigned RLAmt2 = MaskStart; 2059280031Sdim // The first instruction must rotate V so that the overall rotation amount 2060280031Sdim // is RLAmt. 2061280031Sdim unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64; 2062280031Sdim if (RLAmt1) 2063280031Sdim V = SelectRotMask64(V, dl, RLAmt1, false, 0, 63); 2064280031Sdim return SelectRotMaskIns64(Base, V, dl, RLAmt2, false, MaskStart, MaskEnd); 2065280031Sdim } 2066280031Sdim 2067309124Sdim void SelectAndParts64(const SDLoc &dl, SDValue &Res, unsigned *InstCnt) { 2068280031Sdim if (BPermRewriterNoMasking) 2069280031Sdim return; 2070280031Sdim 2071280031Sdim // The idea here is the same as in the 32-bit version, but with additional 2072280031Sdim // complications from the fact that Repl32 might be true. Because we 2073280031Sdim // aggressively convert bit groups to Repl32 form (which, for small 2074280031Sdim // rotation factors, involves no other change), and then coalesce, it might 2075280031Sdim // be the case that a single 64-bit masking operation could handle both 2076280031Sdim // some Repl32 groups and some non-Repl32 groups. If converting to Repl32 2077280031Sdim // form allowed coalescing, then we must use a 32-bit rotaton in order to 2078280031Sdim // completely capture the new combined bit group. 2079280031Sdim 2080280031Sdim for (ValueRotInfo &VRI : ValueRotsVec) { 2081280031Sdim uint64_t Mask = 0; 2082280031Sdim 2083280031Sdim // We need to add to the mask all bits from the associated bit groups. 2084280031Sdim // If Repl32 is false, we need to add bits from bit groups that have 2085280031Sdim // Repl32 true, but are trivially convertable to Repl32 false. Such a 2086280031Sdim // group is trivially convertable if it overlaps only with the lower 32 2087280031Sdim // bits, and the group has not been coalesced. 2088288943Sdim auto MatchingBG = [VRI](const BitGroup &BG) { 2089280031Sdim if (VRI.V != BG.V) 2090280031Sdim return false; 2091280031Sdim 2092280031Sdim unsigned EffRLAmt = BG.RLAmt; 2093280031Sdim if (!VRI.Repl32 && BG.Repl32) { 2094280031Sdim if (BG.StartIdx < 32 && BG.EndIdx < 32 && BG.StartIdx <= BG.EndIdx && 2095280031Sdim !BG.Repl32Coalesced) { 2096280031Sdim if (BG.Repl32CR) 2097280031Sdim EffRLAmt += 32; 2098280031Sdim } else { 2099280031Sdim return false; 2100280031Sdim } 2101280031Sdim } else if (VRI.Repl32 != BG.Repl32) { 2102280031Sdim return false; 2103280031Sdim } 2104280031Sdim 2105296417Sdim return VRI.RLAmt == EffRLAmt; 2106280031Sdim }; 2107280031Sdim 2108280031Sdim for (auto &BG : BitGroups) { 2109280031Sdim if (!MatchingBG(BG)) 2110280031Sdim continue; 2111280031Sdim 2112280031Sdim if (BG.StartIdx <= BG.EndIdx) { 2113280031Sdim for (unsigned i = BG.StartIdx; i <= BG.EndIdx; ++i) 2114280031Sdim Mask |= (UINT64_C(1) << i); 2115280031Sdim } else { 2116280031Sdim for (unsigned i = BG.StartIdx; i < Bits.size(); ++i) 2117280031Sdim Mask |= (UINT64_C(1) << i); 2118280031Sdim for (unsigned i = 0; i <= BG.EndIdx; ++i) 2119280031Sdim Mask |= (UINT64_C(1) << i); 2120280031Sdim } 2121280031Sdim } 2122280031Sdim 2123280031Sdim // We can use the 32-bit andi/andis technique if the mask does not 2124280031Sdim // require any higher-order bits. This can save an instruction compared 2125280031Sdim // to always using the general 64-bit technique. 2126280031Sdim bool Use32BitInsts = isUInt<32>(Mask); 2127280031Sdim // Compute the masks for andi/andis that would be necessary. 2128280031Sdim unsigned ANDIMask = (Mask & UINT16_MAX), 2129280031Sdim ANDISMask = (Mask >> 16) & UINT16_MAX; 2130280031Sdim 2131280031Sdim bool NeedsRotate = VRI.RLAmt || (VRI.Repl32 && !isUInt<32>(Mask)); 2132280031Sdim 2133280031Sdim unsigned NumAndInsts = (unsigned) NeedsRotate + 2134280031Sdim (unsigned) (bool) Res; 2135280031Sdim if (Use32BitInsts) 2136280031Sdim NumAndInsts += (unsigned) (ANDIMask != 0) + (unsigned) (ANDISMask != 0) + 2137280031Sdim (unsigned) (ANDIMask != 0 && ANDISMask != 0); 2138280031Sdim else 2139327952Sdim NumAndInsts += selectI64ImmInstrCount(Mask) + /* and */ 1; 2140280031Sdim 2141280031Sdim unsigned NumRLInsts = 0; 2142280031Sdim bool FirstBG = true; 2143314564Sdim bool MoreBG = false; 2144280031Sdim for (auto &BG : BitGroups) { 2145314564Sdim if (!MatchingBG(BG)) { 2146314564Sdim MoreBG = true; 2147280031Sdim continue; 2148314564Sdim } 2149280031Sdim NumRLInsts += 2150280031Sdim SelectRotMask64Count(BG.RLAmt, BG.Repl32, BG.StartIdx, BG.EndIdx, 2151280031Sdim !FirstBG); 2152280031Sdim FirstBG = false; 2153280031Sdim } 2154280031Sdim 2155341825Sdim LLVM_DEBUG(dbgs() << "\t\trotation groups for " << VRI.V.getNode() 2156341825Sdim << " RL: " << VRI.RLAmt << (VRI.Repl32 ? " (32):" : ":") 2157341825Sdim << "\n\t\t\tisel using masking: " << NumAndInsts 2158341825Sdim << " using rotates: " << NumRLInsts << "\n"); 2159280031Sdim 2160280031Sdim // When we'd use andi/andis, we bias toward using the rotates (andi only 2161280031Sdim // has a record form, and is cracked on POWER cores). However, when using 2162280031Sdim // general 64-bit constant formation, bias toward the constant form, 2163280031Sdim // because that exposes more opportunities for CSE. 2164280031Sdim if (NumAndInsts > NumRLInsts) 2165280031Sdim continue; 2166314564Sdim // When merging multiple bit groups, instruction or is used. 2167314564Sdim // But when rotate is used, rldimi can inert the rotated value into any 2168314564Sdim // register, so instruction or can be avoided. 2169314564Sdim if ((Use32BitInsts || MoreBG) && NumAndInsts == NumRLInsts) 2170280031Sdim continue; 2171280031Sdim 2172341825Sdim LLVM_DEBUG(dbgs() << "\t\t\t\tusing masking\n"); 2173280031Sdim 2174280031Sdim if (InstCnt) *InstCnt += NumAndInsts; 2175280031Sdim 2176280031Sdim SDValue VRot; 2177280031Sdim // We actually need to generate a rotation if we have a non-zero rotation 2178280031Sdim // factor or, in the Repl32 case, if we care about any of the 2179280031Sdim // higher-order replicated bits. In the latter case, we generate a mask 2180280031Sdim // backward so that it actually includes the entire 64 bits. 2181280031Sdim if (VRI.RLAmt || (VRI.Repl32 && !isUInt<32>(Mask))) 2182280031Sdim VRot = SelectRotMask64(VRI.V, dl, VRI.RLAmt, VRI.Repl32, 2183280031Sdim VRI.Repl32 ? 31 : 0, VRI.Repl32 ? 30 : 63); 2184280031Sdim else 2185280031Sdim VRot = VRI.V; 2186280031Sdim 2187280031Sdim SDValue TotalVal; 2188280031Sdim if (Use32BitInsts) { 2189280031Sdim assert((ANDIMask != 0 || ANDISMask != 0) && 2190280031Sdim "No set bits in mask when using 32-bit ands for 64-bit value"); 2191280031Sdim 2192280031Sdim SDValue ANDIVal, ANDISVal; 2193280031Sdim if (ANDIMask != 0) 2194360784Sdim ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI8_rec, dl, MVT::i64, 2195327952Sdim ExtendToInt64(VRot, dl), 2196327952Sdim getI32Imm(ANDIMask, dl)), 2197327952Sdim 0); 2198280031Sdim if (ANDISMask != 0) 2199360784Sdim ANDISVal = 2200360784Sdim SDValue(CurDAG->getMachineNode(PPC::ANDIS8_rec, dl, MVT::i64, 2201360784Sdim ExtendToInt64(VRot, dl), 2202360784Sdim getI32Imm(ANDISMask, dl)), 2203360784Sdim 0); 2204280031Sdim 2205280031Sdim if (!ANDIVal) 2206280031Sdim TotalVal = ANDISVal; 2207280031Sdim else if (!ANDISVal) 2208280031Sdim TotalVal = ANDIVal; 2209280031Sdim else 2210280031Sdim TotalVal = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, 2211327952Sdim ExtendToInt64(ANDIVal, dl), ANDISVal), 0); 2212280031Sdim } else { 2213327952Sdim TotalVal = SDValue(selectI64Imm(CurDAG, dl, Mask), 0); 2214280031Sdim TotalVal = 2215280031Sdim SDValue(CurDAG->getMachineNode(PPC::AND8, dl, MVT::i64, 2216327952Sdim ExtendToInt64(VRot, dl), TotalVal), 2217327952Sdim 0); 2218280031Sdim } 2219280031Sdim 2220280031Sdim if (!Res) 2221280031Sdim Res = TotalVal; 2222280031Sdim else 2223280031Sdim Res = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, 2224327952Sdim ExtendToInt64(Res, dl), TotalVal), 2225327952Sdim 0); 2226280031Sdim 2227280031Sdim // Now, remove all groups with this underlying value and rotation 2228280031Sdim // factor. 2229288943Sdim eraseMatchingBitGroups(MatchingBG); 2230280031Sdim } 2231280031Sdim } 2232280031Sdim 2233280031Sdim // Instruction selection for the 64-bit case. 2234280031Sdim SDNode *Select64(SDNode *N, bool LateMask, unsigned *InstCnt) { 2235280031Sdim SDLoc dl(N); 2236280031Sdim SDValue Res; 2237280031Sdim 2238280031Sdim if (InstCnt) *InstCnt = 0; 2239280031Sdim 2240280031Sdim // Take care of cases that should use andi/andis first. 2241280031Sdim SelectAndParts64(dl, Res, InstCnt); 2242280031Sdim 2243280031Sdim // If we've not yet selected a 'starting' instruction, and we have no zeros 2244280031Sdim // to fill in, select the (Value, RLAmt) with the highest priority (largest 2245280031Sdim // number of groups), and start with this rotated value. 2246344779Sdim if ((!NeedMask || LateMask) && !Res) { 2247280031Sdim // If we have both Repl32 groups and non-Repl32 groups, the non-Repl32 2248280031Sdim // groups will come first, and so the VRI representing the largest number 2249280031Sdim // of groups might not be first (it might be the first Repl32 groups). 2250280031Sdim unsigned MaxGroupsIdx = 0; 2251280031Sdim if (!ValueRotsVec[0].Repl32) { 2252280031Sdim for (unsigned i = 0, ie = ValueRotsVec.size(); i < ie; ++i) 2253280031Sdim if (ValueRotsVec[i].Repl32) { 2254280031Sdim if (ValueRotsVec[i].NumGroups > ValueRotsVec[0].NumGroups) 2255280031Sdim MaxGroupsIdx = i; 2256280031Sdim break; 2257280031Sdim } 2258280031Sdim } 2259280031Sdim 2260280031Sdim ValueRotInfo &VRI = ValueRotsVec[MaxGroupsIdx]; 2261280031Sdim bool NeedsRotate = false; 2262280031Sdim if (VRI.RLAmt) { 2263280031Sdim NeedsRotate = true; 2264280031Sdim } else if (VRI.Repl32) { 2265280031Sdim for (auto &BG : BitGroups) { 2266280031Sdim if (BG.V != VRI.V || BG.RLAmt != VRI.RLAmt || 2267280031Sdim BG.Repl32 != VRI.Repl32) 2268280031Sdim continue; 2269280031Sdim 2270280031Sdim // We don't need a rotate if the bit group is confined to the lower 2271280031Sdim // 32 bits. 2272280031Sdim if (BG.StartIdx < 32 && BG.EndIdx < 32 && BG.StartIdx < BG.EndIdx) 2273280031Sdim continue; 2274280031Sdim 2275280031Sdim NeedsRotate = true; 2276280031Sdim break; 2277280031Sdim } 2278280031Sdim } 2279280031Sdim 2280280031Sdim if (NeedsRotate) 2281280031Sdim Res = SelectRotMask64(VRI.V, dl, VRI.RLAmt, VRI.Repl32, 2282280031Sdim VRI.Repl32 ? 31 : 0, VRI.Repl32 ? 30 : 63, 2283280031Sdim InstCnt); 2284280031Sdim else 2285280031Sdim Res = VRI.V; 2286280031Sdim 2287280031Sdim // Now, remove all groups with this underlying value and rotation factor. 2288280031Sdim if (Res) 2289288943Sdim eraseMatchingBitGroups([VRI](const BitGroup &BG) { 2290288943Sdim return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt && 2291288943Sdim BG.Repl32 == VRI.Repl32; 2292288943Sdim }); 2293280031Sdim } 2294280031Sdim 2295280031Sdim // Because 64-bit rotates are more flexible than inserts, we might have a 2296280031Sdim // preference regarding which one we do first (to save one instruction). 2297280031Sdim if (!Res) 2298280031Sdim for (auto I = BitGroups.begin(), IE = BitGroups.end(); I != IE; ++I) { 2299280031Sdim if (SelectRotMask64Count(I->RLAmt, I->Repl32, I->StartIdx, I->EndIdx, 2300280031Sdim false) < 2301280031Sdim SelectRotMask64Count(I->RLAmt, I->Repl32, I->StartIdx, I->EndIdx, 2302280031Sdim true)) { 2303280031Sdim if (I != BitGroups.begin()) { 2304280031Sdim BitGroup BG = *I; 2305280031Sdim BitGroups.erase(I); 2306280031Sdim BitGroups.insert(BitGroups.begin(), BG); 2307280031Sdim } 2308280031Sdim 2309280031Sdim break; 2310280031Sdim } 2311280031Sdim } 2312280031Sdim 2313280031Sdim // Insert the other groups (one at a time). 2314280031Sdim for (auto &BG : BitGroups) { 2315280031Sdim if (!Res) 2316280031Sdim Res = SelectRotMask64(BG.V, dl, BG.RLAmt, BG.Repl32, BG.StartIdx, 2317280031Sdim BG.EndIdx, InstCnt); 2318280031Sdim else 2319280031Sdim Res = SelectRotMaskIns64(Res, BG.V, dl, BG.RLAmt, BG.Repl32, 2320280031Sdim BG.StartIdx, BG.EndIdx, InstCnt); 2321280031Sdim } 2322280031Sdim 2323280031Sdim if (LateMask) { 2324280031Sdim uint64_t Mask = getZerosMask(); 2325280031Sdim 2326280031Sdim // We can use the 32-bit andi/andis technique if the mask does not 2327280031Sdim // require any higher-order bits. This can save an instruction compared 2328280031Sdim // to always using the general 64-bit technique. 2329280031Sdim bool Use32BitInsts = isUInt<32>(Mask); 2330280031Sdim // Compute the masks for andi/andis that would be necessary. 2331280031Sdim unsigned ANDIMask = (Mask & UINT16_MAX), 2332280031Sdim ANDISMask = (Mask >> 16) & UINT16_MAX; 2333280031Sdim 2334280031Sdim if (Use32BitInsts) { 2335280031Sdim assert((ANDIMask != 0 || ANDISMask != 0) && 2336280031Sdim "No set bits in mask when using 32-bit ands for 64-bit value"); 2337280031Sdim 2338280031Sdim if (InstCnt) *InstCnt += (unsigned) (ANDIMask != 0) + 2339280031Sdim (unsigned) (ANDISMask != 0) + 2340280031Sdim (unsigned) (ANDIMask != 0 && ANDISMask != 0); 2341280031Sdim 2342280031Sdim SDValue ANDIVal, ANDISVal; 2343280031Sdim if (ANDIMask != 0) 2344360784Sdim ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI8_rec, dl, MVT::i64, 2345360784Sdim ExtendToInt64(Res, dl), 2346360784Sdim getI32Imm(ANDIMask, dl)), 2347360784Sdim 0); 2348280031Sdim if (ANDISMask != 0) 2349360784Sdim ANDISVal = 2350360784Sdim SDValue(CurDAG->getMachineNode(PPC::ANDIS8_rec, dl, MVT::i64, 2351360784Sdim ExtendToInt64(Res, dl), 2352360784Sdim getI32Imm(ANDISMask, dl)), 2353360784Sdim 0); 2354280031Sdim 2355280031Sdim if (!ANDIVal) 2356280031Sdim Res = ANDISVal; 2357280031Sdim else if (!ANDISVal) 2358280031Sdim Res = ANDIVal; 2359280031Sdim else 2360280031Sdim Res = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, 2361327952Sdim ExtendToInt64(ANDIVal, dl), ANDISVal), 0); 2362280031Sdim } else { 2363327952Sdim if (InstCnt) *InstCnt += selectI64ImmInstrCount(Mask) + /* and */ 1; 2364280031Sdim 2365327952Sdim SDValue MaskVal = SDValue(selectI64Imm(CurDAG, dl, Mask), 0); 2366280031Sdim Res = 2367280031Sdim SDValue(CurDAG->getMachineNode(PPC::AND8, dl, MVT::i64, 2368327952Sdim ExtendToInt64(Res, dl), MaskVal), 0); 2369280031Sdim } 2370280031Sdim } 2371280031Sdim 2372280031Sdim return Res.getNode(); 2373280031Sdim } 2374280031Sdim 2375280031Sdim SDNode *Select(SDNode *N, bool LateMask, unsigned *InstCnt = nullptr) { 2376280031Sdim // Fill in BitGroups. 2377280031Sdim collectBitGroups(LateMask); 2378280031Sdim if (BitGroups.empty()) 2379280031Sdim return nullptr; 2380280031Sdim 2381280031Sdim // For 64-bit values, figure out when we can use 32-bit instructions. 2382280031Sdim if (Bits.size() == 64) 2383280031Sdim assignRepl32BitGroups(); 2384280031Sdim 2385280031Sdim // Fill in ValueRotsVec. 2386280031Sdim collectValueRotInfo(); 2387280031Sdim 2388280031Sdim if (Bits.size() == 32) { 2389280031Sdim return Select32(N, LateMask, InstCnt); 2390280031Sdim } else { 2391280031Sdim assert(Bits.size() == 64 && "Not 64 bits here?"); 2392280031Sdim return Select64(N, LateMask, InstCnt); 2393280031Sdim } 2394280031Sdim 2395280031Sdim return nullptr; 2396280031Sdim } 2397280031Sdim 2398288943Sdim void eraseMatchingBitGroups(function_ref<bool(const BitGroup &)> F) { 2399314564Sdim BitGroups.erase(remove_if(BitGroups, F), BitGroups.end()); 2400288943Sdim } 2401288943Sdim 2402280031Sdim SmallVector<ValueBit, 64> Bits; 2403280031Sdim 2404360784Sdim bool NeedMask = false; 2405280031Sdim SmallVector<unsigned, 64> RLAmt; 2406280031Sdim 2407280031Sdim SmallVector<BitGroup, 16> BitGroups; 2408280031Sdim 2409280031Sdim DenseMap<std::pair<SDValue, unsigned>, ValueRotInfo> ValueRots; 2410280031Sdim SmallVector<ValueRotInfo, 16> ValueRotsVec; 2411280031Sdim 2412360784Sdim SelectionDAG *CurDAG = nullptr; 2413280031Sdim 2414280031Sdimpublic: 2415280031Sdim BitPermutationSelector(SelectionDAG *DAG) 2416280031Sdim : CurDAG(DAG) {} 2417280031Sdim 2418280031Sdim // Here we try to match complex bit permutations into a set of 2419280031Sdim // rotate-and-shift/shift/and/or instructions, using a set of heuristics 2420353358Sdim // known to produce optimal code for common cases (like i32 byte swapping). 2421280031Sdim SDNode *Select(SDNode *N) { 2422314564Sdim Memoizer.clear(); 2423314564Sdim auto Result = 2424314564Sdim getValueBits(SDValue(N, 0), N->getValueType(0).getSizeInBits()); 2425314564Sdim if (!Result.first) 2426280031Sdim return nullptr; 2427314564Sdim Bits = std::move(*Result.second); 2428280031Sdim 2429341825Sdim LLVM_DEBUG(dbgs() << "Considering bit-permutation-based instruction" 2430341825Sdim " selection for: "); 2431341825Sdim LLVM_DEBUG(N->dump(CurDAG)); 2432280031Sdim 2433344779Sdim // Fill it RLAmt and set NeedMask. 2434280031Sdim computeRotationAmounts(); 2435280031Sdim 2436344779Sdim if (!NeedMask) 2437280031Sdim return Select(N, false); 2438280031Sdim 2439280031Sdim // We currently have two techniques for handling results with zeros: early 2440280031Sdim // masking (the default) and late masking. Late masking is sometimes more 2441280031Sdim // efficient, but because the structure of the bit groups is different, it 2442280031Sdim // is hard to tell without generating both and comparing the results. With 2443280031Sdim // late masking, we ignore zeros in the resulting value when inserting each 2444280031Sdim // set of bit groups, and then mask in the zeros at the end. With early 2445280031Sdim // masking, we only insert the non-zero parts of the result at every step. 2446280031Sdim 2447341825Sdim unsigned InstCnt = 0, InstCntLateMask = 0; 2448341825Sdim LLVM_DEBUG(dbgs() << "\tEarly masking:\n"); 2449280031Sdim SDNode *RN = Select(N, false, &InstCnt); 2450341825Sdim LLVM_DEBUG(dbgs() << "\t\tisel would use " << InstCnt << " instructions\n"); 2451280031Sdim 2452341825Sdim LLVM_DEBUG(dbgs() << "\tLate masking:\n"); 2453280031Sdim SDNode *RNLM = Select(N, true, &InstCntLateMask); 2454341825Sdim LLVM_DEBUG(dbgs() << "\t\tisel would use " << InstCntLateMask 2455341825Sdim << " instructions\n"); 2456280031Sdim 2457280031Sdim if (InstCnt <= InstCntLateMask) { 2458341825Sdim LLVM_DEBUG(dbgs() << "\tUsing early-masking for isel\n"); 2459280031Sdim return RN; 2460280031Sdim } 2461280031Sdim 2462341825Sdim LLVM_DEBUG(dbgs() << "\tUsing late-masking for isel\n"); 2463280031Sdim return RNLM; 2464280031Sdim } 2465280031Sdim}; 2466280031Sdim 2467327952Sdimclass IntegerCompareEliminator { 2468327952Sdim SelectionDAG *CurDAG; 2469327952Sdim PPCDAGToDAGISel *S; 2470327952Sdim // Conversion type for interpreting results of a 32-bit instruction as 2471327952Sdim // a 64-bit value or vice versa. 2472327952Sdim enum ExtOrTruncConversion { Ext, Trunc }; 2473327952Sdim 2474327952Sdim // Modifiers to guide how an ISD::SETCC node's result is to be computed 2475327952Sdim // in a GPR. 2476327952Sdim // ZExtOrig - use the original condition code, zero-extend value 2477327952Sdim // ZExtInvert - invert the condition code, zero-extend value 2478327952Sdim // SExtOrig - use the original condition code, sign-extend value 2479327952Sdim // SExtInvert - invert the condition code, sign-extend value 2480327952Sdim enum SetccInGPROpts { ZExtOrig, ZExtInvert, SExtOrig, SExtInvert }; 2481327952Sdim 2482327952Sdim // Comparisons against zero to emit GPR code sequences for. Each of these 2483327952Sdim // sequences may need to be emitted for two or more equivalent patterns. 2484327952Sdim // For example (a >= 0) == (a > -1). The direction of the comparison (</>) 2485327952Sdim // matters as well as the extension type: sext (-1/0), zext (1/0). 2486327952Sdim // GEZExt - (zext (LHS >= 0)) 2487327952Sdim // GESExt - (sext (LHS >= 0)) 2488327952Sdim // LEZExt - (zext (LHS <= 0)) 2489327952Sdim // LESExt - (sext (LHS <= 0)) 2490327952Sdim enum ZeroCompare { GEZExt, GESExt, LEZExt, LESExt }; 2491327952Sdim 2492327952Sdim SDNode *tryEXTEND(SDNode *N); 2493327952Sdim SDNode *tryLogicOpOfCompares(SDNode *N); 2494327952Sdim SDValue computeLogicOpInGPR(SDValue LogicOp); 2495327952Sdim SDValue signExtendInputIfNeeded(SDValue Input); 2496327952Sdim SDValue zeroExtendInputIfNeeded(SDValue Input); 2497327952Sdim SDValue addExtOrTrunc(SDValue NatWidthRes, ExtOrTruncConversion Conv); 2498327952Sdim SDValue getCompoundZeroComparisonInGPR(SDValue LHS, SDLoc dl, 2499327952Sdim ZeroCompare CmpTy); 2500327952Sdim SDValue get32BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC, 2501327952Sdim int64_t RHSValue, SDLoc dl); 2502327952Sdim SDValue get32BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC, 2503327952Sdim int64_t RHSValue, SDLoc dl); 2504327952Sdim SDValue get64BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC, 2505327952Sdim int64_t RHSValue, SDLoc dl); 2506327952Sdim SDValue get64BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC, 2507327952Sdim int64_t RHSValue, SDLoc dl); 2508327952Sdim SDValue getSETCCInGPR(SDValue Compare, SetccInGPROpts ConvOpts); 2509327952Sdim 2510327952Sdimpublic: 2511327952Sdim IntegerCompareEliminator(SelectionDAG *DAG, 2512327952Sdim PPCDAGToDAGISel *Sel) : CurDAG(DAG), S(Sel) { 2513327952Sdim assert(CurDAG->getTargetLoweringInfo() 2514327952Sdim .getPointerTy(CurDAG->getDataLayout()).getSizeInBits() == 64 && 2515327952Sdim "Only expecting to use this on 64 bit targets."); 2516327952Sdim } 2517327952Sdim SDNode *Select(SDNode *N) { 2518327952Sdim if (CmpInGPR == ICGPR_None) 2519327952Sdim return nullptr; 2520327952Sdim switch (N->getOpcode()) { 2521327952Sdim default: break; 2522327952Sdim case ISD::ZERO_EXTEND: 2523327952Sdim if (CmpInGPR == ICGPR_Sext || CmpInGPR == ICGPR_SextI32 || 2524327952Sdim CmpInGPR == ICGPR_SextI64) 2525327952Sdim return nullptr; 2526327952Sdim LLVM_FALLTHROUGH; 2527327952Sdim case ISD::SIGN_EXTEND: 2528327952Sdim if (CmpInGPR == ICGPR_Zext || CmpInGPR == ICGPR_ZextI32 || 2529327952Sdim CmpInGPR == ICGPR_ZextI64) 2530327952Sdim return nullptr; 2531327952Sdim return tryEXTEND(N); 2532327952Sdim case ISD::AND: 2533327952Sdim case ISD::OR: 2534327952Sdim case ISD::XOR: 2535327952Sdim return tryLogicOpOfCompares(N); 2536327952Sdim } 2537327952Sdim return nullptr; 2538327952Sdim } 2539327952Sdim}; 2540327952Sdim 2541327952Sdimstatic bool isLogicOp(unsigned Opc) { 2542327952Sdim return Opc == ISD::AND || Opc == ISD::OR || Opc == ISD::XOR; 2543327952Sdim} 2544327952Sdim// The obvious case for wanting to keep the value in a GPR. Namely, the 2545327952Sdim// result of the comparison is actually needed in a GPR. 2546327952SdimSDNode *IntegerCompareEliminator::tryEXTEND(SDNode *N) { 2547327952Sdim assert((N->getOpcode() == ISD::ZERO_EXTEND || 2548327952Sdim N->getOpcode() == ISD::SIGN_EXTEND) && 2549327952Sdim "Expecting a zero/sign extend node!"); 2550327952Sdim SDValue WideRes; 2551327952Sdim // If we are zero-extending the result of a logical operation on i1 2552327952Sdim // values, we can keep the values in GPRs. 2553327952Sdim if (isLogicOp(N->getOperand(0).getOpcode()) && 2554327952Sdim N->getOperand(0).getValueType() == MVT::i1 && 2555327952Sdim N->getOpcode() == ISD::ZERO_EXTEND) 2556327952Sdim WideRes = computeLogicOpInGPR(N->getOperand(0)); 2557327952Sdim else if (N->getOperand(0).getOpcode() != ISD::SETCC) 2558327952Sdim return nullptr; 2559327952Sdim else 2560327952Sdim WideRes = 2561327952Sdim getSETCCInGPR(N->getOperand(0), 2562327952Sdim N->getOpcode() == ISD::SIGN_EXTEND ? 2563327952Sdim SetccInGPROpts::SExtOrig : SetccInGPROpts::ZExtOrig); 2564327952Sdim 2565327952Sdim if (!WideRes) 2566327952Sdim return nullptr; 2567327952Sdim 2568327952Sdim SDLoc dl(N); 2569327952Sdim bool Input32Bit = WideRes.getValueType() == MVT::i32; 2570327952Sdim bool Output32Bit = N->getValueType(0) == MVT::i32; 2571327952Sdim 2572327952Sdim NumSextSetcc += N->getOpcode() == ISD::SIGN_EXTEND ? 1 : 0; 2573327952Sdim NumZextSetcc += N->getOpcode() == ISD::SIGN_EXTEND ? 0 : 1; 2574327952Sdim 2575327952Sdim SDValue ConvOp = WideRes; 2576327952Sdim if (Input32Bit != Output32Bit) 2577327952Sdim ConvOp = addExtOrTrunc(WideRes, Input32Bit ? ExtOrTruncConversion::Ext : 2578327952Sdim ExtOrTruncConversion::Trunc); 2579327952Sdim return ConvOp.getNode(); 2580327952Sdim} 2581327952Sdim 2582327952Sdim// Attempt to perform logical operations on the results of comparisons while 2583327952Sdim// keeping the values in GPRs. Without doing so, these would end up being 2584327952Sdim// lowered to CR-logical operations which suffer from significant latency and 2585327952Sdim// low ILP. 2586327952SdimSDNode *IntegerCompareEliminator::tryLogicOpOfCompares(SDNode *N) { 2587327952Sdim if (N->getValueType(0) != MVT::i1) 2588327952Sdim return nullptr; 2589327952Sdim assert(isLogicOp(N->getOpcode()) && 2590327952Sdim "Expected a logic operation on setcc results."); 2591327952Sdim SDValue LoweredLogical = computeLogicOpInGPR(SDValue(N, 0)); 2592327952Sdim if (!LoweredLogical) 2593327952Sdim return nullptr; 2594327952Sdim 2595327952Sdim SDLoc dl(N); 2596327952Sdim bool IsBitwiseNegate = LoweredLogical.getMachineOpcode() == PPC::XORI8; 2597327952Sdim unsigned SubRegToExtract = IsBitwiseNegate ? PPC::sub_eq : PPC::sub_gt; 2598327952Sdim SDValue CR0Reg = CurDAG->getRegister(PPC::CR0, MVT::i32); 2599327952Sdim SDValue LHS = LoweredLogical.getOperand(0); 2600327952Sdim SDValue RHS = LoweredLogical.getOperand(1); 2601327952Sdim SDValue WideOp; 2602327952Sdim SDValue OpToConvToRecForm; 2603327952Sdim 2604327952Sdim // Look through any 32-bit to 64-bit implicit extend nodes to find the 2605327952Sdim // opcode that is input to the XORI. 2606327952Sdim if (IsBitwiseNegate && 2607327952Sdim LoweredLogical.getOperand(0).getMachineOpcode() == PPC::INSERT_SUBREG) 2608327952Sdim OpToConvToRecForm = LoweredLogical.getOperand(0).getOperand(1); 2609327952Sdim else if (IsBitwiseNegate) 2610327952Sdim // If the input to the XORI isn't an extension, that's what we're after. 2611327952Sdim OpToConvToRecForm = LoweredLogical.getOperand(0); 2612327952Sdim else 2613327952Sdim // If this is not an XORI, it is a reg-reg logical op and we can convert 2614327952Sdim // it to record-form. 2615327952Sdim OpToConvToRecForm = LoweredLogical; 2616327952Sdim 2617327952Sdim // Get the record-form version of the node we're looking to use to get the 2618327952Sdim // CR result from. 2619327952Sdim uint16_t NonRecOpc = OpToConvToRecForm.getMachineOpcode(); 2620327952Sdim int NewOpc = PPCInstrInfo::getRecordFormOpcode(NonRecOpc); 2621327952Sdim 2622327952Sdim // Convert the right node to record-form. This is either the logical we're 2623327952Sdim // looking at or it is the input node to the negation (if we're looking at 2624327952Sdim // a bitwise negation). 2625327952Sdim if (NewOpc != -1 && IsBitwiseNegate) { 2626327952Sdim // The input to the XORI has a record-form. Use it. 2627327952Sdim assert(LoweredLogical.getConstantOperandVal(1) == 1 && 2628327952Sdim "Expected a PPC::XORI8 only for bitwise negation."); 2629327952Sdim // Emit the record-form instruction. 2630327952Sdim std::vector<SDValue> Ops; 2631327952Sdim for (int i = 0, e = OpToConvToRecForm.getNumOperands(); i < e; i++) 2632327952Sdim Ops.push_back(OpToConvToRecForm.getOperand(i)); 2633327952Sdim 2634327952Sdim WideOp = 2635327952Sdim SDValue(CurDAG->getMachineNode(NewOpc, dl, 2636327952Sdim OpToConvToRecForm.getValueType(), 2637327952Sdim MVT::Glue, Ops), 0); 2638327952Sdim } else { 2639327952Sdim assert((NewOpc != -1 || !IsBitwiseNegate) && 2640327952Sdim "No record form available for AND8/OR8/XOR8?"); 2641327952Sdim WideOp = 2642360784Sdim SDValue(CurDAG->getMachineNode(NewOpc == -1 ? PPC::ANDI8_rec : NewOpc, 2643360784Sdim dl, MVT::i64, MVT::Glue, LHS, RHS), 2644360784Sdim 0); 2645327952Sdim } 2646327952Sdim 2647327952Sdim // Select this node to a single bit from CR0 set by the record-form node 2648327952Sdim // just created. For bitwise negation, use the EQ bit which is the equivalent 2649327952Sdim // of negating the result (i.e. it is a bit set when the result of the 2650327952Sdim // operation is zero). 2651327952Sdim SDValue SRIdxVal = 2652327952Sdim CurDAG->getTargetConstant(SubRegToExtract, dl, MVT::i32); 2653327952Sdim SDValue CRBit = 2654327952Sdim SDValue(CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, 2655327952Sdim MVT::i1, CR0Reg, SRIdxVal, 2656327952Sdim WideOp.getValue(1)), 0); 2657327952Sdim return CRBit.getNode(); 2658327952Sdim} 2659327952Sdim 2660327952Sdim// Lower a logical operation on i1 values into a GPR sequence if possible. 2661327952Sdim// The result can be kept in a GPR if requested. 2662327952Sdim// Three types of inputs can be handled: 2663327952Sdim// - SETCC 2664327952Sdim// - TRUNCATE 2665327952Sdim// - Logical operation (AND/OR/XOR) 2666327952Sdim// There is also a special case that is handled (namely a complement operation 2667327952Sdim// achieved with xor %a, -1). 2668327952SdimSDValue IntegerCompareEliminator::computeLogicOpInGPR(SDValue LogicOp) { 2669327952Sdim assert(isLogicOp(LogicOp.getOpcode()) && 2670327952Sdim "Can only handle logic operations here."); 2671327952Sdim assert(LogicOp.getValueType() == MVT::i1 && 2672327952Sdim "Can only handle logic operations on i1 values here."); 2673327952Sdim SDLoc dl(LogicOp); 2674327952Sdim SDValue LHS, RHS; 2675327952Sdim 2676327952Sdim // Special case: xor %a, -1 2677327952Sdim bool IsBitwiseNegation = isBitwiseNot(LogicOp); 2678327952Sdim 2679327952Sdim // Produces a GPR sequence for each operand of the binary logic operation. 2680327952Sdim // For SETCC, it produces the respective comparison, for TRUNCATE it truncates 2681327952Sdim // the value in a GPR and for logic operations, it will recursively produce 2682327952Sdim // a GPR sequence for the operation. 2683327952Sdim auto getLogicOperand = [&] (SDValue Operand) -> SDValue { 2684327952Sdim unsigned OperandOpcode = Operand.getOpcode(); 2685327952Sdim if (OperandOpcode == ISD::SETCC) 2686327952Sdim return getSETCCInGPR(Operand, SetccInGPROpts::ZExtOrig); 2687327952Sdim else if (OperandOpcode == ISD::TRUNCATE) { 2688327952Sdim SDValue InputOp = Operand.getOperand(0); 2689327952Sdim EVT InVT = InputOp.getValueType(); 2690327952Sdim return SDValue(CurDAG->getMachineNode(InVT == MVT::i32 ? PPC::RLDICL_32 : 2691327952Sdim PPC::RLDICL, dl, InVT, InputOp, 2692327952Sdim S->getI64Imm(0, dl), 2693327952Sdim S->getI64Imm(63, dl)), 0); 2694327952Sdim } else if (isLogicOp(OperandOpcode)) 2695327952Sdim return computeLogicOpInGPR(Operand); 2696327952Sdim return SDValue(); 2697327952Sdim }; 2698327952Sdim LHS = getLogicOperand(LogicOp.getOperand(0)); 2699327952Sdim RHS = getLogicOperand(LogicOp.getOperand(1)); 2700327952Sdim 2701327952Sdim // If a GPR sequence can't be produced for the LHS we can't proceed. 2702327952Sdim // Not producing a GPR sequence for the RHS is only a problem if this isn't 2703327952Sdim // a bitwise negation operation. 2704327952Sdim if (!LHS || (!RHS && !IsBitwiseNegation)) 2705327952Sdim return SDValue(); 2706327952Sdim 2707327952Sdim NumLogicOpsOnComparison++; 2708327952Sdim 2709327952Sdim // We will use the inputs as 64-bit values. 2710327952Sdim if (LHS.getValueType() == MVT::i32) 2711327952Sdim LHS = addExtOrTrunc(LHS, ExtOrTruncConversion::Ext); 2712327952Sdim if (!IsBitwiseNegation && RHS.getValueType() == MVT::i32) 2713327952Sdim RHS = addExtOrTrunc(RHS, ExtOrTruncConversion::Ext); 2714327952Sdim 2715327952Sdim unsigned NewOpc; 2716327952Sdim switch (LogicOp.getOpcode()) { 2717327952Sdim default: llvm_unreachable("Unknown logic operation."); 2718327952Sdim case ISD::AND: NewOpc = PPC::AND8; break; 2719327952Sdim case ISD::OR: NewOpc = PPC::OR8; break; 2720327952Sdim case ISD::XOR: NewOpc = PPC::XOR8; break; 2721327952Sdim } 2722327952Sdim 2723327952Sdim if (IsBitwiseNegation) { 2724327952Sdim RHS = S->getI64Imm(1, dl); 2725327952Sdim NewOpc = PPC::XORI8; 2726327952Sdim } 2727327952Sdim 2728327952Sdim return SDValue(CurDAG->getMachineNode(NewOpc, dl, MVT::i64, LHS, RHS), 0); 2729327952Sdim 2730327952Sdim} 2731327952Sdim 2732327952Sdim/// If the value isn't guaranteed to be sign-extended to 64-bits, extend it. 2733327952Sdim/// Otherwise just reinterpret it as a 64-bit value. 2734327952Sdim/// Useful when emitting comparison code for 32-bit values without using 2735327952Sdim/// the compare instruction (which only considers the lower 32-bits). 2736327952SdimSDValue IntegerCompareEliminator::signExtendInputIfNeeded(SDValue Input) { 2737327952Sdim assert(Input.getValueType() == MVT::i32 && 2738327952Sdim "Can only sign-extend 32-bit values here."); 2739327952Sdim unsigned Opc = Input.getOpcode(); 2740327952Sdim 2741327952Sdim // The value was sign extended and then truncated to 32-bits. No need to 2742327952Sdim // sign extend it again. 2743327952Sdim if (Opc == ISD::TRUNCATE && 2744327952Sdim (Input.getOperand(0).getOpcode() == ISD::AssertSext || 2745327952Sdim Input.getOperand(0).getOpcode() == ISD::SIGN_EXTEND)) 2746327952Sdim return addExtOrTrunc(Input, ExtOrTruncConversion::Ext); 2747327952Sdim 2748327952Sdim LoadSDNode *InputLoad = dyn_cast<LoadSDNode>(Input); 2749327952Sdim // The input is a sign-extending load. All ppc sign-extending loads 2750327952Sdim // sign-extend to the full 64-bits. 2751327952Sdim if (InputLoad && InputLoad->getExtensionType() == ISD::SEXTLOAD) 2752327952Sdim return addExtOrTrunc(Input, ExtOrTruncConversion::Ext); 2753327952Sdim 2754327952Sdim ConstantSDNode *InputConst = dyn_cast<ConstantSDNode>(Input); 2755327952Sdim // We don't sign-extend constants. 2756327952Sdim if (InputConst) 2757327952Sdim return addExtOrTrunc(Input, ExtOrTruncConversion::Ext); 2758327952Sdim 2759327952Sdim SDLoc dl(Input); 2760327952Sdim SignExtensionsAdded++; 2761327952Sdim return SDValue(CurDAG->getMachineNode(PPC::EXTSW_32_64, dl, 2762327952Sdim MVT::i64, Input), 0); 2763327952Sdim} 2764327952Sdim 2765327952Sdim/// If the value isn't guaranteed to be zero-extended to 64-bits, extend it. 2766327952Sdim/// Otherwise just reinterpret it as a 64-bit value. 2767327952Sdim/// Useful when emitting comparison code for 32-bit values without using 2768327952Sdim/// the compare instruction (which only considers the lower 32-bits). 2769327952SdimSDValue IntegerCompareEliminator::zeroExtendInputIfNeeded(SDValue Input) { 2770327952Sdim assert(Input.getValueType() == MVT::i32 && 2771327952Sdim "Can only zero-extend 32-bit values here."); 2772327952Sdim unsigned Opc = Input.getOpcode(); 2773327952Sdim 2774327952Sdim // The only condition under which we can omit the actual extend instruction: 2775327952Sdim // - The value is a positive constant 2776327952Sdim // - The value comes from a load that isn't a sign-extending load 2777327952Sdim // An ISD::TRUNCATE needs to be zero-extended unless it is fed by a zext. 2778327952Sdim bool IsTruncateOfZExt = Opc == ISD::TRUNCATE && 2779327952Sdim (Input.getOperand(0).getOpcode() == ISD::AssertZext || 2780327952Sdim Input.getOperand(0).getOpcode() == ISD::ZERO_EXTEND); 2781327952Sdim if (IsTruncateOfZExt) 2782327952Sdim return addExtOrTrunc(Input, ExtOrTruncConversion::Ext); 2783327952Sdim 2784327952Sdim ConstantSDNode *InputConst = dyn_cast<ConstantSDNode>(Input); 2785327952Sdim if (InputConst && InputConst->getSExtValue() >= 0) 2786327952Sdim return addExtOrTrunc(Input, ExtOrTruncConversion::Ext); 2787327952Sdim 2788327952Sdim LoadSDNode *InputLoad = dyn_cast<LoadSDNode>(Input); 2789327952Sdim // The input is a load that doesn't sign-extend (it will be zero-extended). 2790327952Sdim if (InputLoad && InputLoad->getExtensionType() != ISD::SEXTLOAD) 2791327952Sdim return addExtOrTrunc(Input, ExtOrTruncConversion::Ext); 2792327952Sdim 2793327952Sdim // None of the above, need to zero-extend. 2794327952Sdim SDLoc dl(Input); 2795327952Sdim ZeroExtensionsAdded++; 2796327952Sdim return SDValue(CurDAG->getMachineNode(PPC::RLDICL_32_64, dl, MVT::i64, Input, 2797327952Sdim S->getI64Imm(0, dl), 2798327952Sdim S->getI64Imm(32, dl)), 0); 2799327952Sdim} 2800327952Sdim 2801327952Sdim// Handle a 32-bit value in a 64-bit register and vice-versa. These are of 2802327952Sdim// course not actual zero/sign extensions that will generate machine code, 2803327952Sdim// they're just a way to reinterpret a 32 bit value in a register as a 2804327952Sdim// 64 bit value and vice-versa. 2805327952SdimSDValue IntegerCompareEliminator::addExtOrTrunc(SDValue NatWidthRes, 2806327952Sdim ExtOrTruncConversion Conv) { 2807327952Sdim SDLoc dl(NatWidthRes); 2808327952Sdim 2809327952Sdim // For reinterpreting 32-bit values as 64 bit values, we generate 2810327952Sdim // INSERT_SUBREG IMPLICIT_DEF:i64, <input>, TargetConstant:i32<1> 2811327952Sdim if (Conv == ExtOrTruncConversion::Ext) { 2812327952Sdim SDValue ImDef(CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, MVT::i64), 0); 2813327952Sdim SDValue SubRegIdx = 2814327952Sdim CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32); 2815327952Sdim return SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl, MVT::i64, 2816327952Sdim ImDef, NatWidthRes, SubRegIdx), 0); 2817327952Sdim } 2818327952Sdim 2819327952Sdim assert(Conv == ExtOrTruncConversion::Trunc && 2820327952Sdim "Unknown convertion between 32 and 64 bit values."); 2821327952Sdim // For reinterpreting 64-bit values as 32-bit values, we just need to 2822327952Sdim // EXTRACT_SUBREG (i.e. extract the low word). 2823327952Sdim SDValue SubRegIdx = 2824327952Sdim CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32); 2825327952Sdim return SDValue(CurDAG->getMachineNode(PPC::EXTRACT_SUBREG, dl, MVT::i32, 2826327952Sdim NatWidthRes, SubRegIdx), 0); 2827327952Sdim} 2828327952Sdim 2829327952Sdim// Produce a GPR sequence for compound comparisons (<=, >=) against zero. 2830327952Sdim// Handle both zero-extensions and sign-extensions. 2831327952SdimSDValue 2832327952SdimIntegerCompareEliminator::getCompoundZeroComparisonInGPR(SDValue LHS, SDLoc dl, 2833327952Sdim ZeroCompare CmpTy) { 2834327952Sdim EVT InVT = LHS.getValueType(); 2835327952Sdim bool Is32Bit = InVT == MVT::i32; 2836327952Sdim SDValue ToExtend; 2837327952Sdim 2838327952Sdim // Produce the value that needs to be either zero or sign extended. 2839327952Sdim switch (CmpTy) { 2840327952Sdim case ZeroCompare::GEZExt: 2841327952Sdim case ZeroCompare::GESExt: 2842327952Sdim ToExtend = SDValue(CurDAG->getMachineNode(Is32Bit ? PPC::NOR : PPC::NOR8, 2843327952Sdim dl, InVT, LHS, LHS), 0); 2844327952Sdim break; 2845327952Sdim case ZeroCompare::LEZExt: 2846327952Sdim case ZeroCompare::LESExt: { 2847327952Sdim if (Is32Bit) { 2848327952Sdim // Upper 32 bits cannot be undefined for this sequence. 2849327952Sdim LHS = signExtendInputIfNeeded(LHS); 2850327952Sdim SDValue Neg = 2851327952Sdim SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0); 2852327952Sdim ToExtend = 2853327952Sdim SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, 2854327952Sdim Neg, S->getI64Imm(1, dl), 2855327952Sdim S->getI64Imm(63, dl)), 0); 2856327952Sdim } else { 2857327952Sdim SDValue Addi = 2858327952Sdim SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS, 2859327952Sdim S->getI64Imm(~0ULL, dl)), 0); 2860327952Sdim ToExtend = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, 2861327952Sdim Addi, LHS), 0); 2862327952Sdim } 2863327952Sdim break; 2864327952Sdim } 2865327952Sdim } 2866327952Sdim 2867327952Sdim // For 64-bit sequences, the extensions are the same for the GE/LE cases. 2868327952Sdim if (!Is32Bit && 2869327952Sdim (CmpTy == ZeroCompare::GEZExt || CmpTy == ZeroCompare::LEZExt)) 2870327952Sdim return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, 2871327952Sdim ToExtend, S->getI64Imm(1, dl), 2872327952Sdim S->getI64Imm(63, dl)), 0); 2873327952Sdim if (!Is32Bit && 2874327952Sdim (CmpTy == ZeroCompare::GESExt || CmpTy == ZeroCompare::LESExt)) 2875327952Sdim return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, ToExtend, 2876327952Sdim S->getI64Imm(63, dl)), 0); 2877327952Sdim 2878327952Sdim assert(Is32Bit && "Should have handled the 32-bit sequences above."); 2879327952Sdim // For 32-bit sequences, the extensions differ between GE/LE cases. 2880327952Sdim switch (CmpTy) { 2881327952Sdim case ZeroCompare::GEZExt: { 2882327952Sdim SDValue ShiftOps[] = { ToExtend, S->getI32Imm(1, dl), S->getI32Imm(31, dl), 2883327952Sdim S->getI32Imm(31, dl) }; 2884327952Sdim return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, 2885327952Sdim ShiftOps), 0); 2886327952Sdim } 2887327952Sdim case ZeroCompare::GESExt: 2888327952Sdim return SDValue(CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, ToExtend, 2889327952Sdim S->getI32Imm(31, dl)), 0); 2890327952Sdim case ZeroCompare::LEZExt: 2891327952Sdim return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, ToExtend, 2892327952Sdim S->getI32Imm(1, dl)), 0); 2893327952Sdim case ZeroCompare::LESExt: 2894327952Sdim return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, ToExtend, 2895327952Sdim S->getI32Imm(-1, dl)), 0); 2896327952Sdim } 2897327952Sdim 2898327952Sdim // The above case covers all the enumerators so it can't have a default clause 2899327952Sdim // to avoid compiler warnings. 2900327952Sdim llvm_unreachable("Unknown zero-comparison type."); 2901327952Sdim} 2902327952Sdim 2903327952Sdim/// Produces a zero-extended result of comparing two 32-bit values according to 2904327952Sdim/// the passed condition code. 2905327952SdimSDValue 2906327952SdimIntegerCompareEliminator::get32BitZExtCompare(SDValue LHS, SDValue RHS, 2907327952Sdim ISD::CondCode CC, 2908327952Sdim int64_t RHSValue, SDLoc dl) { 2909327952Sdim if (CmpInGPR == ICGPR_I64 || CmpInGPR == ICGPR_SextI64 || 2910327952Sdim CmpInGPR == ICGPR_ZextI64 || CmpInGPR == ICGPR_Sext) 2911327952Sdim return SDValue(); 2912327952Sdim bool IsRHSZero = RHSValue == 0; 2913327952Sdim bool IsRHSOne = RHSValue == 1; 2914327952Sdim bool IsRHSNegOne = RHSValue == -1LL; 2915327952Sdim switch (CC) { 2916327952Sdim default: return SDValue(); 2917327952Sdim case ISD::SETEQ: { 2918327952Sdim // (zext (setcc %a, %b, seteq)) -> (lshr (cntlzw (xor %a, %b)), 5) 2919327952Sdim // (zext (setcc %a, 0, seteq)) -> (lshr (cntlzw %a), 5) 2920327952Sdim SDValue Xor = IsRHSZero ? LHS : 2921327952Sdim SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0); 2922327952Sdim SDValue Clz = 2923327952Sdim SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0); 2924327952Sdim SDValue ShiftOps[] = { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl), 2925327952Sdim S->getI32Imm(31, dl) }; 2926327952Sdim return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, 2927327952Sdim ShiftOps), 0); 2928327952Sdim } 2929327952Sdim case ISD::SETNE: { 2930327952Sdim // (zext (setcc %a, %b, setne)) -> (xor (lshr (cntlzw (xor %a, %b)), 5), 1) 2931327952Sdim // (zext (setcc %a, 0, setne)) -> (xor (lshr (cntlzw %a), 5), 1) 2932327952Sdim SDValue Xor = IsRHSZero ? LHS : 2933327952Sdim SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0); 2934327952Sdim SDValue Clz = 2935327952Sdim SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0); 2936327952Sdim SDValue ShiftOps[] = { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl), 2937327952Sdim S->getI32Imm(31, dl) }; 2938327952Sdim SDValue Shift = 2939327952Sdim SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, ShiftOps), 0); 2940327952Sdim return SDValue(CurDAG->getMachineNode(PPC::XORI, dl, MVT::i32, Shift, 2941327952Sdim S->getI32Imm(1, dl)), 0); 2942327952Sdim } 2943327952Sdim case ISD::SETGE: { 2944327952Sdim // (zext (setcc %a, %b, setge)) -> (xor (lshr (sub %a, %b), 63), 1) 2945327952Sdim // (zext (setcc %a, 0, setge)) -> (lshr (~ %a), 31) 2946327952Sdim if(IsRHSZero) 2947327952Sdim return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt); 2948327952Sdim 2949327952Sdim // Not a special case (i.e. RHS == 0). Handle (%a >= %b) as (%b <= %a) 2950327952Sdim // by swapping inputs and falling through. 2951327952Sdim std::swap(LHS, RHS); 2952327952Sdim ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS); 2953327952Sdim IsRHSZero = RHSConst && RHSConst->isNullValue(); 2954327952Sdim LLVM_FALLTHROUGH; 2955327952Sdim } 2956327952Sdim case ISD::SETLE: { 2957327952Sdim if (CmpInGPR == ICGPR_NonExtIn) 2958327952Sdim return SDValue(); 2959327952Sdim // (zext (setcc %a, %b, setle)) -> (xor (lshr (sub %b, %a), 63), 1) 2960327952Sdim // (zext (setcc %a, 0, setle)) -> (xor (lshr (- %a), 63), 1) 2961327952Sdim if(IsRHSZero) { 2962327952Sdim if (CmpInGPR == ICGPR_NonExtIn) 2963327952Sdim return SDValue(); 2964327952Sdim return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt); 2965327952Sdim } 2966327952Sdim 2967327952Sdim // The upper 32-bits of the register can't be undefined for this sequence. 2968327952Sdim LHS = signExtendInputIfNeeded(LHS); 2969327952Sdim RHS = signExtendInputIfNeeded(RHS); 2970327952Sdim SDValue Sub = 2971327952Sdim SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0); 2972327952Sdim SDValue Shift = 2973327952Sdim SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Sub, 2974327952Sdim S->getI64Imm(1, dl), S->getI64Imm(63, dl)), 2975327952Sdim 0); 2976327952Sdim return 2977327952Sdim SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, 2978327952Sdim MVT::i64, Shift, S->getI32Imm(1, dl)), 0); 2979327952Sdim } 2980327952Sdim case ISD::SETGT: { 2981327952Sdim // (zext (setcc %a, %b, setgt)) -> (lshr (sub %b, %a), 63) 2982327952Sdim // (zext (setcc %a, -1, setgt)) -> (lshr (~ %a), 31) 2983327952Sdim // (zext (setcc %a, 0, setgt)) -> (lshr (- %a), 63) 2984327952Sdim // Handle SETLT -1 (which is equivalent to SETGE 0). 2985327952Sdim if (IsRHSNegOne) 2986327952Sdim return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt); 2987327952Sdim 2988327952Sdim if (IsRHSZero) { 2989327952Sdim if (CmpInGPR == ICGPR_NonExtIn) 2990327952Sdim return SDValue(); 2991327952Sdim // The upper 32-bits of the register can't be undefined for this sequence. 2992327952Sdim LHS = signExtendInputIfNeeded(LHS); 2993327952Sdim RHS = signExtendInputIfNeeded(RHS); 2994327952Sdim SDValue Neg = 2995327952Sdim SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0); 2996327952Sdim return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, 2997327952Sdim Neg, S->getI32Imm(1, dl), S->getI32Imm(63, dl)), 0); 2998327952Sdim } 2999327952Sdim // Not a special case (i.e. RHS == 0 or RHS == -1). Handle (%a > %b) as 3000327952Sdim // (%b < %a) by swapping inputs and falling through. 3001327952Sdim std::swap(LHS, RHS); 3002327952Sdim ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS); 3003327952Sdim IsRHSZero = RHSConst && RHSConst->isNullValue(); 3004327952Sdim IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1; 3005327952Sdim LLVM_FALLTHROUGH; 3006327952Sdim } 3007327952Sdim case ISD::SETLT: { 3008327952Sdim // (zext (setcc %a, %b, setlt)) -> (lshr (sub %a, %b), 63) 3009327952Sdim // (zext (setcc %a, 1, setlt)) -> (xor (lshr (- %a), 63), 1) 3010327952Sdim // (zext (setcc %a, 0, setlt)) -> (lshr %a, 31) 3011327952Sdim // Handle SETLT 1 (which is equivalent to SETLE 0). 3012327952Sdim if (IsRHSOne) { 3013327952Sdim if (CmpInGPR == ICGPR_NonExtIn) 3014327952Sdim return SDValue(); 3015327952Sdim return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt); 3016327952Sdim } 3017327952Sdim 3018327952Sdim if (IsRHSZero) { 3019327952Sdim SDValue ShiftOps[] = { LHS, S->getI32Imm(1, dl), S->getI32Imm(31, dl), 3020327952Sdim S->getI32Imm(31, dl) }; 3021327952Sdim return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, 3022327952Sdim ShiftOps), 0); 3023327952Sdim } 3024327952Sdim 3025327952Sdim if (CmpInGPR == ICGPR_NonExtIn) 3026327952Sdim return SDValue(); 3027327952Sdim // The upper 32-bits of the register can't be undefined for this sequence. 3028327952Sdim LHS = signExtendInputIfNeeded(LHS); 3029327952Sdim RHS = signExtendInputIfNeeded(RHS); 3030327952Sdim SDValue SUBFNode = 3031327952Sdim SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0); 3032327952Sdim return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, 3033327952Sdim SUBFNode, S->getI64Imm(1, dl), 3034327952Sdim S->getI64Imm(63, dl)), 0); 3035327952Sdim } 3036327952Sdim case ISD::SETUGE: 3037327952Sdim // (zext (setcc %a, %b, setuge)) -> (xor (lshr (sub %b, %a), 63), 1) 3038327952Sdim // (zext (setcc %a, %b, setule)) -> (xor (lshr (sub %a, %b), 63), 1) 3039327952Sdim std::swap(LHS, RHS); 3040327952Sdim LLVM_FALLTHROUGH; 3041327952Sdim case ISD::SETULE: { 3042327952Sdim if (CmpInGPR == ICGPR_NonExtIn) 3043327952Sdim return SDValue(); 3044327952Sdim // The upper 32-bits of the register can't be undefined for this sequence. 3045327952Sdim LHS = zeroExtendInputIfNeeded(LHS); 3046327952Sdim RHS = zeroExtendInputIfNeeded(RHS); 3047327952Sdim SDValue Subtract = 3048327952Sdim SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0); 3049327952Sdim SDValue SrdiNode = 3050327952Sdim SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, 3051327952Sdim Subtract, S->getI64Imm(1, dl), 3052327952Sdim S->getI64Imm(63, dl)), 0); 3053327952Sdim return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, SrdiNode, 3054327952Sdim S->getI32Imm(1, dl)), 0); 3055327952Sdim } 3056327952Sdim case ISD::SETUGT: 3057327952Sdim // (zext (setcc %a, %b, setugt)) -> (lshr (sub %b, %a), 63) 3058327952Sdim // (zext (setcc %a, %b, setult)) -> (lshr (sub %a, %b), 63) 3059327952Sdim std::swap(LHS, RHS); 3060327952Sdim LLVM_FALLTHROUGH; 3061327952Sdim case ISD::SETULT: { 3062327952Sdim if (CmpInGPR == ICGPR_NonExtIn) 3063327952Sdim return SDValue(); 3064327952Sdim // The upper 32-bits of the register can't be undefined for this sequence. 3065327952Sdim LHS = zeroExtendInputIfNeeded(LHS); 3066327952Sdim RHS = zeroExtendInputIfNeeded(RHS); 3067327952Sdim SDValue Subtract = 3068327952Sdim SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0); 3069327952Sdim return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, 3070327952Sdim Subtract, S->getI64Imm(1, dl), 3071327952Sdim S->getI64Imm(63, dl)), 0); 3072327952Sdim } 3073327952Sdim } 3074327952Sdim} 3075327952Sdim 3076327952Sdim/// Produces a sign-extended result of comparing two 32-bit values according to 3077327952Sdim/// the passed condition code. 3078327952SdimSDValue 3079327952SdimIntegerCompareEliminator::get32BitSExtCompare(SDValue LHS, SDValue RHS, 3080327952Sdim ISD::CondCode CC, 3081327952Sdim int64_t RHSValue, SDLoc dl) { 3082327952Sdim if (CmpInGPR == ICGPR_I64 || CmpInGPR == ICGPR_SextI64 || 3083327952Sdim CmpInGPR == ICGPR_ZextI64 || CmpInGPR == ICGPR_Zext) 3084327952Sdim return SDValue(); 3085327952Sdim bool IsRHSZero = RHSValue == 0; 3086327952Sdim bool IsRHSOne = RHSValue == 1; 3087327952Sdim bool IsRHSNegOne = RHSValue == -1LL; 3088327952Sdim 3089327952Sdim switch (CC) { 3090327952Sdim default: return SDValue(); 3091327952Sdim case ISD::SETEQ: { 3092327952Sdim // (sext (setcc %a, %b, seteq)) -> 3093327952Sdim // (ashr (shl (ctlz (xor %a, %b)), 58), 63) 3094327952Sdim // (sext (setcc %a, 0, seteq)) -> 3095327952Sdim // (ashr (shl (ctlz %a), 58), 63) 3096327952Sdim SDValue CountInput = IsRHSZero ? LHS : 3097327952Sdim SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0); 3098327952Sdim SDValue Cntlzw = 3099327952Sdim SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, CountInput), 0); 3100327952Sdim SDValue SHLOps[] = { Cntlzw, S->getI32Imm(27, dl), 3101327952Sdim S->getI32Imm(5, dl), S->getI32Imm(31, dl) }; 3102327952Sdim SDValue Slwi = 3103327952Sdim SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, SHLOps), 0); 3104327952Sdim return SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Slwi), 0); 3105327952Sdim } 3106327952Sdim case ISD::SETNE: { 3107327952Sdim // Bitwise xor the operands, count leading zeros, shift right by 5 bits and 3108327952Sdim // flip the bit, finally take 2's complement. 3109327952Sdim // (sext (setcc %a, %b, setne)) -> 3110327952Sdim // (neg (xor (lshr (ctlz (xor %a, %b)), 5), 1)) 3111327952Sdim // Same as above, but the first xor is not needed. 3112327952Sdim // (sext (setcc %a, 0, setne)) -> 3113327952Sdim // (neg (xor (lshr (ctlz %a), 5), 1)) 3114327952Sdim SDValue Xor = IsRHSZero ? LHS : 3115327952Sdim SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0); 3116327952Sdim SDValue Clz = 3117327952Sdim SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0); 3118327952Sdim SDValue ShiftOps[] = 3119327952Sdim { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl), S->getI32Imm(31, dl) }; 3120327952Sdim SDValue Shift = 3121327952Sdim SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, ShiftOps), 0); 3122327952Sdim SDValue Xori = 3123327952Sdim SDValue(CurDAG->getMachineNode(PPC::XORI, dl, MVT::i32, Shift, 3124327952Sdim S->getI32Imm(1, dl)), 0); 3125327952Sdim return SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Xori), 0); 3126327952Sdim } 3127327952Sdim case ISD::SETGE: { 3128327952Sdim // (sext (setcc %a, %b, setge)) -> (add (lshr (sub %a, %b), 63), -1) 3129327952Sdim // (sext (setcc %a, 0, setge)) -> (ashr (~ %a), 31) 3130327952Sdim if (IsRHSZero) 3131327952Sdim return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt); 3132327952Sdim 3133327952Sdim // Not a special case (i.e. RHS == 0). Handle (%a >= %b) as (%b <= %a) 3134327952Sdim // by swapping inputs and falling through. 3135327952Sdim std::swap(LHS, RHS); 3136327952Sdim ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS); 3137327952Sdim IsRHSZero = RHSConst && RHSConst->isNullValue(); 3138327952Sdim LLVM_FALLTHROUGH; 3139327952Sdim } 3140327952Sdim case ISD::SETLE: { 3141327952Sdim if (CmpInGPR == ICGPR_NonExtIn) 3142327952Sdim return SDValue(); 3143327952Sdim // (sext (setcc %a, %b, setge)) -> (add (lshr (sub %b, %a), 63), -1) 3144327952Sdim // (sext (setcc %a, 0, setle)) -> (add (lshr (- %a), 63), -1) 3145327952Sdim if (IsRHSZero) 3146327952Sdim return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt); 3147327952Sdim 3148327952Sdim // The upper 32-bits of the register can't be undefined for this sequence. 3149327952Sdim LHS = signExtendInputIfNeeded(LHS); 3150327952Sdim RHS = signExtendInputIfNeeded(RHS); 3151327952Sdim SDValue SUBFNode = 3152327952Sdim SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, MVT::Glue, 3153327952Sdim LHS, RHS), 0); 3154327952Sdim SDValue Srdi = 3155327952Sdim SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, 3156327952Sdim SUBFNode, S->getI64Imm(1, dl), 3157327952Sdim S->getI64Imm(63, dl)), 0); 3158327952Sdim return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, Srdi, 3159327952Sdim S->getI32Imm(-1, dl)), 0); 3160327952Sdim } 3161327952Sdim case ISD::SETGT: { 3162327952Sdim // (sext (setcc %a, %b, setgt)) -> (ashr (sub %b, %a), 63) 3163327952Sdim // (sext (setcc %a, -1, setgt)) -> (ashr (~ %a), 31) 3164327952Sdim // (sext (setcc %a, 0, setgt)) -> (ashr (- %a), 63) 3165327952Sdim if (IsRHSNegOne) 3166327952Sdim return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt); 3167327952Sdim if (IsRHSZero) { 3168327952Sdim if (CmpInGPR == ICGPR_NonExtIn) 3169327952Sdim return SDValue(); 3170327952Sdim // The upper 32-bits of the register can't be undefined for this sequence. 3171327952Sdim LHS = signExtendInputIfNeeded(LHS); 3172327952Sdim RHS = signExtendInputIfNeeded(RHS); 3173327952Sdim SDValue Neg = 3174327952Sdim SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0); 3175327952Sdim return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, Neg, 3176327952Sdim S->getI64Imm(63, dl)), 0); 3177327952Sdim } 3178327952Sdim // Not a special case (i.e. RHS == 0 or RHS == -1). Handle (%a > %b) as 3179327952Sdim // (%b < %a) by swapping inputs and falling through. 3180327952Sdim std::swap(LHS, RHS); 3181327952Sdim ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS); 3182327952Sdim IsRHSZero = RHSConst && RHSConst->isNullValue(); 3183327952Sdim IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1; 3184327952Sdim LLVM_FALLTHROUGH; 3185327952Sdim } 3186327952Sdim case ISD::SETLT: { 3187327952Sdim // (sext (setcc %a, %b, setgt)) -> (ashr (sub %a, %b), 63) 3188327952Sdim // (sext (setcc %a, 1, setgt)) -> (add (lshr (- %a), 63), -1) 3189327952Sdim // (sext (setcc %a, 0, setgt)) -> (ashr %a, 31) 3190327952Sdim if (IsRHSOne) { 3191327952Sdim if (CmpInGPR == ICGPR_NonExtIn) 3192327952Sdim return SDValue(); 3193327952Sdim return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt); 3194327952Sdim } 3195327952Sdim if (IsRHSZero) 3196327952Sdim return SDValue(CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, LHS, 3197327952Sdim S->getI32Imm(31, dl)), 0); 3198327952Sdim 3199327952Sdim if (CmpInGPR == ICGPR_NonExtIn) 3200327952Sdim return SDValue(); 3201327952Sdim // The upper 32-bits of the register can't be undefined for this sequence. 3202327952Sdim LHS = signExtendInputIfNeeded(LHS); 3203327952Sdim RHS = signExtendInputIfNeeded(RHS); 3204327952Sdim SDValue SUBFNode = 3205327952Sdim SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0); 3206327952Sdim return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, 3207327952Sdim SUBFNode, S->getI64Imm(63, dl)), 0); 3208327952Sdim } 3209327952Sdim case ISD::SETUGE: 3210327952Sdim // (sext (setcc %a, %b, setuge)) -> (add (lshr (sub %a, %b), 63), -1) 3211327952Sdim // (sext (setcc %a, %b, setule)) -> (add (lshr (sub %b, %a), 63), -1) 3212327952Sdim std::swap(LHS, RHS); 3213327952Sdim LLVM_FALLTHROUGH; 3214327952Sdim case ISD::SETULE: { 3215327952Sdim if (CmpInGPR == ICGPR_NonExtIn) 3216327952Sdim return SDValue(); 3217327952Sdim // The upper 32-bits of the register can't be undefined for this sequence. 3218327952Sdim LHS = zeroExtendInputIfNeeded(LHS); 3219327952Sdim RHS = zeroExtendInputIfNeeded(RHS); 3220327952Sdim SDValue Subtract = 3221327952Sdim SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0); 3222327952Sdim SDValue Shift = 3223327952Sdim SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Subtract, 3224327952Sdim S->getI32Imm(1, dl), S->getI32Imm(63,dl)), 3225327952Sdim 0); 3226327952Sdim return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, Shift, 3227327952Sdim S->getI32Imm(-1, dl)), 0); 3228327952Sdim } 3229327952Sdim case ISD::SETUGT: 3230327952Sdim // (sext (setcc %a, %b, setugt)) -> (ashr (sub %b, %a), 63) 3231327952Sdim // (sext (setcc %a, %b, setugt)) -> (ashr (sub %a, %b), 63) 3232327952Sdim std::swap(LHS, RHS); 3233327952Sdim LLVM_FALLTHROUGH; 3234327952Sdim case ISD::SETULT: { 3235327952Sdim if (CmpInGPR == ICGPR_NonExtIn) 3236327952Sdim return SDValue(); 3237327952Sdim // The upper 32-bits of the register can't be undefined for this sequence. 3238327952Sdim LHS = zeroExtendInputIfNeeded(LHS); 3239327952Sdim RHS = zeroExtendInputIfNeeded(RHS); 3240327952Sdim SDValue Subtract = 3241327952Sdim SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0); 3242327952Sdim return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, 3243327952Sdim Subtract, S->getI64Imm(63, dl)), 0); 3244327952Sdim } 3245327952Sdim } 3246327952Sdim} 3247327952Sdim 3248327952Sdim/// Produces a zero-extended result of comparing two 64-bit values according to 3249327952Sdim/// the passed condition code. 3250327952SdimSDValue 3251327952SdimIntegerCompareEliminator::get64BitZExtCompare(SDValue LHS, SDValue RHS, 3252327952Sdim ISD::CondCode CC, 3253327952Sdim int64_t RHSValue, SDLoc dl) { 3254327952Sdim if (CmpInGPR == ICGPR_I32 || CmpInGPR == ICGPR_SextI32 || 3255327952Sdim CmpInGPR == ICGPR_ZextI32 || CmpInGPR == ICGPR_Sext) 3256327952Sdim return SDValue(); 3257327952Sdim bool IsRHSZero = RHSValue == 0; 3258327952Sdim bool IsRHSOne = RHSValue == 1; 3259327952Sdim bool IsRHSNegOne = RHSValue == -1LL; 3260327952Sdim switch (CC) { 3261327952Sdim default: return SDValue(); 3262327952Sdim case ISD::SETEQ: { 3263327952Sdim // (zext (setcc %a, %b, seteq)) -> (lshr (ctlz (xor %a, %b)), 6) 3264327952Sdim // (zext (setcc %a, 0, seteq)) -> (lshr (ctlz %a), 6) 3265327952Sdim SDValue Xor = IsRHSZero ? LHS : 3266327952Sdim SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0); 3267327952Sdim SDValue Clz = 3268327952Sdim SDValue(CurDAG->getMachineNode(PPC::CNTLZD, dl, MVT::i64, Xor), 0); 3269327952Sdim return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Clz, 3270327952Sdim S->getI64Imm(58, dl), 3271327952Sdim S->getI64Imm(63, dl)), 0); 3272327952Sdim } 3273327952Sdim case ISD::SETNE: { 3274327952Sdim // {addc.reg, addc.CA} = (addcarry (xor %a, %b), -1) 3275327952Sdim // (zext (setcc %a, %b, setne)) -> (sube addc.reg, addc.reg, addc.CA) 3276327952Sdim // {addcz.reg, addcz.CA} = (addcarry %a, -1) 3277327952Sdim // (zext (setcc %a, 0, setne)) -> (sube addcz.reg, addcz.reg, addcz.CA) 3278327952Sdim SDValue Xor = IsRHSZero ? LHS : 3279327952Sdim SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0); 3280327952Sdim SDValue AC = 3281327952Sdim SDValue(CurDAG->getMachineNode(PPC::ADDIC8, dl, MVT::i64, MVT::Glue, 3282327952Sdim Xor, S->getI32Imm(~0U, dl)), 0); 3283327952Sdim return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, AC, 3284327952Sdim Xor, AC.getValue(1)), 0); 3285327952Sdim } 3286327952Sdim case ISD::SETGE: { 3287327952Sdim // {subc.reg, subc.CA} = (subcarry %a, %b) 3288327952Sdim // (zext (setcc %a, %b, setge)) -> 3289327952Sdim // (adde (lshr %b, 63), (ashr %a, 63), subc.CA) 3290327952Sdim // (zext (setcc %a, 0, setge)) -> (lshr (~ %a), 63) 3291327952Sdim if (IsRHSZero) 3292327952Sdim return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt); 3293327952Sdim std::swap(LHS, RHS); 3294327952Sdim ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS); 3295327952Sdim IsRHSZero = RHSConst && RHSConst->isNullValue(); 3296327952Sdim LLVM_FALLTHROUGH; 3297327952Sdim } 3298327952Sdim case ISD::SETLE: { 3299327952Sdim // {subc.reg, subc.CA} = (subcarry %b, %a) 3300327952Sdim // (zext (setcc %a, %b, setge)) -> 3301327952Sdim // (adde (lshr %a, 63), (ashr %b, 63), subc.CA) 3302327952Sdim // (zext (setcc %a, 0, setge)) -> (lshr (or %a, (add %a, -1)), 63) 3303327952Sdim if (IsRHSZero) 3304327952Sdim return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt); 3305327952Sdim SDValue ShiftL = 3306327952Sdim SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS, 3307327952Sdim S->getI64Imm(1, dl), 3308327952Sdim S->getI64Imm(63, dl)), 0); 3309327952Sdim SDValue ShiftR = 3310327952Sdim SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, RHS, 3311327952Sdim S->getI64Imm(63, dl)), 0); 3312327952Sdim SDValue SubtractCarry = 3313327952Sdim SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, 3314327952Sdim LHS, RHS), 1); 3315327952Sdim return SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue, 3316327952Sdim ShiftR, ShiftL, SubtractCarry), 0); 3317327952Sdim } 3318327952Sdim case ISD::SETGT: { 3319327952Sdim // {subc.reg, subc.CA} = (subcarry %b, %a) 3320327952Sdim // (zext (setcc %a, %b, setgt)) -> 3321327952Sdim // (xor (adde (lshr %a, 63), (ashr %b, 63), subc.CA), 1) 3322327952Sdim // (zext (setcc %a, 0, setgt)) -> (lshr (nor (add %a, -1), %a), 63) 3323327952Sdim if (IsRHSNegOne) 3324327952Sdim return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt); 3325327952Sdim if (IsRHSZero) { 3326327952Sdim SDValue Addi = 3327327952Sdim SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS, 3328327952Sdim S->getI64Imm(~0ULL, dl)), 0); 3329327952Sdim SDValue Nor = 3330327952Sdim SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, Addi, LHS), 0); 3331327952Sdim return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Nor, 3332327952Sdim S->getI64Imm(1, dl), 3333327952Sdim S->getI64Imm(63, dl)), 0); 3334327952Sdim } 3335327952Sdim std::swap(LHS, RHS); 3336327952Sdim ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS); 3337327952Sdim IsRHSZero = RHSConst && RHSConst->isNullValue(); 3338327952Sdim IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1; 3339327952Sdim LLVM_FALLTHROUGH; 3340327952Sdim } 3341327952Sdim case ISD::SETLT: { 3342327952Sdim // {subc.reg, subc.CA} = (subcarry %a, %b) 3343327952Sdim // (zext (setcc %a, %b, setlt)) -> 3344327952Sdim // (xor (adde (lshr %b, 63), (ashr %a, 63), subc.CA), 1) 3345327952Sdim // (zext (setcc %a, 0, setlt)) -> (lshr %a, 63) 3346327952Sdim if (IsRHSOne) 3347327952Sdim return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt); 3348327952Sdim if (IsRHSZero) 3349327952Sdim return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS, 3350327952Sdim S->getI64Imm(1, dl), 3351327952Sdim S->getI64Imm(63, dl)), 0); 3352327952Sdim SDValue SRADINode = 3353327952Sdim SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, 3354327952Sdim LHS, S->getI64Imm(63, dl)), 0); 3355327952Sdim SDValue SRDINode = 3356327952Sdim SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, 3357327952Sdim RHS, S->getI64Imm(1, dl), 3358327952Sdim S->getI64Imm(63, dl)), 0); 3359327952Sdim SDValue SUBFC8Carry = 3360327952Sdim SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, 3361327952Sdim RHS, LHS), 1); 3362327952Sdim SDValue ADDE8Node = 3363327952Sdim SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue, 3364327952Sdim SRDINode, SRADINode, SUBFC8Carry), 0); 3365327952Sdim return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, 3366327952Sdim ADDE8Node, S->getI64Imm(1, dl)), 0); 3367327952Sdim } 3368327952Sdim case ISD::SETUGE: 3369327952Sdim // {subc.reg, subc.CA} = (subcarry %a, %b) 3370327952Sdim // (zext (setcc %a, %b, setuge)) -> (add (sube %b, %b, subc.CA), 1) 3371327952Sdim std::swap(LHS, RHS); 3372327952Sdim LLVM_FALLTHROUGH; 3373327952Sdim case ISD::SETULE: { 3374327952Sdim // {subc.reg, subc.CA} = (subcarry %b, %a) 3375327952Sdim // (zext (setcc %a, %b, setule)) -> (add (sube %a, %a, subc.CA), 1) 3376327952Sdim SDValue SUBFC8Carry = 3377327952Sdim SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, 3378327952Sdim LHS, RHS), 1); 3379327952Sdim SDValue SUBFE8Node = 3380327952Sdim SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, MVT::Glue, 3381327952Sdim LHS, LHS, SUBFC8Carry), 0); 3382327952Sdim return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, 3383327952Sdim SUBFE8Node, S->getI64Imm(1, dl)), 0); 3384327952Sdim } 3385327952Sdim case ISD::SETUGT: 3386327952Sdim // {subc.reg, subc.CA} = (subcarry %b, %a) 3387327952Sdim // (zext (setcc %a, %b, setugt)) -> -(sube %b, %b, subc.CA) 3388327952Sdim std::swap(LHS, RHS); 3389327952Sdim LLVM_FALLTHROUGH; 3390327952Sdim case ISD::SETULT: { 3391327952Sdim // {subc.reg, subc.CA} = (subcarry %a, %b) 3392327952Sdim // (zext (setcc %a, %b, setult)) -> -(sube %a, %a, subc.CA) 3393327952Sdim SDValue SubtractCarry = 3394327952Sdim SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, 3395327952Sdim RHS, LHS), 1); 3396327952Sdim SDValue ExtSub = 3397327952Sdim SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, 3398327952Sdim LHS, LHS, SubtractCarry), 0); 3399327952Sdim return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, 3400327952Sdim ExtSub), 0); 3401327952Sdim } 3402327952Sdim } 3403327952Sdim} 3404327952Sdim 3405327952Sdim/// Produces a sign-extended result of comparing two 64-bit values according to 3406327952Sdim/// the passed condition code. 3407327952SdimSDValue 3408327952SdimIntegerCompareEliminator::get64BitSExtCompare(SDValue LHS, SDValue RHS, 3409327952Sdim ISD::CondCode CC, 3410327952Sdim int64_t RHSValue, SDLoc dl) { 3411327952Sdim if (CmpInGPR == ICGPR_I32 || CmpInGPR == ICGPR_SextI32 || 3412327952Sdim CmpInGPR == ICGPR_ZextI32 || CmpInGPR == ICGPR_Zext) 3413327952Sdim return SDValue(); 3414327952Sdim bool IsRHSZero = RHSValue == 0; 3415327952Sdim bool IsRHSOne = RHSValue == 1; 3416327952Sdim bool IsRHSNegOne = RHSValue == -1LL; 3417327952Sdim switch (CC) { 3418327952Sdim default: return SDValue(); 3419327952Sdim case ISD::SETEQ: { 3420327952Sdim // {addc.reg, addc.CA} = (addcarry (xor %a, %b), -1) 3421327952Sdim // (sext (setcc %a, %b, seteq)) -> (sube addc.reg, addc.reg, addc.CA) 3422327952Sdim // {addcz.reg, addcz.CA} = (addcarry %a, -1) 3423327952Sdim // (sext (setcc %a, 0, seteq)) -> (sube addcz.reg, addcz.reg, addcz.CA) 3424327952Sdim SDValue AddInput = IsRHSZero ? LHS : 3425327952Sdim SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0); 3426327952Sdim SDValue Addic = 3427327952Sdim SDValue(CurDAG->getMachineNode(PPC::ADDIC8, dl, MVT::i64, MVT::Glue, 3428327952Sdim AddInput, S->getI32Imm(~0U, dl)), 0); 3429327952Sdim return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, Addic, 3430327952Sdim Addic, Addic.getValue(1)), 0); 3431327952Sdim } 3432327952Sdim case ISD::SETNE: { 3433327952Sdim // {subfc.reg, subfc.CA} = (subcarry 0, (xor %a, %b)) 3434327952Sdim // (sext (setcc %a, %b, setne)) -> (sube subfc.reg, subfc.reg, subfc.CA) 3435327952Sdim // {subfcz.reg, subfcz.CA} = (subcarry 0, %a) 3436327952Sdim // (sext (setcc %a, 0, setne)) -> (sube subfcz.reg, subfcz.reg, subfcz.CA) 3437327952Sdim SDValue Xor = IsRHSZero ? LHS : 3438327952Sdim SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0); 3439327952Sdim SDValue SC = 3440327952Sdim SDValue(CurDAG->getMachineNode(PPC::SUBFIC8, dl, MVT::i64, MVT::Glue, 3441327952Sdim Xor, S->getI32Imm(0, dl)), 0); 3442327952Sdim return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, SC, 3443327952Sdim SC, SC.getValue(1)), 0); 3444327952Sdim } 3445327952Sdim case ISD::SETGE: { 3446327952Sdim // {subc.reg, subc.CA} = (subcarry %a, %b) 3447327952Sdim // (zext (setcc %a, %b, setge)) -> 3448327952Sdim // (- (adde (lshr %b, 63), (ashr %a, 63), subc.CA)) 3449327952Sdim // (zext (setcc %a, 0, setge)) -> (~ (ashr %a, 63)) 3450327952Sdim if (IsRHSZero) 3451327952Sdim return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt); 3452327952Sdim std::swap(LHS, RHS); 3453327952Sdim ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS); 3454327952Sdim IsRHSZero = RHSConst && RHSConst->isNullValue(); 3455327952Sdim LLVM_FALLTHROUGH; 3456327952Sdim } 3457327952Sdim case ISD::SETLE: { 3458327952Sdim // {subc.reg, subc.CA} = (subcarry %b, %a) 3459327952Sdim // (zext (setcc %a, %b, setge)) -> 3460327952Sdim // (- (adde (lshr %a, 63), (ashr %b, 63), subc.CA)) 3461327952Sdim // (zext (setcc %a, 0, setge)) -> (ashr (or %a, (add %a, -1)), 63) 3462327952Sdim if (IsRHSZero) 3463327952Sdim return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt); 3464327952Sdim SDValue ShiftR = 3465327952Sdim SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, RHS, 3466327952Sdim S->getI64Imm(63, dl)), 0); 3467327952Sdim SDValue ShiftL = 3468327952Sdim SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS, 3469327952Sdim S->getI64Imm(1, dl), 3470327952Sdim S->getI64Imm(63, dl)), 0); 3471327952Sdim SDValue SubtractCarry = 3472327952Sdim SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, 3473327952Sdim LHS, RHS), 1); 3474327952Sdim SDValue Adde = 3475327952Sdim SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue, 3476327952Sdim ShiftR, ShiftL, SubtractCarry), 0); 3477327952Sdim return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, Adde), 0); 3478327952Sdim } 3479327952Sdim case ISD::SETGT: { 3480327952Sdim // {subc.reg, subc.CA} = (subcarry %b, %a) 3481327952Sdim // (zext (setcc %a, %b, setgt)) -> 3482327952Sdim // -(xor (adde (lshr %a, 63), (ashr %b, 63), subc.CA), 1) 3483327952Sdim // (zext (setcc %a, 0, setgt)) -> (ashr (nor (add %a, -1), %a), 63) 3484327952Sdim if (IsRHSNegOne) 3485327952Sdim return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt); 3486327952Sdim if (IsRHSZero) { 3487327952Sdim SDValue Add = 3488327952Sdim SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS, 3489327952Sdim S->getI64Imm(-1, dl)), 0); 3490327952Sdim SDValue Nor = 3491327952Sdim SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, Add, LHS), 0); 3492327952Sdim return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, Nor, 3493327952Sdim S->getI64Imm(63, dl)), 0); 3494327952Sdim } 3495327952Sdim std::swap(LHS, RHS); 3496327952Sdim ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS); 3497327952Sdim IsRHSZero = RHSConst && RHSConst->isNullValue(); 3498327952Sdim IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1; 3499327952Sdim LLVM_FALLTHROUGH; 3500327952Sdim } 3501327952Sdim case ISD::SETLT: { 3502327952Sdim // {subc.reg, subc.CA} = (subcarry %a, %b) 3503327952Sdim // (zext (setcc %a, %b, setlt)) -> 3504327952Sdim // -(xor (adde (lshr %b, 63), (ashr %a, 63), subc.CA), 1) 3505327952Sdim // (zext (setcc %a, 0, setlt)) -> (ashr %a, 63) 3506327952Sdim if (IsRHSOne) 3507327952Sdim return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt); 3508327952Sdim if (IsRHSZero) { 3509327952Sdim return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, LHS, 3510327952Sdim S->getI64Imm(63, dl)), 0); 3511327952Sdim } 3512327952Sdim SDValue SRADINode = 3513327952Sdim SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, 3514327952Sdim LHS, S->getI64Imm(63, dl)), 0); 3515327952Sdim SDValue SRDINode = 3516327952Sdim SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, 3517327952Sdim RHS, S->getI64Imm(1, dl), 3518327952Sdim S->getI64Imm(63, dl)), 0); 3519327952Sdim SDValue SUBFC8Carry = 3520327952Sdim SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, 3521327952Sdim RHS, LHS), 1); 3522327952Sdim SDValue ADDE8Node = 3523327952Sdim SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, 3524327952Sdim SRDINode, SRADINode, SUBFC8Carry), 0); 3525327952Sdim SDValue XORI8Node = 3526327952Sdim SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, 3527327952Sdim ADDE8Node, S->getI64Imm(1, dl)), 0); 3528327952Sdim return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, 3529327952Sdim XORI8Node), 0); 3530327952Sdim } 3531327952Sdim case ISD::SETUGE: 3532327952Sdim // {subc.reg, subc.CA} = (subcarry %a, %b) 3533327952Sdim // (sext (setcc %a, %b, setuge)) -> ~(sube %b, %b, subc.CA) 3534327952Sdim std::swap(LHS, RHS); 3535327952Sdim LLVM_FALLTHROUGH; 3536327952Sdim case ISD::SETULE: { 3537327952Sdim // {subc.reg, subc.CA} = (subcarry %b, %a) 3538327952Sdim // (sext (setcc %a, %b, setule)) -> ~(sube %a, %a, subc.CA) 3539327952Sdim SDValue SubtractCarry = 3540327952Sdim SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, 3541327952Sdim LHS, RHS), 1); 3542327952Sdim SDValue ExtSub = 3543327952Sdim SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, MVT::Glue, LHS, 3544327952Sdim LHS, SubtractCarry), 0); 3545327952Sdim return SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, 3546327952Sdim ExtSub, ExtSub), 0); 3547327952Sdim } 3548327952Sdim case ISD::SETUGT: 3549327952Sdim // {subc.reg, subc.CA} = (subcarry %b, %a) 3550327952Sdim // (sext (setcc %a, %b, setugt)) -> (sube %b, %b, subc.CA) 3551327952Sdim std::swap(LHS, RHS); 3552327952Sdim LLVM_FALLTHROUGH; 3553327952Sdim case ISD::SETULT: { 3554327952Sdim // {subc.reg, subc.CA} = (subcarry %a, %b) 3555327952Sdim // (sext (setcc %a, %b, setult)) -> (sube %a, %a, subc.CA) 3556327952Sdim SDValue SubCarry = 3557327952Sdim SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, 3558327952Sdim RHS, LHS), 1); 3559327952Sdim return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, 3560327952Sdim LHS, LHS, SubCarry), 0); 3561327952Sdim } 3562327952Sdim } 3563327952Sdim} 3564327952Sdim 3565327952Sdim/// Do all uses of this SDValue need the result in a GPR? 3566327952Sdim/// This is meant to be used on values that have type i1 since 3567327952Sdim/// it is somewhat meaningless to ask if values of other types 3568327952Sdim/// should be kept in GPR's. 3569327952Sdimstatic bool allUsesExtend(SDValue Compare, SelectionDAG *CurDAG) { 3570327952Sdim assert(Compare.getOpcode() == ISD::SETCC && 3571327952Sdim "An ISD::SETCC node required here."); 3572327952Sdim 3573327952Sdim // For values that have a single use, the caller should obviously already have 3574327952Sdim // checked if that use is an extending use. We check the other uses here. 3575327952Sdim if (Compare.hasOneUse()) 3576327952Sdim return true; 3577327952Sdim // We want the value in a GPR if it is being extended, used for a select, or 3578327952Sdim // used in logical operations. 3579327952Sdim for (auto CompareUse : Compare.getNode()->uses()) 3580327952Sdim if (CompareUse->getOpcode() != ISD::SIGN_EXTEND && 3581327952Sdim CompareUse->getOpcode() != ISD::ZERO_EXTEND && 3582327952Sdim CompareUse->getOpcode() != ISD::SELECT && 3583327952Sdim !isLogicOp(CompareUse->getOpcode())) { 3584327952Sdim OmittedForNonExtendUses++; 3585327952Sdim return false; 3586327952Sdim } 3587327952Sdim return true; 3588327952Sdim} 3589327952Sdim 3590327952Sdim/// Returns an equivalent of a SETCC node but with the result the same width as 3591341825Sdim/// the inputs. This can also be used for SELECT_CC if either the true or false 3592327952Sdim/// values is a power of two while the other is zero. 3593327952SdimSDValue IntegerCompareEliminator::getSETCCInGPR(SDValue Compare, 3594327952Sdim SetccInGPROpts ConvOpts) { 3595327952Sdim assert((Compare.getOpcode() == ISD::SETCC || 3596327952Sdim Compare.getOpcode() == ISD::SELECT_CC) && 3597327952Sdim "An ISD::SETCC node required here."); 3598327952Sdim 3599327952Sdim // Don't convert this comparison to a GPR sequence because there are uses 3600327952Sdim // of the i1 result (i.e. uses that require the result in the CR). 3601327952Sdim if ((Compare.getOpcode() == ISD::SETCC) && !allUsesExtend(Compare, CurDAG)) 3602327952Sdim return SDValue(); 3603327952Sdim 3604327952Sdim SDValue LHS = Compare.getOperand(0); 3605327952Sdim SDValue RHS = Compare.getOperand(1); 3606327952Sdim 3607327952Sdim // The condition code is operand 2 for SETCC and operand 4 for SELECT_CC. 3608327952Sdim int CCOpNum = Compare.getOpcode() == ISD::SELECT_CC ? 4 : 2; 3609327952Sdim ISD::CondCode CC = 3610327952Sdim cast<CondCodeSDNode>(Compare.getOperand(CCOpNum))->get(); 3611327952Sdim EVT InputVT = LHS.getValueType(); 3612327952Sdim if (InputVT != MVT::i32 && InputVT != MVT::i64) 3613327952Sdim return SDValue(); 3614327952Sdim 3615327952Sdim if (ConvOpts == SetccInGPROpts::ZExtInvert || 3616327952Sdim ConvOpts == SetccInGPROpts::SExtInvert) 3617360784Sdim CC = ISD::getSetCCInverse(CC, InputVT); 3618327952Sdim 3619327952Sdim bool Inputs32Bit = InputVT == MVT::i32; 3620327952Sdim 3621327952Sdim SDLoc dl(Compare); 3622327952Sdim ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS); 3623327952Sdim int64_t RHSValue = RHSConst ? RHSConst->getSExtValue() : INT64_MAX; 3624327952Sdim bool IsSext = ConvOpts == SetccInGPROpts::SExtOrig || 3625327952Sdim ConvOpts == SetccInGPROpts::SExtInvert; 3626327952Sdim 3627327952Sdim if (IsSext && Inputs32Bit) 3628327952Sdim return get32BitSExtCompare(LHS, RHS, CC, RHSValue, dl); 3629327952Sdim else if (Inputs32Bit) 3630327952Sdim return get32BitZExtCompare(LHS, RHS, CC, RHSValue, dl); 3631327952Sdim else if (IsSext) 3632327952Sdim return get64BitSExtCompare(LHS, RHS, CC, RHSValue, dl); 3633327952Sdim return get64BitZExtCompare(LHS, RHS, CC, RHSValue, dl); 3634327952Sdim} 3635327952Sdim 3636321369Sdim} // end anonymous namespace 3637321369Sdim 3638327952Sdimbool PPCDAGToDAGISel::tryIntCompareInGPR(SDNode *N) { 3639327952Sdim if (N->getValueType(0) != MVT::i32 && 3640327952Sdim N->getValueType(0) != MVT::i64) 3641327952Sdim return false; 3642327952Sdim 3643327952Sdim // This optimization will emit code that assumes 64-bit registers 3644327952Sdim // so we don't want to run it in 32-bit mode. Also don't run it 3645327952Sdim // on functions that are not to be optimized. 3646327952Sdim if (TM.getOptLevel() == CodeGenOpt::None || !TM.isPPC64()) 3647327952Sdim return false; 3648327952Sdim 3649327952Sdim switch (N->getOpcode()) { 3650327952Sdim default: break; 3651327952Sdim case ISD::ZERO_EXTEND: 3652327952Sdim case ISD::SIGN_EXTEND: 3653327952Sdim case ISD::AND: 3654327952Sdim case ISD::OR: 3655327952Sdim case ISD::XOR: { 3656327952Sdim IntegerCompareEliminator ICmpElim(CurDAG, this); 3657327952Sdim if (SDNode *New = ICmpElim.Select(N)) { 3658327952Sdim ReplaceNode(N, New); 3659327952Sdim return true; 3660327952Sdim } 3661327952Sdim } 3662327952Sdim } 3663327952Sdim return false; 3664327952Sdim} 3665327952Sdim 3666309124Sdimbool PPCDAGToDAGISel::tryBitPermutation(SDNode *N) { 3667280031Sdim if (N->getValueType(0) != MVT::i32 && 3668280031Sdim N->getValueType(0) != MVT::i64) 3669309124Sdim return false; 3670280031Sdim 3671280031Sdim if (!UseBitPermRewriter) 3672309124Sdim return false; 3673280031Sdim 3674280031Sdim switch (N->getOpcode()) { 3675280031Sdim default: break; 3676280031Sdim case ISD::ROTL: 3677280031Sdim case ISD::SHL: 3678280031Sdim case ISD::SRL: 3679280031Sdim case ISD::AND: 3680280031Sdim case ISD::OR: { 3681280031Sdim BitPermutationSelector BPS(CurDAG); 3682309124Sdim if (SDNode *New = BPS.Select(N)) { 3683309124Sdim ReplaceNode(N, New); 3684309124Sdim return true; 3685309124Sdim } 3686309124Sdim return false; 3687280031Sdim } 3688280031Sdim } 3689280031Sdim 3690309124Sdim return false; 3691280031Sdim} 3692280031Sdim 3693193323Sed/// SelectCC - Select a comparison of the specified values with the specified 3694193323Sed/// condition code, returning the CR# of the expression. 3695309124SdimSDValue PPCDAGToDAGISel::SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, 3696309124Sdim const SDLoc &dl) { 3697193323Sed // Always select the LHS. 3698193323Sed unsigned Opc; 3699218893Sdim 3700193323Sed if (LHS.getValueType() == MVT::i32) { 3701193323Sed unsigned Imm; 3702193323Sed if (CC == ISD::SETEQ || CC == ISD::SETNE) { 3703193323Sed if (isInt32Immediate(RHS, Imm)) { 3704193323Sed // SETEQ/SETNE comparison with 16-bit immediate, fold it. 3705206083Srdivacky if (isUInt<16>(Imm)) 3706198090Srdivacky return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS, 3707288943Sdim getI32Imm(Imm & 0xFFFF, dl)), 3708288943Sdim 0); 3709193323Sed // If this is a 16-bit signed immediate, fold it. 3710206083Srdivacky if (isInt<16>((int)Imm)) 3711198090Srdivacky return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS, 3712288943Sdim getI32Imm(Imm & 0xFFFF, dl)), 3713288943Sdim 0); 3714218893Sdim 3715193323Sed // For non-equality comparisons, the default code would materialize the 3716193323Sed // constant, then compare against it, like this: 3717193323Sed // lis r2, 4660 3718218893Sdim // ori r2, r2, 22136 3719193323Sed // cmpw cr0, r3, r2 3720193323Sed // Since we are just comparing for equality, we can emit this instead: 3721193323Sed // xoris r0,r3,0x1234 3722193323Sed // cmplwi cr0,r0,0x5678 3723193323Sed // beq cr0,L6 3724198090Srdivacky SDValue Xor(CurDAG->getMachineNode(PPC::XORIS, dl, MVT::i32, LHS, 3725288943Sdim getI32Imm(Imm >> 16, dl)), 0); 3726198090Srdivacky return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, Xor, 3727288943Sdim getI32Imm(Imm & 0xFFFF, dl)), 0); 3728193323Sed } 3729193323Sed Opc = PPC::CMPLW; 3730193323Sed } else if (ISD::isUnsignedIntSetCC(CC)) { 3731206083Srdivacky if (isInt32Immediate(RHS, Imm) && isUInt<16>(Imm)) 3732198090Srdivacky return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS, 3733288943Sdim getI32Imm(Imm & 0xFFFF, dl)), 0); 3734193323Sed Opc = PPC::CMPLW; 3735193323Sed } else { 3736321369Sdim int16_t SImm; 3737193323Sed if (isIntS16Immediate(RHS, SImm)) 3738198090Srdivacky return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS, 3739288943Sdim getI32Imm((int)SImm & 0xFFFF, 3740288943Sdim dl)), 3741193323Sed 0); 3742193323Sed Opc = PPC::CMPW; 3743193323Sed } 3744193323Sed } else if (LHS.getValueType() == MVT::i64) { 3745193323Sed uint64_t Imm; 3746193323Sed if (CC == ISD::SETEQ || CC == ISD::SETNE) { 3747193323Sed if (isInt64Immediate(RHS.getNode(), Imm)) { 3748193323Sed // SETEQ/SETNE comparison with 16-bit immediate, fold it. 3749206083Srdivacky if (isUInt<16>(Imm)) 3750198090Srdivacky return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS, 3751288943Sdim getI32Imm(Imm & 0xFFFF, dl)), 3752288943Sdim 0); 3753193323Sed // If this is a 16-bit signed immediate, fold it. 3754206083Srdivacky if (isInt<16>(Imm)) 3755198090Srdivacky return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS, 3756288943Sdim getI32Imm(Imm & 0xFFFF, dl)), 3757288943Sdim 0); 3758218893Sdim 3759193323Sed // For non-equality comparisons, the default code would materialize the 3760193323Sed // constant, then compare against it, like this: 3761193323Sed // lis r2, 4660 3762218893Sdim // ori r2, r2, 22136 3763193323Sed // cmpd cr0, r3, r2 3764193323Sed // Since we are just comparing for equality, we can emit this instead: 3765193323Sed // xoris r0,r3,0x1234 3766193323Sed // cmpldi cr0,r0,0x5678 3767193323Sed // beq cr0,L6 3768206083Srdivacky if (isUInt<32>(Imm)) { 3769198090Srdivacky SDValue Xor(CurDAG->getMachineNode(PPC::XORIS8, dl, MVT::i64, LHS, 3770288943Sdim getI64Imm(Imm >> 16, dl)), 0); 3771198090Srdivacky return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, Xor, 3772288943Sdim getI64Imm(Imm & 0xFFFF, dl)), 3773288943Sdim 0); 3774193323Sed } 3775193323Sed } 3776193323Sed Opc = PPC::CMPLD; 3777193323Sed } else if (ISD::isUnsignedIntSetCC(CC)) { 3778206083Srdivacky if (isInt64Immediate(RHS.getNode(), Imm) && isUInt<16>(Imm)) 3779198090Srdivacky return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS, 3780288943Sdim getI64Imm(Imm & 0xFFFF, dl)), 0); 3781193323Sed Opc = PPC::CMPLD; 3782193323Sed } else { 3783321369Sdim int16_t SImm; 3784193323Sed if (isIntS16Immediate(RHS, SImm)) 3785198090Srdivacky return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS, 3786288943Sdim getI64Imm(SImm & 0xFFFF, dl)), 3787193323Sed 0); 3788193323Sed Opc = PPC::CMPD; 3789193323Sed } 3790193323Sed } else if (LHS.getValueType() == MVT::f32) { 3791341825Sdim if (PPCSubTarget->hasSPE()) { 3792341825Sdim switch (CC) { 3793341825Sdim default: 3794341825Sdim case ISD::SETEQ: 3795341825Sdim case ISD::SETNE: 3796341825Sdim Opc = PPC::EFSCMPEQ; 3797341825Sdim break; 3798341825Sdim case ISD::SETLT: 3799341825Sdim case ISD::SETGE: 3800341825Sdim case ISD::SETOLT: 3801341825Sdim case ISD::SETOGE: 3802341825Sdim case ISD::SETULT: 3803341825Sdim case ISD::SETUGE: 3804341825Sdim Opc = PPC::EFSCMPLT; 3805341825Sdim break; 3806341825Sdim case ISD::SETGT: 3807341825Sdim case ISD::SETLE: 3808341825Sdim case ISD::SETOGT: 3809341825Sdim case ISD::SETOLE: 3810341825Sdim case ISD::SETUGT: 3811341825Sdim case ISD::SETULE: 3812341825Sdim Opc = PPC::EFSCMPGT; 3813341825Sdim break; 3814341825Sdim } 3815341825Sdim } else 3816341825Sdim Opc = PPC::FCMPUS; 3817341825Sdim } else if (LHS.getValueType() == MVT::f64) { 3818341825Sdim if (PPCSubTarget->hasSPE()) { 3819341825Sdim switch (CC) { 3820341825Sdim default: 3821341825Sdim case ISD::SETEQ: 3822341825Sdim case ISD::SETNE: 3823341825Sdim Opc = PPC::EFDCMPEQ; 3824341825Sdim break; 3825341825Sdim case ISD::SETLT: 3826341825Sdim case ISD::SETGE: 3827341825Sdim case ISD::SETOLT: 3828341825Sdim case ISD::SETOGE: 3829341825Sdim case ISD::SETULT: 3830341825Sdim case ISD::SETUGE: 3831341825Sdim Opc = PPC::EFDCMPLT; 3832341825Sdim break; 3833341825Sdim case ISD::SETGT: 3834341825Sdim case ISD::SETLE: 3835341825Sdim case ISD::SETOGT: 3836341825Sdim case ISD::SETOLE: 3837341825Sdim case ISD::SETUGT: 3838341825Sdim case ISD::SETULE: 3839341825Sdim Opc = PPC::EFDCMPGT; 3840341825Sdim break; 3841341825Sdim } 3842341825Sdim } else 3843341825Sdim Opc = PPCSubTarget->hasVSX() ? PPC::XSCMPUDP : PPC::FCMPUD; 3844193323Sed } else { 3845341825Sdim assert(LHS.getValueType() == MVT::f128 && "Unknown vt!"); 3846341825Sdim assert(PPCSubTarget->hasVSX() && "__float128 requires VSX"); 3847341825Sdim Opc = PPC::XSCMPUQP; 3848193323Sed } 3849198090Srdivacky return SDValue(CurDAG->getMachineNode(Opc, dl, MVT::i32, LHS, RHS), 0); 3850193323Sed} 3851193323Sed 3852360784Sdimstatic PPC::Predicate getPredicateForSetCC(ISD::CondCode CC, const EVT &VT, 3853360784Sdim const PPCSubtarget *Subtarget) { 3854360784Sdim // For SPE instructions, the result is in GT bit of the CR 3855360784Sdim bool UseSPE = Subtarget->hasSPE() && VT.isFloatingPoint(); 3856360784Sdim 3857193323Sed switch (CC) { 3858193323Sed case ISD::SETUEQ: 3859193323Sed case ISD::SETONE: 3860193323Sed case ISD::SETOLE: 3861193323Sed case ISD::SETOGE: 3862198090Srdivacky llvm_unreachable("Should be lowered by legalize!"); 3863198090Srdivacky default: llvm_unreachable("Unknown condition!"); 3864193323Sed case ISD::SETOEQ: 3865360784Sdim case ISD::SETEQ: 3866360784Sdim return UseSPE ? PPC::PRED_GT : PPC::PRED_EQ; 3867193323Sed case ISD::SETUNE: 3868360784Sdim case ISD::SETNE: 3869360784Sdim return UseSPE ? PPC::PRED_LE : PPC::PRED_NE; 3870193323Sed case ISD::SETOLT: 3871360784Sdim case ISD::SETLT: 3872360784Sdim return UseSPE ? PPC::PRED_GT : PPC::PRED_LT; 3873193323Sed case ISD::SETULE: 3874360784Sdim case ISD::SETLE: 3875360784Sdim return UseSPE ? PPC::PRED_LE : PPC::PRED_LE; 3876193323Sed case ISD::SETOGT: 3877360784Sdim case ISD::SETGT: 3878360784Sdim return UseSPE ? PPC::PRED_GT : PPC::PRED_GT; 3879193323Sed case ISD::SETUGE: 3880360784Sdim case ISD::SETGE: 3881360784Sdim return UseSPE ? PPC::PRED_LE : PPC::PRED_GE; 3882193323Sed case ISD::SETO: return PPC::PRED_NU; 3883193323Sed case ISD::SETUO: return PPC::PRED_UN; 3884193323Sed // These two are invalid for floating point. Assume we have int. 3885193323Sed case ISD::SETULT: return PPC::PRED_LT; 3886193323Sed case ISD::SETUGT: return PPC::PRED_GT; 3887193323Sed } 3888193323Sed} 3889193323Sed 3890193323Sed/// getCRIdxForSetCC - Return the index of the condition register field 3891193323Sed/// associated with the SetCC condition, and whether or not the field is 3892193323Sed/// treated as inverted. That is, lt = 0; ge = 0 inverted. 3893261991Sdimstatic unsigned getCRIdxForSetCC(ISD::CondCode CC, bool &Invert) { 3894193323Sed Invert = false; 3895193323Sed switch (CC) { 3896198090Srdivacky default: llvm_unreachable("Unknown condition!"); 3897193323Sed case ISD::SETOLT: 3898193323Sed case ISD::SETLT: return 0; // Bit #0 = SETOLT 3899193323Sed case ISD::SETOGT: 3900193323Sed case ISD::SETGT: return 1; // Bit #1 = SETOGT 3901193323Sed case ISD::SETOEQ: 3902193323Sed case ISD::SETEQ: return 2; // Bit #2 = SETOEQ 3903193323Sed case ISD::SETUO: return 3; // Bit #3 = SETUO 3904193323Sed case ISD::SETUGE: 3905193323Sed case ISD::SETGE: Invert = true; return 0; // !Bit #0 = SETUGE 3906193323Sed case ISD::SETULE: 3907193323Sed case ISD::SETLE: Invert = true; return 1; // !Bit #1 = SETULE 3908193323Sed case ISD::SETUNE: 3909193323Sed case ISD::SETNE: Invert = true; return 2; // !Bit #2 = SETUNE 3910193323Sed case ISD::SETO: Invert = true; return 3; // !Bit #3 = SETO 3911218893Sdim case ISD::SETUEQ: 3912218893Sdim case ISD::SETOGE: 3913218893Sdim case ISD::SETOLE: 3914193323Sed case ISD::SETONE: 3915198090Srdivacky llvm_unreachable("Invalid branch code: should be expanded by legalize"); 3916193323Sed // These are invalid for floating point. Assume integer. 3917193323Sed case ISD::SETULT: return 0; 3918193323Sed case ISD::SETUGT: return 1; 3919193323Sed } 3920193323Sed} 3921193323Sed 3922243830Sdim// getVCmpInst: return the vector compare instruction for the specified 3923243830Sdim// vector type and condition code. Since this is for altivec specific code, 3924288943Sdim// only support the altivec types (v16i8, v8i16, v4i32, v2i64, and v4f32). 3925276479Sdimstatic unsigned int getVCmpInst(MVT VecVT, ISD::CondCode CC, 3926276479Sdim bool HasVSX, bool &Swap, bool &Negate) { 3927276479Sdim Swap = false; 3928276479Sdim Negate = false; 3929243830Sdim 3930276479Sdim if (VecVT.isFloatingPoint()) { 3931276479Sdim /* Handle some cases by swapping input operands. */ 3932276479Sdim switch (CC) { 3933276479Sdim case ISD::SETLE: CC = ISD::SETGE; Swap = true; break; 3934276479Sdim case ISD::SETLT: CC = ISD::SETGT; Swap = true; break; 3935276479Sdim case ISD::SETOLE: CC = ISD::SETOGE; Swap = true; break; 3936276479Sdim case ISD::SETOLT: CC = ISD::SETOGT; Swap = true; break; 3937276479Sdim case ISD::SETUGE: CC = ISD::SETULE; Swap = true; break; 3938276479Sdim case ISD::SETUGT: CC = ISD::SETULT; Swap = true; break; 3939276479Sdim default: break; 3940276479Sdim } 3941276479Sdim /* Handle some cases by negating the result. */ 3942276479Sdim switch (CC) { 3943276479Sdim case ISD::SETNE: CC = ISD::SETEQ; Negate = true; break; 3944276479Sdim case ISD::SETUNE: CC = ISD::SETOEQ; Negate = true; break; 3945276479Sdim case ISD::SETULE: CC = ISD::SETOGT; Negate = true; break; 3946276479Sdim case ISD::SETULT: CC = ISD::SETOGE; Negate = true; break; 3947276479Sdim default: break; 3948276479Sdim } 3949276479Sdim /* We have instructions implementing the remaining cases. */ 3950276479Sdim switch (CC) { 3951276479Sdim case ISD::SETEQ: 3952276479Sdim case ISD::SETOEQ: 3953276479Sdim if (VecVT == MVT::v4f32) 3954276479Sdim return HasVSX ? PPC::XVCMPEQSP : PPC::VCMPEQFP; 3955276479Sdim else if (VecVT == MVT::v2f64) 3956276479Sdim return PPC::XVCMPEQDP; 3957276479Sdim break; 3958276479Sdim case ISD::SETGT: 3959276479Sdim case ISD::SETOGT: 3960276479Sdim if (VecVT == MVT::v4f32) 3961276479Sdim return HasVSX ? PPC::XVCMPGTSP : PPC::VCMPGTFP; 3962276479Sdim else if (VecVT == MVT::v2f64) 3963276479Sdim return PPC::XVCMPGTDP; 3964276479Sdim break; 3965276479Sdim case ISD::SETGE: 3966276479Sdim case ISD::SETOGE: 3967276479Sdim if (VecVT == MVT::v4f32) 3968276479Sdim return HasVSX ? PPC::XVCMPGESP : PPC::VCMPGEFP; 3969276479Sdim else if (VecVT == MVT::v2f64) 3970276479Sdim return PPC::XVCMPGEDP; 3971276479Sdim break; 3972276479Sdim default: 3973276479Sdim break; 3974276479Sdim } 3975276479Sdim llvm_unreachable("Invalid floating-point vector compare condition"); 3976276479Sdim } else { 3977276479Sdim /* Handle some cases by swapping input operands. */ 3978276479Sdim switch (CC) { 3979276479Sdim case ISD::SETGE: CC = ISD::SETLE; Swap = true; break; 3980276479Sdim case ISD::SETLT: CC = ISD::SETGT; Swap = true; break; 3981276479Sdim case ISD::SETUGE: CC = ISD::SETULE; Swap = true; break; 3982276479Sdim case ISD::SETULT: CC = ISD::SETUGT; Swap = true; break; 3983276479Sdim default: break; 3984276479Sdim } 3985276479Sdim /* Handle some cases by negating the result. */ 3986276479Sdim switch (CC) { 3987276479Sdim case ISD::SETNE: CC = ISD::SETEQ; Negate = true; break; 3988276479Sdim case ISD::SETUNE: CC = ISD::SETUEQ; Negate = true; break; 3989276479Sdim case ISD::SETLE: CC = ISD::SETGT; Negate = true; break; 3990276479Sdim case ISD::SETULE: CC = ISD::SETUGT; Negate = true; break; 3991276479Sdim default: break; 3992276479Sdim } 3993276479Sdim /* We have instructions implementing the remaining cases. */ 3994276479Sdim switch (CC) { 3995276479Sdim case ISD::SETEQ: 3996276479Sdim case ISD::SETUEQ: 3997276479Sdim if (VecVT == MVT::v16i8) 3998276479Sdim return PPC::VCMPEQUB; 3999276479Sdim else if (VecVT == MVT::v8i16) 4000276479Sdim return PPC::VCMPEQUH; 4001276479Sdim else if (VecVT == MVT::v4i32) 4002276479Sdim return PPC::VCMPEQUW; 4003288943Sdim else if (VecVT == MVT::v2i64) 4004288943Sdim return PPC::VCMPEQUD; 4005276479Sdim break; 4006276479Sdim case ISD::SETGT: 4007276479Sdim if (VecVT == MVT::v16i8) 4008276479Sdim return PPC::VCMPGTSB; 4009276479Sdim else if (VecVT == MVT::v8i16) 4010276479Sdim return PPC::VCMPGTSH; 4011276479Sdim else if (VecVT == MVT::v4i32) 4012276479Sdim return PPC::VCMPGTSW; 4013288943Sdim else if (VecVT == MVT::v2i64) 4014288943Sdim return PPC::VCMPGTSD; 4015276479Sdim break; 4016276479Sdim case ISD::SETUGT: 4017276479Sdim if (VecVT == MVT::v16i8) 4018276479Sdim return PPC::VCMPGTUB; 4019276479Sdim else if (VecVT == MVT::v8i16) 4020276479Sdim return PPC::VCMPGTUH; 4021276479Sdim else if (VecVT == MVT::v4i32) 4022276479Sdim return PPC::VCMPGTUW; 4023288943Sdim else if (VecVT == MVT::v2i64) 4024288943Sdim return PPC::VCMPGTUD; 4025276479Sdim break; 4026276479Sdim default: 4027276479Sdim break; 4028276479Sdim } 4029276479Sdim llvm_unreachable("Invalid integer vector compare condition"); 4030243830Sdim } 4031243830Sdim} 4032243830Sdim 4033309124Sdimbool PPCDAGToDAGISel::trySETCC(SDNode *N) { 4034261991Sdim SDLoc dl(N); 4035193323Sed unsigned Imm; 4036193323Sed ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get(); 4037288943Sdim EVT PtrVT = 4038288943Sdim CurDAG->getTargetLoweringInfo().getPointerTy(CurDAG->getDataLayout()); 4039224145Sdim bool isPPC64 = (PtrVT == MVT::i64); 4040224145Sdim 4041276479Sdim if (!PPCSubTarget->useCRBits() && 4042276479Sdim isInt32Immediate(N->getOperand(1), Imm)) { 4043193323Sed // We can codegen setcc op, imm very efficiently compared to a brcond. 4044193323Sed // Check for those cases here. 4045193323Sed // setcc op, 0 4046193323Sed if (Imm == 0) { 4047193323Sed SDValue Op = N->getOperand(0); 4048193323Sed switch (CC) { 4049193323Sed default: break; 4050193323Sed case ISD::SETEQ: { 4051198090Srdivacky Op = SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Op), 0); 4052288943Sdim SDValue Ops[] = { Op, getI32Imm(27, dl), getI32Imm(5, dl), 4053288943Sdim getI32Imm(31, dl) }; 4054309124Sdim CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); 4055309124Sdim return true; 4056193323Sed } 4057193323Sed case ISD::SETNE: { 4058224145Sdim if (isPPC64) break; 4059193323Sed SDValue AD = 4060218893Sdim SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, 4061288943Sdim Op, getI32Imm(~0U, dl)), 0); 4062309124Sdim CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, AD, Op, AD.getValue(1)); 4063309124Sdim return true; 4064193323Sed } 4065193323Sed case ISD::SETLT: { 4066288943Sdim SDValue Ops[] = { Op, getI32Imm(1, dl), getI32Imm(31, dl), 4067288943Sdim getI32Imm(31, dl) }; 4068309124Sdim CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); 4069309124Sdim return true; 4070193323Sed } 4071193323Sed case ISD::SETGT: { 4072193323Sed SDValue T = 4073198090Srdivacky SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Op), 0); 4074198090Srdivacky T = SDValue(CurDAG->getMachineNode(PPC::ANDC, dl, MVT::i32, T, Op), 0); 4075288943Sdim SDValue Ops[] = { T, getI32Imm(1, dl), getI32Imm(31, dl), 4076288943Sdim getI32Imm(31, dl) }; 4077309124Sdim CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); 4078309124Sdim return true; 4079193323Sed } 4080193323Sed } 4081193323Sed } else if (Imm == ~0U) { // setcc op, -1 4082193323Sed SDValue Op = N->getOperand(0); 4083193323Sed switch (CC) { 4084193323Sed default: break; 4085193323Sed case ISD::SETEQ: 4086224145Sdim if (isPPC64) break; 4087218893Sdim Op = SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, 4088288943Sdim Op, getI32Imm(1, dl)), 0); 4089309124Sdim CurDAG->SelectNodeTo(N, PPC::ADDZE, MVT::i32, 4090309124Sdim SDValue(CurDAG->getMachineNode(PPC::LI, dl, 4091309124Sdim MVT::i32, 4092309124Sdim getI32Imm(0, dl)), 4093309124Sdim 0), Op.getValue(1)); 4094309124Sdim return true; 4095193323Sed case ISD::SETNE: { 4096224145Sdim if (isPPC64) break; 4097198090Srdivacky Op = SDValue(CurDAG->getMachineNode(PPC::NOR, dl, MVT::i32, Op, Op), 0); 4098218893Sdim SDNode *AD = CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, 4099288943Sdim Op, getI32Imm(~0U, dl)); 4100309124Sdim CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, SDValue(AD, 0), Op, 4101309124Sdim SDValue(AD, 1)); 4102309124Sdim return true; 4103193323Sed } 4104193323Sed case ISD::SETLT: { 4105198090Srdivacky SDValue AD = SDValue(CurDAG->getMachineNode(PPC::ADDI, dl, MVT::i32, Op, 4106288943Sdim getI32Imm(1, dl)), 0); 4107198090Srdivacky SDValue AN = SDValue(CurDAG->getMachineNode(PPC::AND, dl, MVT::i32, AD, 4108198090Srdivacky Op), 0); 4109288943Sdim SDValue Ops[] = { AN, getI32Imm(1, dl), getI32Imm(31, dl), 4110288943Sdim getI32Imm(31, dl) }; 4111309124Sdim CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); 4112309124Sdim return true; 4113193323Sed } 4114193323Sed case ISD::SETGT: { 4115288943Sdim SDValue Ops[] = { Op, getI32Imm(1, dl), getI32Imm(31, dl), 4116288943Sdim getI32Imm(31, dl) }; 4117288943Sdim Op = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); 4118309124Sdim CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Op, getI32Imm(1, dl)); 4119309124Sdim return true; 4120193323Sed } 4121193323Sed } 4122193323Sed } 4123193323Sed } 4124218893Sdim 4125243830Sdim SDValue LHS = N->getOperand(0); 4126243830Sdim SDValue RHS = N->getOperand(1); 4127243830Sdim 4128243830Sdim // Altivec Vector compare instructions do not set any CR register by default and 4129243830Sdim // vector compare operations return the same type as the operands. 4130243830Sdim if (LHS.getValueType().isVector()) { 4131341825Sdim if (PPCSubTarget->hasQPX() || PPCSubTarget->hasSPE()) 4132309124Sdim return false; 4133288943Sdim 4134243830Sdim EVT VecVT = LHS.getValueType(); 4135276479Sdim bool Swap, Negate; 4136276479Sdim unsigned int VCmpInst = getVCmpInst(VecVT.getSimpleVT(), CC, 4137276479Sdim PPCSubTarget->hasVSX(), Swap, Negate); 4138276479Sdim if (Swap) 4139276479Sdim std::swap(LHS, RHS); 4140243830Sdim 4141288943Sdim EVT ResVT = VecVT.changeVectorElementTypeToInteger(); 4142276479Sdim if (Negate) { 4143288943Sdim SDValue VCmp(CurDAG->getMachineNode(VCmpInst, dl, ResVT, LHS, RHS), 0); 4144309124Sdim CurDAG->SelectNodeTo(N, PPCSubTarget->hasVSX() ? PPC::XXLNOR : PPC::VNOR, 4145309124Sdim ResVT, VCmp, VCmp); 4146309124Sdim return true; 4147243830Sdim } 4148276479Sdim 4149309124Sdim CurDAG->SelectNodeTo(N, VCmpInst, ResVT, LHS, RHS); 4150309124Sdim return true; 4151243830Sdim } 4152243830Sdim 4153276479Sdim if (PPCSubTarget->useCRBits()) 4154309124Sdim return false; 4155276479Sdim 4156193323Sed bool Inv; 4157261991Sdim unsigned Idx = getCRIdxForSetCC(CC, Inv); 4158243830Sdim SDValue CCReg = SelectCC(LHS, RHS, CC, dl); 4159193323Sed SDValue IntCR; 4160218893Sdim 4161341825Sdim // SPE e*cmp* instructions only set the 'gt' bit, so hard-code that 4162341825Sdim // The correct compare instruction is already set by SelectCC() 4163341825Sdim if (PPCSubTarget->hasSPE() && LHS.getValueType().isFloatingPoint()) { 4164341825Sdim Idx = 1; 4165341825Sdim } 4166341825Sdim 4167193323Sed // Force the ccreg into CR7. 4168193323Sed SDValue CR7Reg = CurDAG->getRegister(PPC::CR7, MVT::i32); 4169218893Sdim 4170276479Sdim SDValue InFlag(nullptr, 0); // Null incoming flag value. 4171218893Sdim CCReg = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, CR7Reg, CCReg, 4172193323Sed InFlag).getValue(1); 4173218893Sdim 4174261991Sdim IntCR = SDValue(CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, CR7Reg, 4175261991Sdim CCReg), 0); 4176218893Sdim 4177288943Sdim SDValue Ops[] = { IntCR, getI32Imm((32 - (3 - Idx)) & 31, dl), 4178288943Sdim getI32Imm(31, dl), getI32Imm(31, dl) }; 4179309124Sdim if (!Inv) { 4180309124Sdim CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); 4181309124Sdim return true; 4182309124Sdim } 4183193323Sed 4184193323Sed // Get the specified bit. 4185193323Sed SDValue Tmp = 4186251662Sdim SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); 4187309124Sdim CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Tmp, getI32Imm(1, dl)); 4188309124Sdim return true; 4189193323Sed} 4190193323Sed 4191321369Sdim/// Does this node represent a load/store node whose address can be represented 4192321369Sdim/// with a register plus an immediate that's a multiple of \p Val: 4193321369Sdimbool PPCDAGToDAGISel::isOffsetMultipleOf(SDNode *N, unsigned Val) const { 4194321369Sdim LoadSDNode *LDN = dyn_cast<LoadSDNode>(N); 4195321369Sdim StoreSDNode *STN = dyn_cast<StoreSDNode>(N); 4196321369Sdim SDValue AddrOp; 4197321369Sdim if (LDN) 4198321369Sdim AddrOp = LDN->getOperand(1); 4199321369Sdim else if (STN) 4200321369Sdim AddrOp = STN->getOperand(2); 4201321369Sdim 4202341825Sdim // If the address points a frame object or a frame object with an offset, 4203341825Sdim // we need to check the object alignment. 4204321369Sdim short Imm = 0; 4205341825Sdim if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>( 4206341825Sdim AddrOp.getOpcode() == ISD::ADD ? AddrOp.getOperand(0) : 4207341825Sdim AddrOp)) { 4208327952Sdim // If op0 is a frame index that is under aligned, we can't do it either, 4209327952Sdim // because it is translated to r31 or r1 + slot + offset. We won't know the 4210327952Sdim // slot number until the stack frame is finalized. 4211341825Sdim const MachineFrameInfo &MFI = CurDAG->getMachineFunction().getFrameInfo(); 4212341825Sdim unsigned SlotAlign = MFI.getObjectAlignment(FI->getIndex()); 4213341825Sdim if ((SlotAlign % Val) != 0) 4214341825Sdim return false; 4215341825Sdim 4216341825Sdim // If we have an offset, we need further check on the offset. 4217341825Sdim if (AddrOp.getOpcode() != ISD::ADD) 4218341825Sdim return true; 4219327952Sdim } 4220321369Sdim 4221341825Sdim if (AddrOp.getOpcode() == ISD::ADD) 4222341825Sdim return isIntS16Immediate(AddrOp.getOperand(1), Imm) && !(Imm % Val); 4223341825Sdim 4224321369Sdim // If the address comes from the outside, the offset will be zero. 4225321369Sdim return AddrOp.getOpcode() == ISD::CopyFromReg; 4226321369Sdim} 4227321369Sdim 4228309124Sdimvoid PPCDAGToDAGISel::transferMemOperands(SDNode *N, SDNode *Result) { 4229288943Sdim // Transfer memoperands. 4230344779Sdim MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand(); 4231344779Sdim CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {MemOp}); 4232288943Sdim} 4233193323Sed 4234344779Sdimstatic bool mayUseP9Setb(SDNode *N, const ISD::CondCode &CC, SelectionDAG *DAG, 4235344779Sdim bool &NeedSwapOps, bool &IsUnCmp) { 4236344779Sdim 4237344779Sdim assert(N->getOpcode() == ISD::SELECT_CC && "Expecting a SELECT_CC here."); 4238344779Sdim 4239344779Sdim SDValue LHS = N->getOperand(0); 4240344779Sdim SDValue RHS = N->getOperand(1); 4241344779Sdim SDValue TrueRes = N->getOperand(2); 4242344779Sdim SDValue FalseRes = N->getOperand(3); 4243344779Sdim ConstantSDNode *TrueConst = dyn_cast<ConstantSDNode>(TrueRes); 4244344779Sdim if (!TrueConst) 4245344779Sdim return false; 4246344779Sdim 4247344779Sdim assert((N->getSimpleValueType(0) == MVT::i64 || 4248344779Sdim N->getSimpleValueType(0) == MVT::i32) && 4249344779Sdim "Expecting either i64 or i32 here."); 4250344779Sdim 4251344779Sdim // We are looking for any of: 4252344779Sdim // (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, cc2)), cc1) 4253344779Sdim // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, cc2)), cc1) 4254344779Sdim // (select_cc lhs, rhs, 0, (select_cc [lr]hs, [lr]hs, 1, -1, cc2), seteq) 4255344779Sdim // (select_cc lhs, rhs, 0, (select_cc [lr]hs, [lr]hs, -1, 1, cc2), seteq) 4256344779Sdim int64_t TrueResVal = TrueConst->getSExtValue(); 4257344779Sdim if ((TrueResVal < -1 || TrueResVal > 1) || 4258344779Sdim (TrueResVal == -1 && FalseRes.getOpcode() != ISD::ZERO_EXTEND) || 4259344779Sdim (TrueResVal == 1 && FalseRes.getOpcode() != ISD::SIGN_EXTEND) || 4260344779Sdim (TrueResVal == 0 && 4261344779Sdim (FalseRes.getOpcode() != ISD::SELECT_CC || CC != ISD::SETEQ))) 4262344779Sdim return false; 4263344779Sdim 4264344779Sdim bool InnerIsSel = FalseRes.getOpcode() == ISD::SELECT_CC; 4265344779Sdim SDValue SetOrSelCC = InnerIsSel ? FalseRes : FalseRes.getOperand(0); 4266344779Sdim if (SetOrSelCC.getOpcode() != ISD::SETCC && 4267344779Sdim SetOrSelCC.getOpcode() != ISD::SELECT_CC) 4268344779Sdim return false; 4269344779Sdim 4270344779Sdim // Without this setb optimization, the outer SELECT_CC will be manually 4271344779Sdim // selected to SELECT_CC_I4/SELECT_CC_I8 Pseudo, then expand-isel-pseudos pass 4272353358Sdim // transforms pseudo instruction to isel instruction. When there are more than 4273344779Sdim // one use for result like zext/sext, with current optimization we only see 4274344779Sdim // isel is replaced by setb but can't see any significant gain. Since 4275344779Sdim // setb has longer latency than original isel, we should avoid this. Another 4276344779Sdim // point is that setb requires comparison always kept, it can break the 4277353358Sdim // opportunity to get the comparison away if we have in future. 4278344779Sdim if (!SetOrSelCC.hasOneUse() || (!InnerIsSel && !FalseRes.hasOneUse())) 4279344779Sdim return false; 4280344779Sdim 4281344779Sdim SDValue InnerLHS = SetOrSelCC.getOperand(0); 4282344779Sdim SDValue InnerRHS = SetOrSelCC.getOperand(1); 4283344779Sdim ISD::CondCode InnerCC = 4284344779Sdim cast<CondCodeSDNode>(SetOrSelCC.getOperand(InnerIsSel ? 4 : 2))->get(); 4285344779Sdim // If the inner comparison is a select_cc, make sure the true/false values are 4286344779Sdim // 1/-1 and canonicalize it if needed. 4287344779Sdim if (InnerIsSel) { 4288344779Sdim ConstantSDNode *SelCCTrueConst = 4289344779Sdim dyn_cast<ConstantSDNode>(SetOrSelCC.getOperand(2)); 4290344779Sdim ConstantSDNode *SelCCFalseConst = 4291344779Sdim dyn_cast<ConstantSDNode>(SetOrSelCC.getOperand(3)); 4292344779Sdim if (!SelCCTrueConst || !SelCCFalseConst) 4293344779Sdim return false; 4294344779Sdim int64_t SelCCTVal = SelCCTrueConst->getSExtValue(); 4295344779Sdim int64_t SelCCFVal = SelCCFalseConst->getSExtValue(); 4296344779Sdim // The values must be -1/1 (requiring a swap) or 1/-1. 4297344779Sdim if (SelCCTVal == -1 && SelCCFVal == 1) { 4298344779Sdim std::swap(InnerLHS, InnerRHS); 4299344779Sdim } else if (SelCCTVal != 1 || SelCCFVal != -1) 4300344779Sdim return false; 4301341825Sdim } 4302344779Sdim 4303344779Sdim // Canonicalize unsigned case 4304344779Sdim if (InnerCC == ISD::SETULT || InnerCC == ISD::SETUGT) { 4305344779Sdim IsUnCmp = true; 4306344779Sdim InnerCC = (InnerCC == ISD::SETULT) ? ISD::SETLT : ISD::SETGT; 4307341825Sdim } 4308344779Sdim 4309344779Sdim bool InnerSwapped = false; 4310344779Sdim if (LHS == InnerRHS && RHS == InnerLHS) 4311344779Sdim InnerSwapped = true; 4312344779Sdim else if (LHS != InnerLHS || RHS != InnerRHS) 4313344779Sdim return false; 4314344779Sdim 4315344779Sdim switch (CC) { 4316344779Sdim // (select_cc lhs, rhs, 0, \ 4317344779Sdim // (select_cc [lr]hs, [lr]hs, 1, -1, setlt/setgt), seteq) 4318344779Sdim case ISD::SETEQ: 4319344779Sdim if (!InnerIsSel) 4320344779Sdim return false; 4321344779Sdim if (InnerCC != ISD::SETLT && InnerCC != ISD::SETGT) 4322344779Sdim return false; 4323344779Sdim NeedSwapOps = (InnerCC == ISD::SETGT) ? InnerSwapped : !InnerSwapped; 4324344779Sdim break; 4325344779Sdim 4326344779Sdim // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, setne)), setu?lt) 4327344779Sdim // (select_cc lhs, rhs, -1, (zext (setcc lhs, rhs, setgt)), setu?lt) 4328344779Sdim // (select_cc lhs, rhs, -1, (zext (setcc rhs, lhs, setlt)), setu?lt) 4329344779Sdim // (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, setne)), setu?lt) 4330344779Sdim // (select_cc lhs, rhs, 1, (sext (setcc lhs, rhs, setgt)), setu?lt) 4331344779Sdim // (select_cc lhs, rhs, 1, (sext (setcc rhs, lhs, setlt)), setu?lt) 4332344779Sdim case ISD::SETULT: 4333344779Sdim if (!IsUnCmp && InnerCC != ISD::SETNE) 4334344779Sdim return false; 4335344779Sdim IsUnCmp = true; 4336344779Sdim LLVM_FALLTHROUGH; 4337344779Sdim case ISD::SETLT: 4338344779Sdim if (InnerCC == ISD::SETNE || (InnerCC == ISD::SETGT && !InnerSwapped) || 4339344779Sdim (InnerCC == ISD::SETLT && InnerSwapped)) 4340344779Sdim NeedSwapOps = (TrueResVal == 1); 4341344779Sdim else 4342344779Sdim return false; 4343344779Sdim break; 4344344779Sdim 4345344779Sdim // (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, setne)), setu?gt) 4346344779Sdim // (select_cc lhs, rhs, 1, (sext (setcc lhs, rhs, setlt)), setu?gt) 4347344779Sdim // (select_cc lhs, rhs, 1, (sext (setcc rhs, lhs, setgt)), setu?gt) 4348344779Sdim // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, setne)), setu?gt) 4349344779Sdim // (select_cc lhs, rhs, -1, (zext (setcc lhs, rhs, setlt)), setu?gt) 4350344779Sdim // (select_cc lhs, rhs, -1, (zext (setcc rhs, lhs, setgt)), setu?gt) 4351344779Sdim case ISD::SETUGT: 4352344779Sdim if (!IsUnCmp && InnerCC != ISD::SETNE) 4353344779Sdim return false; 4354344779Sdim IsUnCmp = true; 4355344779Sdim LLVM_FALLTHROUGH; 4356344779Sdim case ISD::SETGT: 4357344779Sdim if (InnerCC == ISD::SETNE || (InnerCC == ISD::SETLT && !InnerSwapped) || 4358344779Sdim (InnerCC == ISD::SETGT && InnerSwapped)) 4359344779Sdim NeedSwapOps = (TrueResVal == -1); 4360344779Sdim else 4361344779Sdim return false; 4362344779Sdim break; 4363344779Sdim 4364344779Sdim default: 4365344779Sdim return false; 4366341825Sdim } 4367344779Sdim 4368344779Sdim LLVM_DEBUG(dbgs() << "Found a node that can be lowered to a SETB: "); 4369344779Sdim LLVM_DEBUG(N->dump()); 4370344779Sdim 4371344779Sdim return true; 4372341825Sdim} 4373341825Sdim 4374360784Sdimbool PPCDAGToDAGISel::tryAndWithMask(SDNode *N) { 4375360784Sdim if (N->getOpcode() != ISD::AND) 4376360784Sdim return false; 4377360784Sdim 4378360784Sdim SDLoc dl(N); 4379360784Sdim SDValue Val = N->getOperand(0); 4380360784Sdim unsigned Imm, Imm2, SH, MB, ME; 4381360784Sdim uint64_t Imm64; 4382360784Sdim 4383360784Sdim // If this is an and of a value rotated between 0 and 31 bits and then and'd 4384360784Sdim // with a mask, emit rlwinm 4385360784Sdim if (isInt32Immediate(N->getOperand(1), Imm) && 4386360784Sdim isRotateAndMask(N->getOperand(0).getNode(), Imm, false, SH, MB, ME)) { 4387360784Sdim SDValue Val = N->getOperand(0).getOperand(0); 4388360784Sdim SDValue Ops[] = { Val, getI32Imm(SH, dl), getI32Imm(MB, dl), 4389360784Sdim getI32Imm(ME, dl) }; 4390360784Sdim CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); 4391360784Sdim return true; 4392360784Sdim } 4393360784Sdim 4394360784Sdim // If this is just a masked value where the input is not handled, and 4395360784Sdim // is not a rotate-left (handled by a pattern in the .td file), emit rlwinm 4396360784Sdim if (isInt32Immediate(N->getOperand(1), Imm)) { 4397360784Sdim if (isRunOfOnes(Imm, MB, ME) && 4398360784Sdim N->getOperand(0).getOpcode() != ISD::ROTL) { 4399360784Sdim SDValue Ops[] = { Val, getI32Imm(0, dl), getI32Imm(MB, dl), 4400360784Sdim getI32Imm(ME, dl) }; 4401360784Sdim CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); 4402360784Sdim return true; 4403360784Sdim } 4404360784Sdim // AND X, 0 -> 0, not "rlwinm 32". 4405360784Sdim if (Imm == 0) { 4406360784Sdim ReplaceUses(SDValue(N, 0), N->getOperand(1)); 4407360784Sdim return true; 4408360784Sdim } 4409360784Sdim 4410360784Sdim // ISD::OR doesn't get all the bitfield insertion fun. 4411360784Sdim // (and (or x, c1), c2) where isRunOfOnes(~(c1^c2)) might be a 4412360784Sdim // bitfield insert. 4413360784Sdim if (N->getOperand(0).getOpcode() == ISD::OR && 4414360784Sdim isInt32Immediate(N->getOperand(0).getOperand(1), Imm2)) { 4415360784Sdim // The idea here is to check whether this is equivalent to: 4416360784Sdim // (c1 & m) | (x & ~m) 4417360784Sdim // where m is a run-of-ones mask. The logic here is that, for each bit in 4418360784Sdim // c1 and c2: 4419360784Sdim // - if both are 1, then the output will be 1. 4420360784Sdim // - if both are 0, then the output will be 0. 4421360784Sdim // - if the bit in c1 is 0, and the bit in c2 is 1, then the output will 4422360784Sdim // come from x. 4423360784Sdim // - if the bit in c1 is 1, and the bit in c2 is 0, then the output will 4424360784Sdim // be 0. 4425360784Sdim // If that last condition is never the case, then we can form m from the 4426360784Sdim // bits that are the same between c1 and c2. 4427360784Sdim unsigned MB, ME; 4428360784Sdim if (isRunOfOnes(~(Imm^Imm2), MB, ME) && !(~Imm & Imm2)) { 4429360784Sdim SDValue Ops[] = { N->getOperand(0).getOperand(0), 4430360784Sdim N->getOperand(0).getOperand(1), 4431360784Sdim getI32Imm(0, dl), getI32Imm(MB, dl), 4432360784Sdim getI32Imm(ME, dl) }; 4433360784Sdim ReplaceNode(N, CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops)); 4434360784Sdim return true; 4435360784Sdim } 4436360784Sdim } 4437360784Sdim } else if (isInt64Immediate(N->getOperand(1).getNode(), Imm64)) { 4438360784Sdim // If this is a 64-bit zero-extension mask, emit rldicl. 4439360784Sdim if (isMask_64(Imm64)) { 4440360784Sdim MB = 64 - countTrailingOnes(Imm64); 4441360784Sdim SH = 0; 4442360784Sdim 4443360784Sdim if (Val.getOpcode() == ISD::ANY_EXTEND) { 4444360784Sdim auto Op0 = Val.getOperand(0); 4445360784Sdim if ( Op0.getOpcode() == ISD::SRL && 4446360784Sdim isInt32Immediate(Op0.getOperand(1).getNode(), Imm) && Imm <= MB) { 4447360784Sdim 4448360784Sdim auto ResultType = Val.getNode()->getValueType(0); 4449360784Sdim auto ImDef = CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, 4450360784Sdim ResultType); 4451360784Sdim SDValue IDVal (ImDef, 0); 4452360784Sdim 4453360784Sdim Val = SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl, 4454360784Sdim ResultType, IDVal, Op0.getOperand(0), 4455360784Sdim getI32Imm(1, dl)), 0); 4456360784Sdim SH = 64 - Imm; 4457360784Sdim } 4458360784Sdim } 4459360784Sdim 4460360784Sdim // If the operand is a logical right shift, we can fold it into this 4461360784Sdim // instruction: rldicl(rldicl(x, 64-n, n), 0, mb) -> rldicl(x, 64-n, mb) 4462360784Sdim // for n <= mb. The right shift is really a left rotate followed by a 4463360784Sdim // mask, and this mask is a more-restrictive sub-mask of the mask implied 4464360784Sdim // by the shift. 4465360784Sdim if (Val.getOpcode() == ISD::SRL && 4466360784Sdim isInt32Immediate(Val.getOperand(1).getNode(), Imm) && Imm <= MB) { 4467360784Sdim assert(Imm < 64 && "Illegal shift amount"); 4468360784Sdim Val = Val.getOperand(0); 4469360784Sdim SH = 64 - Imm; 4470360784Sdim } 4471360784Sdim 4472360784Sdim SDValue Ops[] = { Val, getI32Imm(SH, dl), getI32Imm(MB, dl) }; 4473360784Sdim CurDAG->SelectNodeTo(N, PPC::RLDICL, MVT::i64, Ops); 4474360784Sdim return true; 4475360784Sdim } else if (isMask_64(~Imm64)) { 4476360784Sdim // If this is a negated 64-bit zero-extension mask, 4477360784Sdim // i.e. the immediate is a sequence of ones from most significant side 4478360784Sdim // and all zero for reminder, we should use rldicr. 4479360784Sdim MB = 63 - countTrailingOnes(~Imm64); 4480360784Sdim SH = 0; 4481360784Sdim SDValue Ops[] = { Val, getI32Imm(SH, dl), getI32Imm(MB, dl) }; 4482360784Sdim CurDAG->SelectNodeTo(N, PPC::RLDICR, MVT::i64, Ops); 4483360784Sdim return true; 4484360784Sdim } 4485360784Sdim 4486360784Sdim // It is not 16-bit imm that means we need two instructions at least if 4487360784Sdim // using "and" instruction. Try to exploit it with rotate mask instructions. 4488360784Sdim if (isRunOfOnes64(Imm64, MB, ME)) { 4489360784Sdim if (MB >= 32 && MB <= ME) { 4490360784Sdim // MB ME 4491360784Sdim // +----------------------+ 4492360784Sdim // |xxxxxxxxxxx00011111000| 4493360784Sdim // +----------------------+ 4494360784Sdim // 0 32 64 4495360784Sdim // We can only do it if the MB is larger than 32 and MB <= ME 4496360784Sdim // as RLWINM will replace the content of [0 - 32) with [32 - 64) even 4497360784Sdim // we didn't rotate it. 4498360784Sdim SDValue Ops[] = { Val, getI64Imm(0, dl), getI64Imm(MB - 32, dl), 4499360784Sdim getI64Imm(ME - 32, dl) }; 4500360784Sdim CurDAG->SelectNodeTo(N, PPC::RLWINM8, MVT::i64, Ops); 4501360784Sdim return true; 4502360784Sdim } 4503360784Sdim // TODO - handle it with rldicl + rldicl 4504360784Sdim } 4505360784Sdim } 4506360784Sdim 4507360784Sdim return false; 4508360784Sdim} 4509360784Sdim 4510193323Sed// Select - Convert the specified operand from a target-independent to a 4511193323Sed// target-specific node if it hasn't already been changed. 4512309124Sdimvoid PPCDAGToDAGISel::Select(SDNode *N) { 4513261991Sdim SDLoc dl(N); 4514255804Sdim if (N->isMachineOpcode()) { 4515255804Sdim N->setNodeId(-1); 4516309124Sdim return; // Already selected. 4517255804Sdim } 4518193323Sed 4519280031Sdim // In case any misguided DAG-level optimizations form an ADD with a 4520280031Sdim // TargetConstant operand, crash here instead of miscompiling (by selecting 4521280031Sdim // an r+r add instead of some kind of r+i add). 4522280031Sdim if (N->getOpcode() == ISD::ADD && 4523280031Sdim N->getOperand(1).getOpcode() == ISD::TargetConstant) 4524280031Sdim llvm_unreachable("Invalid ADD with TargetConstant operand"); 4525280031Sdim 4526280031Sdim // Try matching complex bit permutations before doing anything else. 4527309124Sdim if (tryBitPermutation(N)) 4528309124Sdim return; 4529280031Sdim 4530327952Sdim // Try to emit integer compares as GPR-only sequences (i.e. no use of CR). 4531327952Sdim if (tryIntCompareInGPR(N)) 4532327952Sdim return; 4533327952Sdim 4534193323Sed switch (N->getOpcode()) { 4535193323Sed default: break; 4536218893Sdim 4537321369Sdim case ISD::Constant: 4538309124Sdim if (N->getValueType(0) == MVT::i64) { 4539327952Sdim ReplaceNode(N, selectI64Imm(CurDAG, N)); 4540309124Sdim return; 4541309124Sdim } 4542193323Sed break; 4543218893Sdim 4544321369Sdim case ISD::SETCC: 4545309124Sdim if (trySETCC(N)) 4546309124Sdim return; 4547276479Sdim break; 4548353358Sdim // These nodes will be transformed into GETtlsADDR32 node, which 4549353358Sdim // later becomes BL_TLS __tls_get_addr(sym at tlsgd)@PLT 4550353358Sdim case PPCISD::ADDI_TLSLD_L_ADDR: 4551353358Sdim case PPCISD::ADDI_TLSGD_L_ADDR: { 4552353358Sdim const Module *Mod = MF->getFunction().getParent(); 4553353358Sdim if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) != MVT::i32 || 4554353358Sdim !PPCSubTarget->isSecurePlt() || !PPCSubTarget->isTargetELF() || 4555353358Sdim Mod->getPICLevel() == PICLevel::SmallPIC) 4556353358Sdim break; 4557353358Sdim // Attach global base pointer on GETtlsADDR32 node in order to 4558353358Sdim // generate secure plt code for TLS symbols. 4559353358Sdim getGlobalBaseReg(); 4560353358Sdim } break; 4561341825Sdim case PPCISD::CALL: { 4562341825Sdim if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) != MVT::i32 || 4563353358Sdim !TM.isPositionIndependent() || !PPCSubTarget->isSecurePlt() || 4564353358Sdim !PPCSubTarget->isTargetELF()) 4565341825Sdim break; 4566341825Sdim 4567341825Sdim SDValue Op = N->getOperand(1); 4568341825Sdim 4569341825Sdim if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op)) { 4570341825Sdim if (GA->getTargetFlags() == PPCII::MO_PLT) 4571341825Sdim getGlobalBaseReg(); 4572341825Sdim } 4573341825Sdim else if (ExternalSymbolSDNode *ES = dyn_cast<ExternalSymbolSDNode>(Op)) { 4574341825Sdim if (ES->getTargetFlags() == PPCII::MO_PLT) 4575341825Sdim getGlobalBaseReg(); 4576341825Sdim } 4577341825Sdim } 4578341825Sdim break; 4579341825Sdim 4580193323Sed case PPCISD::GlobalBaseReg: 4581309124Sdim ReplaceNode(N, getGlobalBaseReg()); 4582309124Sdim return; 4583218893Sdim 4584280031Sdim case ISD::FrameIndex: 4585309124Sdim selectFrameIndex(N, N); 4586309124Sdim return; 4587193323Sed 4588261991Sdim case PPCISD::MFOCRF: { 4589193323Sed SDValue InFlag = N->getOperand(1); 4590309124Sdim ReplaceNode(N, CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, 4591309124Sdim N->getOperand(0), InFlag)); 4592309124Sdim return; 4593193323Sed } 4594218893Sdim 4595321369Sdim case PPCISD::READ_TIME_BASE: 4596309124Sdim ReplaceNode(N, CurDAG->getMachineNode(PPC::ReadTB, dl, MVT::i32, MVT::i32, 4597309124Sdim MVT::Other, N->getOperand(0))); 4598309124Sdim return; 4599280031Sdim 4600280031Sdim case PPCISD::SRA_ADDZE: { 4601280031Sdim SDValue N0 = N->getOperand(0); 4602280031Sdim SDValue ShiftAmt = 4603280031Sdim CurDAG->getTargetConstant(*cast<ConstantSDNode>(N->getOperand(1))-> 4604288943Sdim getConstantIntValue(), dl, 4605288943Sdim N->getValueType(0)); 4606280031Sdim if (N->getValueType(0) == MVT::i64) { 4607280031Sdim SDNode *Op = 4608280031Sdim CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, MVT::Glue, 4609280031Sdim N0, ShiftAmt); 4610309124Sdim CurDAG->SelectNodeTo(N, PPC::ADDZE8, MVT::i64, SDValue(Op, 0), 4611309124Sdim SDValue(Op, 1)); 4612309124Sdim return; 4613280031Sdim } else { 4614280031Sdim assert(N->getValueType(0) == MVT::i32 && 4615280031Sdim "Expecting i64 or i32 in PPCISD::SRA_ADDZE"); 4616280031Sdim SDNode *Op = 4617280031Sdim CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, MVT::Glue, 4618280031Sdim N0, ShiftAmt); 4619309124Sdim CurDAG->SelectNodeTo(N, PPC::ADDZE, MVT::i32, SDValue(Op, 0), 4620309124Sdim SDValue(Op, 1)); 4621309124Sdim return; 4622193323Sed } 4623193323Sed } 4624218893Sdim 4625341825Sdim case ISD::STORE: { 4626341825Sdim // Change TLS initial-exec D-form stores to X-form stores. 4627341825Sdim StoreSDNode *ST = cast<StoreSDNode>(N); 4628341825Sdim if (EnableTLSOpt && PPCSubTarget->isELFv2ABI() && 4629341825Sdim ST->getAddressingMode() != ISD::PRE_INC) 4630341825Sdim if (tryTLSXFormStore(ST)) 4631341825Sdim return; 4632341825Sdim break; 4633341825Sdim } 4634193323Sed case ISD::LOAD: { 4635193323Sed // Handle preincrement loads. 4636202375Srdivacky LoadSDNode *LD = cast<LoadSDNode>(N); 4637198090Srdivacky EVT LoadedVT = LD->getMemoryVT(); 4638218893Sdim 4639193323Sed // Normal loads are handled by code generated from the .td file. 4640341825Sdim if (LD->getAddressingMode() != ISD::PRE_INC) { 4641341825Sdim // Change TLS initial-exec D-form loads to X-form loads. 4642341825Sdim if (EnableTLSOpt && PPCSubTarget->isELFv2ABI()) 4643341825Sdim if (tryTLSXFormLoad(LD)) 4644341825Sdim return; 4645193323Sed break; 4646341825Sdim } 4647218893Sdim 4648193323Sed SDValue Offset = LD->getOffset(); 4649249423Sdim if (Offset.getOpcode() == ISD::TargetConstant || 4650193323Sed Offset.getOpcode() == ISD::TargetGlobalAddress) { 4651218893Sdim 4652193323Sed unsigned Opcode; 4653193323Sed bool isSExt = LD->getExtensionType() == ISD::SEXTLOAD; 4654193323Sed if (LD->getValueType(0) != MVT::i64) { 4655193323Sed // Handle PPC32 integer and normal FP loads. 4656193323Sed assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load"); 4657198090Srdivacky switch (LoadedVT.getSimpleVT().SimpleTy) { 4658198090Srdivacky default: llvm_unreachable("Invalid PPC load type!"); 4659193323Sed case MVT::f64: Opcode = PPC::LFDU; break; 4660193323Sed case MVT::f32: Opcode = PPC::LFSU; break; 4661193323Sed case MVT::i32: Opcode = PPC::LWZU; break; 4662193323Sed case MVT::i16: Opcode = isSExt ? PPC::LHAU : PPC::LHZU; break; 4663193323Sed case MVT::i1: 4664193323Sed case MVT::i8: Opcode = PPC::LBZU; break; 4665193323Sed } 4666193323Sed } else { 4667193323Sed assert(LD->getValueType(0) == MVT::i64 && "Unknown load result type!"); 4668193323Sed assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load"); 4669198090Srdivacky switch (LoadedVT.getSimpleVT().SimpleTy) { 4670198090Srdivacky default: llvm_unreachable("Invalid PPC load type!"); 4671193323Sed case MVT::i64: Opcode = PPC::LDU; break; 4672193323Sed case MVT::i32: Opcode = PPC::LWZU8; break; 4673193323Sed case MVT::i16: Opcode = isSExt ? PPC::LHAU8 : PPC::LHZU8; break; 4674193323Sed case MVT::i1: 4675193323Sed case MVT::i8: Opcode = PPC::LBZU8; break; 4676193323Sed } 4677193323Sed } 4678218893Sdim 4679193323Sed SDValue Chain = LD->getChain(); 4680193323Sed SDValue Base = LD->getBasePtr(); 4681193323Sed SDValue Ops[] = { Offset, Base, Chain }; 4682309124Sdim SDNode *MN = CurDAG->getMachineNode( 4683309124Sdim Opcode, dl, LD->getValueType(0), 4684309124Sdim PPCLowering->getPointerTy(CurDAG->getDataLayout()), MVT::Other, Ops); 4685309124Sdim transferMemOperands(N, MN); 4686309124Sdim ReplaceNode(N, MN); 4687309124Sdim return; 4688193323Sed } else { 4689239462Sdim unsigned Opcode; 4690239462Sdim bool isSExt = LD->getExtensionType() == ISD::SEXTLOAD; 4691239462Sdim if (LD->getValueType(0) != MVT::i64) { 4692239462Sdim // Handle PPC32 integer and normal FP loads. 4693239462Sdim assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load"); 4694239462Sdim switch (LoadedVT.getSimpleVT().SimpleTy) { 4695239462Sdim default: llvm_unreachable("Invalid PPC load type!"); 4696288943Sdim case MVT::v4f64: Opcode = PPC::QVLFDUX; break; // QPX 4697288943Sdim case MVT::v4f32: Opcode = PPC::QVLFSUX; break; // QPX 4698239462Sdim case MVT::f64: Opcode = PPC::LFDUX; break; 4699239462Sdim case MVT::f32: Opcode = PPC::LFSUX; break; 4700239462Sdim case MVT::i32: Opcode = PPC::LWZUX; break; 4701239462Sdim case MVT::i16: Opcode = isSExt ? PPC::LHAUX : PPC::LHZUX; break; 4702239462Sdim case MVT::i1: 4703239462Sdim case MVT::i8: Opcode = PPC::LBZUX; break; 4704239462Sdim } 4705239462Sdim } else { 4706239462Sdim assert(LD->getValueType(0) == MVT::i64 && "Unknown load result type!"); 4707239462Sdim assert((!isSExt || LoadedVT == MVT::i16 || LoadedVT == MVT::i32) && 4708239462Sdim "Invalid sext update load"); 4709239462Sdim switch (LoadedVT.getSimpleVT().SimpleTy) { 4710239462Sdim default: llvm_unreachable("Invalid PPC load type!"); 4711239462Sdim case MVT::i64: Opcode = PPC::LDUX; break; 4712239462Sdim case MVT::i32: Opcode = isSExt ? PPC::LWAUX : PPC::LWZUX8; break; 4713239462Sdim case MVT::i16: Opcode = isSExt ? PPC::LHAUX8 : PPC::LHZUX8; break; 4714239462Sdim case MVT::i1: 4715239462Sdim case MVT::i8: Opcode = PPC::LBZUX8; break; 4716239462Sdim } 4717239462Sdim } 4718239462Sdim 4719239462Sdim SDValue Chain = LD->getChain(); 4720239462Sdim SDValue Base = LD->getBasePtr(); 4721249423Sdim SDValue Ops[] = { Base, Offset, Chain }; 4722309124Sdim SDNode *MN = CurDAG->getMachineNode( 4723309124Sdim Opcode, dl, LD->getValueType(0), 4724309124Sdim PPCLowering->getPointerTy(CurDAG->getDataLayout()), MVT::Other, Ops); 4725309124Sdim transferMemOperands(N, MN); 4726309124Sdim ReplaceNode(N, MN); 4727309124Sdim return; 4728193323Sed } 4729193323Sed } 4730218893Sdim 4731360784Sdim case ISD::AND: 4732360784Sdim // If this is an 'and' with a mask, try to emit rlwinm/rldicl/rldicr 4733360784Sdim if (tryAndWithMask(N)) 4734309124Sdim return; 4735276479Sdim 4736193323Sed // Other cases are autogenerated. 4737193323Sed break; 4738280031Sdim case ISD::OR: { 4739193323Sed if (N->getValueType(0) == MVT::i32) 4740309124Sdim if (tryBitfieldInsert(N)) 4741309124Sdim return; 4742218893Sdim 4743321369Sdim int16_t Imm; 4744280031Sdim if (N->getOperand(0)->getOpcode() == ISD::FrameIndex && 4745280031Sdim isIntS16Immediate(N->getOperand(1), Imm)) { 4746344779Sdim KnownBits LHSKnown = CurDAG->computeKnownBits(N->getOperand(0)); 4747280031Sdim 4748280031Sdim // If this is equivalent to an add, then we can fold it with the 4749280031Sdim // FrameIndex calculation. 4750321369Sdim if ((LHSKnown.Zero.getZExtValue()|~(uint64_t)Imm) == ~0ULL) { 4751309124Sdim selectFrameIndex(N, N->getOperand(0).getNode(), (int)Imm); 4752309124Sdim return; 4753309124Sdim } 4754280031Sdim } 4755280031Sdim 4756327952Sdim // OR with a 32-bit immediate can be handled by ori + oris 4757327952Sdim // without creating an immediate in a GPR. 4758327952Sdim uint64_t Imm64 = 0; 4759327952Sdim bool IsPPC64 = PPCSubTarget->isPPC64(); 4760327952Sdim if (IsPPC64 && isInt64Immediate(N->getOperand(1), Imm64) && 4761327952Sdim (Imm64 & ~0xFFFFFFFFuLL) == 0) { 4762327952Sdim // If ImmHi (ImmHi) is zero, only one ori (oris) is generated later. 4763327952Sdim uint64_t ImmHi = Imm64 >> 16; 4764327952Sdim uint64_t ImmLo = Imm64 & 0xFFFF; 4765327952Sdim if (ImmHi != 0 && ImmLo != 0) { 4766327952Sdim SDNode *Lo = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, 4767327952Sdim N->getOperand(0), 4768327952Sdim getI16Imm(ImmLo, dl)); 4769327952Sdim SDValue Ops1[] = { SDValue(Lo, 0), getI16Imm(ImmHi, dl)}; 4770327952Sdim CurDAG->SelectNodeTo(N, PPC::ORIS8, MVT::i64, Ops1); 4771327952Sdim return; 4772327952Sdim } 4773327952Sdim } 4774327952Sdim 4775193323Sed // Other cases are autogenerated. 4776193323Sed break; 4777280031Sdim } 4778321369Sdim case ISD::XOR: { 4779327952Sdim // XOR with a 32-bit immediate can be handled by xori + xoris 4780327952Sdim // without creating an immediate in a GPR. 4781327952Sdim uint64_t Imm64 = 0; 4782327952Sdim bool IsPPC64 = PPCSubTarget->isPPC64(); 4783327952Sdim if (IsPPC64 && isInt64Immediate(N->getOperand(1), Imm64) && 4784327952Sdim (Imm64 & ~0xFFFFFFFFuLL) == 0) { 4785327952Sdim // If ImmHi (ImmHi) is zero, only one xori (xoris) is generated later. 4786327952Sdim uint64_t ImmHi = Imm64 >> 16; 4787327952Sdim uint64_t ImmLo = Imm64 & 0xFFFF; 4788327952Sdim if (ImmHi != 0 && ImmLo != 0) { 4789327952Sdim SDNode *Lo = CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, 4790327952Sdim N->getOperand(0), 4791327952Sdim getI16Imm(ImmLo, dl)); 4792327952Sdim SDValue Ops1[] = { SDValue(Lo, 0), getI16Imm(ImmHi, dl)}; 4793327952Sdim CurDAG->SelectNodeTo(N, PPC::XORIS8, MVT::i64, Ops1); 4794327952Sdim return; 4795327952Sdim } 4796327952Sdim } 4797327952Sdim 4798321369Sdim break; 4799321369Sdim } 4800280031Sdim case ISD::ADD: { 4801321369Sdim int16_t Imm; 4802280031Sdim if (N->getOperand(0)->getOpcode() == ISD::FrameIndex && 4803309124Sdim isIntS16Immediate(N->getOperand(1), Imm)) { 4804309124Sdim selectFrameIndex(N, N->getOperand(0).getNode(), (int)Imm); 4805309124Sdim return; 4806309124Sdim } 4807280031Sdim 4808280031Sdim break; 4809280031Sdim } 4810193323Sed case ISD::SHL: { 4811193323Sed unsigned Imm, SH, MB, ME; 4812193323Sed if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, Imm) && 4813193323Sed isRotateAndMask(N, Imm, true, SH, MB, ME)) { 4814193323Sed SDValue Ops[] = { N->getOperand(0).getOperand(0), 4815288943Sdim getI32Imm(SH, dl), getI32Imm(MB, dl), 4816288943Sdim getI32Imm(ME, dl) }; 4817309124Sdim CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); 4818309124Sdim return; 4819193323Sed } 4820218893Sdim 4821193323Sed // Other cases are autogenerated. 4822193323Sed break; 4823193323Sed } 4824193323Sed case ISD::SRL: { 4825193323Sed unsigned Imm, SH, MB, ME; 4826193323Sed if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, Imm) && 4827218893Sdim isRotateAndMask(N, Imm, true, SH, MB, ME)) { 4828193323Sed SDValue Ops[] = { N->getOperand(0).getOperand(0), 4829288943Sdim getI32Imm(SH, dl), getI32Imm(MB, dl), 4830288943Sdim getI32Imm(ME, dl) }; 4831309124Sdim CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); 4832309124Sdim return; 4833193323Sed } 4834218893Sdim 4835193323Sed // Other cases are autogenerated. 4836193323Sed break; 4837193323Sed } 4838276479Sdim // FIXME: Remove this once the ANDI glue bug is fixed: 4839360784Sdim case PPCISD::ANDI_rec_1_EQ_BIT: 4840360784Sdim case PPCISD::ANDI_rec_1_GT_BIT: { 4841276479Sdim if (!ANDIGlueBug) 4842276479Sdim break; 4843276479Sdim 4844276479Sdim EVT InVT = N->getOperand(0).getValueType(); 4845276479Sdim assert((InVT == MVT::i64 || InVT == MVT::i32) && 4846360784Sdim "Invalid input type for ANDI_rec_1_EQ_BIT"); 4847276479Sdim 4848360784Sdim unsigned Opcode = (InVT == MVT::i64) ? PPC::ANDI8_rec : PPC::ANDI_rec; 4849276479Sdim SDValue AndI(CurDAG->getMachineNode(Opcode, dl, InVT, MVT::Glue, 4850276479Sdim N->getOperand(0), 4851288943Sdim CurDAG->getTargetConstant(1, dl, InVT)), 4852288943Sdim 0); 4853276479Sdim SDValue CR0Reg = CurDAG->getRegister(PPC::CR0, MVT::i32); 4854360784Sdim SDValue SRIdxVal = CurDAG->getTargetConstant( 4855360784Sdim N->getOpcode() == PPCISD::ANDI_rec_1_EQ_BIT ? PPC::sub_eq : PPC::sub_gt, 4856360784Sdim dl, MVT::i32); 4857276479Sdim 4858309124Sdim CurDAG->SelectNodeTo(N, TargetOpcode::EXTRACT_SUBREG, MVT::i1, CR0Reg, 4859309124Sdim SRIdxVal, SDValue(AndI.getNode(), 1) /* glue */); 4860309124Sdim return; 4861276479Sdim } 4862193323Sed case ISD::SELECT_CC: { 4863193323Sed ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(4))->get(); 4864288943Sdim EVT PtrVT = 4865288943Sdim CurDAG->getTargetLoweringInfo().getPointerTy(CurDAG->getDataLayout()); 4866224145Sdim bool isPPC64 = (PtrVT == MVT::i64); 4867218893Sdim 4868276479Sdim // If this is a select of i1 operands, we'll pattern match it. 4869276479Sdim if (PPCSubTarget->useCRBits() && 4870276479Sdim N->getOperand(0).getValueType() == MVT::i1) 4871276479Sdim break; 4872276479Sdim 4873344779Sdim if (PPCSubTarget->isISA3_0() && PPCSubTarget->isPPC64()) { 4874344779Sdim bool NeedSwapOps = false; 4875344779Sdim bool IsUnCmp = false; 4876344779Sdim if (mayUseP9Setb(N, CC, CurDAG, NeedSwapOps, IsUnCmp)) { 4877344779Sdim SDValue LHS = N->getOperand(0); 4878344779Sdim SDValue RHS = N->getOperand(1); 4879344779Sdim if (NeedSwapOps) 4880344779Sdim std::swap(LHS, RHS); 4881344779Sdim 4882344779Sdim // Make use of SelectCC to generate the comparison to set CR bits, for 4883344779Sdim // equality comparisons having one literal operand, SelectCC probably 4884344779Sdim // doesn't need to materialize the whole literal and just use xoris to 4885344779Sdim // check it first, it leads the following comparison result can't 4886344779Sdim // exactly represent GT/LT relationship. So to avoid this we specify 4887344779Sdim // SETGT/SETUGT here instead of SETEQ. 4888344779Sdim SDValue GenCC = 4889344779Sdim SelectCC(LHS, RHS, IsUnCmp ? ISD::SETUGT : ISD::SETGT, dl); 4890344779Sdim CurDAG->SelectNodeTo( 4891344779Sdim N, N->getSimpleValueType(0) == MVT::i64 ? PPC::SETB8 : PPC::SETB, 4892344779Sdim N->getValueType(0), GenCC); 4893344779Sdim NumP9Setb++; 4894344779Sdim return; 4895344779Sdim } 4896344779Sdim } 4897344779Sdim 4898193323Sed // Handle the setcc cases here. select_cc lhs, 0, 1, 0, cc 4899224145Sdim if (!isPPC64) 4900224145Sdim if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1))) 4901224145Sdim if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N->getOperand(2))) 4902224145Sdim if (ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N->getOperand(3))) 4903224145Sdim if (N1C->isNullValue() && N3C->isNullValue() && 4904224145Sdim N2C->getZExtValue() == 1ULL && CC == ISD::SETNE && 4905224145Sdim // FIXME: Implement this optzn for PPC64. 4906224145Sdim N->getValueType(0) == MVT::i32) { 4907224145Sdim SDNode *Tmp = 4908224145Sdim CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, 4909288943Sdim N->getOperand(0), getI32Imm(~0U, dl)); 4910309124Sdim CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, SDValue(Tmp, 0), 4911309124Sdim N->getOperand(0), SDValue(Tmp, 1)); 4912309124Sdim return; 4913224145Sdim } 4914193323Sed 4915193323Sed SDValue CCReg = SelectCC(N->getOperand(0), N->getOperand(1), CC, dl); 4916276479Sdim 4917276479Sdim if (N->getValueType(0) == MVT::i1) { 4918276479Sdim // An i1 select is: (c & t) | (!c & f). 4919276479Sdim bool Inv; 4920276479Sdim unsigned Idx = getCRIdxForSetCC(CC, Inv); 4921276479Sdim 4922276479Sdim unsigned SRI; 4923276479Sdim switch (Idx) { 4924276479Sdim default: llvm_unreachable("Invalid CC index"); 4925276479Sdim case 0: SRI = PPC::sub_lt; break; 4926276479Sdim case 1: SRI = PPC::sub_gt; break; 4927276479Sdim case 2: SRI = PPC::sub_eq; break; 4928276479Sdim case 3: SRI = PPC::sub_un; break; 4929276479Sdim } 4930276479Sdim 4931276479Sdim SDValue CCBit = CurDAG->getTargetExtractSubreg(SRI, dl, MVT::i1, CCReg); 4932276479Sdim 4933276479Sdim SDValue NotCCBit(CurDAG->getMachineNode(PPC::CRNOR, dl, MVT::i1, 4934276479Sdim CCBit, CCBit), 0); 4935276479Sdim SDValue C = Inv ? NotCCBit : CCBit, 4936276479Sdim NotC = Inv ? CCBit : NotCCBit; 4937276479Sdim 4938276479Sdim SDValue CAndT(CurDAG->getMachineNode(PPC::CRAND, dl, MVT::i1, 4939276479Sdim C, N->getOperand(2)), 0); 4940276479Sdim SDValue NotCAndF(CurDAG->getMachineNode(PPC::CRAND, dl, MVT::i1, 4941276479Sdim NotC, N->getOperand(3)), 0); 4942276479Sdim 4943309124Sdim CurDAG->SelectNodeTo(N, PPC::CROR, MVT::i1, CAndT, NotCAndF); 4944309124Sdim return; 4945276479Sdim } 4946276479Sdim 4947360784Sdim unsigned BROpc = 4948360784Sdim getPredicateForSetCC(CC, N->getOperand(0).getValueType(), PPCSubTarget); 4949193323Sed 4950193323Sed unsigned SelectCCOp; 4951193323Sed if (N->getValueType(0) == MVT::i32) 4952193323Sed SelectCCOp = PPC::SELECT_CC_I4; 4953193323Sed else if (N->getValueType(0) == MVT::i64) 4954193323Sed SelectCCOp = PPC::SELECT_CC_I8; 4955341825Sdim else if (N->getValueType(0) == MVT::f32) { 4956288943Sdim if (PPCSubTarget->hasP8Vector()) 4957288943Sdim SelectCCOp = PPC::SELECT_CC_VSSRC; 4958341825Sdim else if (PPCSubTarget->hasSPE()) 4959341825Sdim SelectCCOp = PPC::SELECT_CC_SPE4; 4960288943Sdim else 4961288943Sdim SelectCCOp = PPC::SELECT_CC_F4; 4962341825Sdim } else if (N->getValueType(0) == MVT::f64) { 4963280031Sdim if (PPCSubTarget->hasVSX()) 4964280031Sdim SelectCCOp = PPC::SELECT_CC_VSFRC; 4965341825Sdim else if (PPCSubTarget->hasSPE()) 4966341825Sdim SelectCCOp = PPC::SELECT_CC_SPE; 4967280031Sdim else 4968280031Sdim SelectCCOp = PPC::SELECT_CC_F8; 4969341825Sdim } else if (N->getValueType(0) == MVT::f128) 4970341825Sdim SelectCCOp = PPC::SELECT_CC_F16; 4971341825Sdim else if (PPCSubTarget->hasSPE()) 4972341825Sdim SelectCCOp = PPC::SELECT_CC_SPE; 4973288943Sdim else if (PPCSubTarget->hasQPX() && N->getValueType(0) == MVT::v4f64) 4974288943Sdim SelectCCOp = PPC::SELECT_CC_QFRC; 4975288943Sdim else if (PPCSubTarget->hasQPX() && N->getValueType(0) == MVT::v4f32) 4976288943Sdim SelectCCOp = PPC::SELECT_CC_QSRC; 4977288943Sdim else if (PPCSubTarget->hasQPX() && N->getValueType(0) == MVT::v4i1) 4978288943Sdim SelectCCOp = PPC::SELECT_CC_QBRC; 4979280031Sdim else if (N->getValueType(0) == MVT::v2f64 || 4980280031Sdim N->getValueType(0) == MVT::v2i64) 4981280031Sdim SelectCCOp = PPC::SELECT_CC_VSRC; 4982193323Sed else 4983193323Sed SelectCCOp = PPC::SELECT_CC_VRRC; 4984193323Sed 4985193323Sed SDValue Ops[] = { CCReg, N->getOperand(2), N->getOperand(3), 4986288943Sdim getI32Imm(BROpc, dl) }; 4987309124Sdim CurDAG->SelectNodeTo(N, SelectCCOp, N->getValueType(0), Ops); 4988309124Sdim return; 4989193323Sed } 4990276479Sdim case ISD::VECTOR_SHUFFLE: 4991276479Sdim if (PPCSubTarget->hasVSX() && (N->getValueType(0) == MVT::v2f64 || 4992276479Sdim N->getValueType(0) == MVT::v2i64)) { 4993276479Sdim ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N); 4994296417Sdim 4995276479Sdim SDValue Op1 = N->getOperand(SVN->getMaskElt(0) < 2 ? 0 : 1), 4996276479Sdim Op2 = N->getOperand(SVN->getMaskElt(1) < 2 ? 0 : 1); 4997276479Sdim unsigned DM[2]; 4998276479Sdim 4999276479Sdim for (int i = 0; i < 2; ++i) 5000276479Sdim if (SVN->getMaskElt(i) <= 0 || SVN->getMaskElt(i) == 2) 5001276479Sdim DM[i] = 0; 5002276479Sdim else 5003276479Sdim DM[i] = 1; 5004276479Sdim 5005276479Sdim if (Op1 == Op2 && DM[0] == 0 && DM[1] == 0 && 5006276479Sdim Op1.getOpcode() == ISD::SCALAR_TO_VECTOR && 5007276479Sdim isa<LoadSDNode>(Op1.getOperand(0))) { 5008276479Sdim LoadSDNode *LD = cast<LoadSDNode>(Op1.getOperand(0)); 5009276479Sdim SDValue Base, Offset; 5010276479Sdim 5011296417Sdim if (LD->isUnindexed() && LD->hasOneUse() && Op1.hasOneUse() && 5012292735Sdim (LD->getMemoryVT() == MVT::f64 || 5013292735Sdim LD->getMemoryVT() == MVT::i64) && 5014276479Sdim SelectAddrIdxOnly(LD->getBasePtr(), Base, Offset)) { 5015276479Sdim SDValue Chain = LD->getChain(); 5016276479Sdim SDValue Ops[] = { Base, Offset, Chain }; 5017344779Sdim MachineMemOperand *MemOp = LD->getMemOperand(); 5018321369Sdim SDNode *NewN = CurDAG->SelectNodeTo(N, PPC::LXVDSX, 5019321369Sdim N->getValueType(0), Ops); 5020344779Sdim CurDAG->setNodeMemRefs(cast<MachineSDNode>(NewN), {MemOp}); 5021309124Sdim return; 5022276479Sdim } 5023276479Sdim } 5024276479Sdim 5025288943Sdim // For little endian, we must swap the input operands and adjust 5026288943Sdim // the mask elements (reverse and invert them). 5027288943Sdim if (PPCSubTarget->isLittleEndian()) { 5028288943Sdim std::swap(Op1, Op2); 5029288943Sdim unsigned tmp = DM[0]; 5030288943Sdim DM[0] = 1 - DM[1]; 5031288943Sdim DM[1] = 1 - tmp; 5032288943Sdim } 5033288943Sdim 5034288943Sdim SDValue DMV = CurDAG->getTargetConstant(DM[1] | (DM[0] << 1), dl, 5035288943Sdim MVT::i32); 5036276479Sdim SDValue Ops[] = { Op1, Op2, DMV }; 5037309124Sdim CurDAG->SelectNodeTo(N, PPC::XXPERMDI, N->getValueType(0), Ops); 5038309124Sdim return; 5039276479Sdim } 5040276479Sdim 5041276479Sdim break; 5042261991Sdim case PPCISD::BDNZ: 5043261991Sdim case PPCISD::BDZ: { 5044276479Sdim bool IsPPC64 = PPCSubTarget->isPPC64(); 5045261991Sdim SDValue Ops[] = { N->getOperand(1), N->getOperand(0) }; 5046309124Sdim CurDAG->SelectNodeTo(N, N->getOpcode() == PPCISD::BDNZ 5047309124Sdim ? (IsPPC64 ? PPC::BDNZ8 : PPC::BDNZ) 5048309124Sdim : (IsPPC64 ? PPC::BDZ8 : PPC::BDZ), 5049309124Sdim MVT::Other, Ops); 5050309124Sdim return; 5051261991Sdim } 5052193323Sed case PPCISD::COND_BRANCH: { 5053193323Sed // Op #0 is the Chain. 5054193323Sed // Op #1 is the PPC::PRED_* number. 5055193323Sed // Op #2 is the CR# 5056193323Sed // Op #3 is the Dest MBB 5057193323Sed // Op #4 is the Flag. 5058193323Sed // Prevent PPC::PRED_* from being selected into LI. 5059296417Sdim unsigned PCC = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 5060296417Sdim if (EnableBranchHint) 5061360784Sdim PCC |= getBranchHint(PCC, *FuncInfo, N->getOperand(3)); 5062296417Sdim 5063296417Sdim SDValue Pred = getI32Imm(PCC, dl); 5064193323Sed SDValue Ops[] = { Pred, N->getOperand(2), N->getOperand(3), 5065193323Sed N->getOperand(0), N->getOperand(4) }; 5066309124Sdim CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops); 5067309124Sdim return; 5068193323Sed } 5069193323Sed case ISD::BR_CC: { 5070193323Sed ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(1))->get(); 5071360784Sdim unsigned PCC = 5072360784Sdim getPredicateForSetCC(CC, N->getOperand(2).getValueType(), PPCSubTarget); 5073276479Sdim 5074276479Sdim if (N->getOperand(2).getValueType() == MVT::i1) { 5075276479Sdim unsigned Opc; 5076276479Sdim bool Swap; 5077276479Sdim switch (PCC) { 5078276479Sdim default: llvm_unreachable("Unexpected Boolean-operand predicate"); 5079276479Sdim case PPC::PRED_LT: Opc = PPC::CRANDC; Swap = true; break; 5080276479Sdim case PPC::PRED_LE: Opc = PPC::CRORC; Swap = true; break; 5081276479Sdim case PPC::PRED_EQ: Opc = PPC::CREQV; Swap = false; break; 5082276479Sdim case PPC::PRED_GE: Opc = PPC::CRORC; Swap = false; break; 5083276479Sdim case PPC::PRED_GT: Opc = PPC::CRANDC; Swap = false; break; 5084276479Sdim case PPC::PRED_NE: Opc = PPC::CRXOR; Swap = false; break; 5085276479Sdim } 5086276479Sdim 5087344779Sdim // A signed comparison of i1 values produces the opposite result to an 5088344779Sdim // unsigned one if the condition code includes less-than or greater-than. 5089344779Sdim // This is because 1 is the most negative signed i1 number and the most 5090344779Sdim // positive unsigned i1 number. The CR-logical operations used for such 5091344779Sdim // comparisons are non-commutative so for signed comparisons vs. unsigned 5092344779Sdim // ones, the input operands just need to be swapped. 5093344779Sdim if (ISD::isSignedIntSetCC(CC)) 5094344779Sdim Swap = !Swap; 5095344779Sdim 5096276479Sdim SDValue BitComp(CurDAG->getMachineNode(Opc, dl, MVT::i1, 5097276479Sdim N->getOperand(Swap ? 3 : 2), 5098276479Sdim N->getOperand(Swap ? 2 : 3)), 0); 5099309124Sdim CurDAG->SelectNodeTo(N, PPC::BC, MVT::Other, BitComp, N->getOperand(4), 5100309124Sdim N->getOperand(0)); 5101309124Sdim return; 5102276479Sdim } 5103276479Sdim 5104296417Sdim if (EnableBranchHint) 5105360784Sdim PCC |= getBranchHint(PCC, *FuncInfo, N->getOperand(4)); 5106296417Sdim 5107193323Sed SDValue CondCode = SelectCC(N->getOperand(2), N->getOperand(3), CC, dl); 5108288943Sdim SDValue Ops[] = { getI32Imm(PCC, dl), CondCode, 5109193323Sed N->getOperand(4), N->getOperand(0) }; 5110309124Sdim CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops); 5111309124Sdim return; 5112193323Sed } 5113193323Sed case ISD::BRIND: { 5114193323Sed // FIXME: Should custom lower this. 5115193323Sed SDValue Chain = N->getOperand(0); 5116193323Sed SDValue Target = N->getOperand(1); 5117193323Sed unsigned Opc = Target.getValueType() == MVT::i32 ? PPC::MTCTR : PPC::MTCTR8; 5118223017Sdim unsigned Reg = Target.getValueType() == MVT::i32 ? PPC::BCTR : PPC::BCTR8; 5119234353Sdim Chain = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, Target, 5120198090Srdivacky Chain), 0); 5121309124Sdim CurDAG->SelectNodeTo(N, Reg, MVT::Other, Chain); 5122309124Sdim return; 5123193323Sed } 5124249423Sdim case PPCISD::TOC_ENTRY: { 5125360784Sdim const bool isPPC64 = PPCSubTarget->isPPC64(); 5126360784Sdim const bool isELFABI = PPCSubTarget->isSVR4ABI(); 5127360784Sdim const bool isAIXABI = PPCSubTarget->isAIXABI(); 5128249423Sdim 5129360784Sdim assert(!PPCSubTarget->isDarwin() && "TOC is an ELF/XCOFF construct"); 5130360784Sdim 5131360784Sdim // PowerPC only support small, medium and large code model. 5132360784Sdim const CodeModel::Model CModel = TM.getCodeModel(); 5133360784Sdim assert(!(CModel == CodeModel::Tiny || CModel == CodeModel::Kernel) && 5134360784Sdim "PowerPC doesn't support tiny or kernel code models."); 5135360784Sdim 5136360784Sdim if (isAIXABI && CModel == CodeModel::Medium) 5137360784Sdim report_fatal_error("Medium code model is not supported on AIX."); 5138360784Sdim 5139360784Sdim // For 64-bit small code model, we allow SelectCodeCommon to handle this, 5140277320Sdim // selecting one of LDtoc, LDtocJTI, LDtocCPT, and LDtocBA. 5141360784Sdim if (isPPC64 && CModel == CodeModel::Small) 5142249423Sdim break; 5143249423Sdim 5144360784Sdim // Handle 32-bit small code model. 5145360784Sdim if (!isPPC64) { 5146360784Sdim // Transforms the ISD::TOC_ENTRY node to a PPCISD::LWZtoc. 5147360784Sdim auto replaceWithLWZtoc = [this, &dl](SDNode *TocEntry) { 5148360784Sdim SDValue GA = TocEntry->getOperand(0); 5149360784Sdim SDValue TocBase = TocEntry->getOperand(1); 5150360784Sdim SDNode *MN = CurDAG->getMachineNode(PPC::LWZtoc, dl, MVT::i32, GA, 5151360784Sdim TocBase); 5152360784Sdim transferMemOperands(TocEntry, MN); 5153360784Sdim ReplaceNode(TocEntry, MN); 5154360784Sdim }; 5155360784Sdim 5156360784Sdim if (isELFABI) { 5157360784Sdim assert(TM.isPositionIndependent() && 5158360784Sdim "32-bit ELF can only have TOC entries in position independent" 5159360784Sdim " code."); 5160360784Sdim // 32-bit ELF always uses a small code model toc access. 5161360784Sdim replaceWithLWZtoc(N); 5162360784Sdim return; 5163360784Sdim } 5164360784Sdim 5165360784Sdim if (isAIXABI && CModel == CodeModel::Small) { 5166360784Sdim replaceWithLWZtoc(N); 5167360784Sdim return; 5168360784Sdim } 5169360784Sdim } 5170360784Sdim 5171360784Sdim assert(CModel != CodeModel::Small && "All small code models handled."); 5172360784Sdim 5173360784Sdim assert((isPPC64 || (isAIXABI && !isPPC64)) && "We are dealing with 64-bit" 5174360784Sdim " ELF/AIX or 32-bit AIX in the following."); 5175360784Sdim 5176360784Sdim // Transforms the ISD::TOC_ENTRY node for 32-bit AIX large code model mode 5177360784Sdim // or 64-bit medium (ELF-only) or large (ELF and AIX) code model code. We 5178360784Sdim // generate two instructions as described below. The first source operand 5179360784Sdim // is a symbol reference. If it must be toc-referenced according to 5180360784Sdim // PPCSubTarget, we generate: 5181360784Sdim // [32-bit AIX] 5182360784Sdim // LWZtocL(@sym, ADDIStocHA(%r2, @sym)) 5183360784Sdim // [64-bit ELF/AIX] 5184360784Sdim // LDtocL(@sym, ADDIStocHA8(%x2, @sym)) 5185249423Sdim // Otherwise we generate: 5186360784Sdim // ADDItocL(ADDIStocHA8(%x2, @sym), @sym) 5187249423Sdim SDValue GA = N->getOperand(0); 5188249423Sdim SDValue TOCbase = N->getOperand(1); 5189360784Sdim 5190360784Sdim EVT VT = isPPC64 ? MVT::i64 : MVT::i32; 5191360784Sdim SDNode *Tmp = CurDAG->getMachineNode( 5192360784Sdim isPPC64 ? PPC::ADDIStocHA8 : PPC::ADDIStocHA, dl, VT, TOCbase, GA); 5193360784Sdim 5194344779Sdim if (PPCLowering->isAccessedAsGotIndirect(GA)) { 5195360784Sdim // If it is accessed as got-indirect, we need an extra LWZ/LD to load 5196344779Sdim // the address. 5197360784Sdim SDNode *MN = CurDAG->getMachineNode( 5198360784Sdim isPPC64 ? PPC::LDtocL : PPC::LWZtocL, dl, VT, GA, SDValue(Tmp, 0)); 5199360784Sdim 5200309124Sdim transferMemOperands(N, MN); 5201309124Sdim ReplaceNode(N, MN); 5202309124Sdim return; 5203309124Sdim } 5204249423Sdim 5205360784Sdim // Build the address relative to the TOC-pointer. 5206309124Sdim ReplaceNode(N, CurDAG->getMachineNode(PPC::ADDItocL, dl, MVT::i64, 5207309124Sdim SDValue(Tmp, 0), GA)); 5208309124Sdim return; 5209193323Sed } 5210321369Sdim case PPCISD::PPC32_PICGOT: 5211270147Srdivacky // Generate a PIC-safe GOT reference. 5212360784Sdim assert(PPCSubTarget->is32BitELFABI() && 5213360784Sdim "PPCISD::PPC32_PICGOT is only supported for 32-bit SVR4"); 5214309124Sdim CurDAG->SelectNodeTo(N, PPC::PPC32PICGOT, 5215309124Sdim PPCLowering->getPointerTy(CurDAG->getDataLayout()), 5216309124Sdim MVT::i32); 5217309124Sdim return; 5218321369Sdim 5219249423Sdim case PPCISD::VADD_SPLAT: { 5220249423Sdim // This expands into one of three sequences, depending on whether 5221249423Sdim // the first operand is odd or even, positive or negative. 5222249423Sdim assert(isa<ConstantSDNode>(N->getOperand(0)) && 5223249423Sdim isa<ConstantSDNode>(N->getOperand(1)) && 5224249423Sdim "Invalid operand on VADD_SPLAT!"); 5225218893Sdim 5226249423Sdim int Elt = N->getConstantOperandVal(0); 5227249423Sdim int EltSize = N->getConstantOperandVal(1); 5228249423Sdim unsigned Opc1, Opc2, Opc3; 5229249423Sdim EVT VT; 5230249423Sdim 5231249423Sdim if (EltSize == 1) { 5232249423Sdim Opc1 = PPC::VSPLTISB; 5233249423Sdim Opc2 = PPC::VADDUBM; 5234249423Sdim Opc3 = PPC::VSUBUBM; 5235249423Sdim VT = MVT::v16i8; 5236249423Sdim } else if (EltSize == 2) { 5237249423Sdim Opc1 = PPC::VSPLTISH; 5238249423Sdim Opc2 = PPC::VADDUHM; 5239249423Sdim Opc3 = PPC::VSUBUHM; 5240249423Sdim VT = MVT::v8i16; 5241249423Sdim } else { 5242249423Sdim assert(EltSize == 4 && "Invalid element size on VADD_SPLAT!"); 5243249423Sdim Opc1 = PPC::VSPLTISW; 5244249423Sdim Opc2 = PPC::VADDUWM; 5245249423Sdim Opc3 = PPC::VSUBUWM; 5246249423Sdim VT = MVT::v4i32; 5247249423Sdim } 5248249423Sdim 5249249423Sdim if ((Elt & 1) == 0) { 5250249423Sdim // Elt is even, in the range [-32,-18] + [16,30]. 5251249423Sdim // 5252249423Sdim // Convert: VADD_SPLAT elt, size 5253249423Sdim // Into: tmp = VSPLTIS[BHW] elt 5254249423Sdim // VADDU[BHW]M tmp, tmp 5255249423Sdim // Where: [BHW] = B for size = 1, H for size = 2, W for size = 4 5256288943Sdim SDValue EltVal = getI32Imm(Elt >> 1, dl); 5257249423Sdim SDNode *Tmp = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); 5258249423Sdim SDValue TmpVal = SDValue(Tmp, 0); 5259309124Sdim ReplaceNode(N, CurDAG->getMachineNode(Opc2, dl, VT, TmpVal, TmpVal)); 5260309124Sdim return; 5261249423Sdim } else if (Elt > 0) { 5262249423Sdim // Elt is odd and positive, in the range [17,31]. 5263249423Sdim // 5264249423Sdim // Convert: VADD_SPLAT elt, size 5265249423Sdim // Into: tmp1 = VSPLTIS[BHW] elt-16 5266249423Sdim // tmp2 = VSPLTIS[BHW] -16 5267249423Sdim // VSUBU[BHW]M tmp1, tmp2 5268288943Sdim SDValue EltVal = getI32Imm(Elt - 16, dl); 5269249423Sdim SDNode *Tmp1 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); 5270288943Sdim EltVal = getI32Imm(-16, dl); 5271249423Sdim SDNode *Tmp2 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); 5272309124Sdim ReplaceNode(N, CurDAG->getMachineNode(Opc3, dl, VT, SDValue(Tmp1, 0), 5273309124Sdim SDValue(Tmp2, 0))); 5274309124Sdim return; 5275249423Sdim } else { 5276249423Sdim // Elt is odd and negative, in the range [-31,-17]. 5277249423Sdim // 5278249423Sdim // Convert: VADD_SPLAT elt, size 5279249423Sdim // Into: tmp1 = VSPLTIS[BHW] elt+16 5280249423Sdim // tmp2 = VSPLTIS[BHW] -16 5281249423Sdim // VADDU[BHW]M tmp1, tmp2 5282288943Sdim SDValue EltVal = getI32Imm(Elt + 16, dl); 5283249423Sdim SDNode *Tmp1 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); 5284288943Sdim EltVal = getI32Imm(-16, dl); 5285249423Sdim SDNode *Tmp2 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); 5286309124Sdim ReplaceNode(N, CurDAG->getMachineNode(Opc2, dl, VT, SDValue(Tmp1, 0), 5287309124Sdim SDValue(Tmp2, 0))); 5288309124Sdim return; 5289249423Sdim } 5290249423Sdim } 5291249423Sdim } 5292249423Sdim 5293309124Sdim SelectCode(N); 5294193323Sed} 5295193323Sed 5296280031Sdim// If the target supports the cmpb instruction, do the idiom recognition here. 5297280031Sdim// We don't do this as a DAG combine because we don't want to do it as nodes 5298280031Sdim// are being combined (because we might miss part of the eventual idiom). We 5299280031Sdim// don't want to do it during instruction selection because we want to reuse 5300280031Sdim// the logic for lowering the masking operations already part of the 5301280031Sdim// instruction selector. 5302280031SdimSDValue PPCDAGToDAGISel::combineToCMPB(SDNode *N) { 5303280031Sdim SDLoc dl(N); 5304280031Sdim 5305280031Sdim assert(N->getOpcode() == ISD::OR && 5306280031Sdim "Only OR nodes are supported for CMPB"); 5307280031Sdim 5308280031Sdim SDValue Res; 5309280031Sdim if (!PPCSubTarget->hasCMPB()) 5310280031Sdim return Res; 5311280031Sdim 5312280031Sdim if (N->getValueType(0) != MVT::i32 && 5313280031Sdim N->getValueType(0) != MVT::i64) 5314280031Sdim return Res; 5315280031Sdim 5316280031Sdim EVT VT = N->getValueType(0); 5317280031Sdim 5318280031Sdim SDValue RHS, LHS; 5319321369Sdim bool BytesFound[8] = {false, false, false, false, false, false, false, false}; 5320280031Sdim uint64_t Mask = 0, Alt = 0; 5321280031Sdim 5322280031Sdim auto IsByteSelectCC = [this](SDValue O, unsigned &b, 5323280031Sdim uint64_t &Mask, uint64_t &Alt, 5324280031Sdim SDValue &LHS, SDValue &RHS) { 5325280031Sdim if (O.getOpcode() != ISD::SELECT_CC) 5326280031Sdim return false; 5327280031Sdim ISD::CondCode CC = cast<CondCodeSDNode>(O.getOperand(4))->get(); 5328280031Sdim 5329280031Sdim if (!isa<ConstantSDNode>(O.getOperand(2)) || 5330280031Sdim !isa<ConstantSDNode>(O.getOperand(3))) 5331280031Sdim return false; 5332280031Sdim 5333280031Sdim uint64_t PM = O.getConstantOperandVal(2); 5334280031Sdim uint64_t PAlt = O.getConstantOperandVal(3); 5335280031Sdim for (b = 0; b < 8; ++b) { 5336280031Sdim uint64_t Mask = UINT64_C(0xFF) << (8*b); 5337280031Sdim if (PM && (PM & Mask) == PM && (PAlt & Mask) == PAlt) 5338280031Sdim break; 5339280031Sdim } 5340280031Sdim 5341280031Sdim if (b == 8) 5342280031Sdim return false; 5343280031Sdim Mask |= PM; 5344280031Sdim Alt |= PAlt; 5345280031Sdim 5346280031Sdim if (!isa<ConstantSDNode>(O.getOperand(1)) || 5347280031Sdim O.getConstantOperandVal(1) != 0) { 5348280031Sdim SDValue Op0 = O.getOperand(0), Op1 = O.getOperand(1); 5349280031Sdim if (Op0.getOpcode() == ISD::TRUNCATE) 5350280031Sdim Op0 = Op0.getOperand(0); 5351280031Sdim if (Op1.getOpcode() == ISD::TRUNCATE) 5352280031Sdim Op1 = Op1.getOperand(0); 5353280031Sdim 5354280031Sdim if (Op0.getOpcode() == ISD::SRL && Op1.getOpcode() == ISD::SRL && 5355280031Sdim Op0.getOperand(1) == Op1.getOperand(1) && CC == ISD::SETEQ && 5356280031Sdim isa<ConstantSDNode>(Op0.getOperand(1))) { 5357280031Sdim 5358314564Sdim unsigned Bits = Op0.getValueSizeInBits(); 5359280031Sdim if (b != Bits/8-1) 5360280031Sdim return false; 5361280031Sdim if (Op0.getConstantOperandVal(1) != Bits-8) 5362280031Sdim return false; 5363280031Sdim 5364280031Sdim LHS = Op0.getOperand(0); 5365280031Sdim RHS = Op1.getOperand(0); 5366280031Sdim return true; 5367280031Sdim } 5368280031Sdim 5369280031Sdim // When we have small integers (i16 to be specific), the form present 5370280031Sdim // post-legalization uses SETULT in the SELECT_CC for the 5371280031Sdim // higher-order byte, depending on the fact that the 5372280031Sdim // even-higher-order bytes are known to all be zero, for example: 5373280031Sdim // select_cc (xor $lhs, $rhs), 256, 65280, 0, setult 5374280031Sdim // (so when the second byte is the same, because all higher-order 5375280031Sdim // bits from bytes 3 and 4 are known to be zero, the result of the 5376280031Sdim // xor can be at most 255) 5377280031Sdim if (Op0.getOpcode() == ISD::XOR && CC == ISD::SETULT && 5378280031Sdim isa<ConstantSDNode>(O.getOperand(1))) { 5379280031Sdim 5380280031Sdim uint64_t ULim = O.getConstantOperandVal(1); 5381280031Sdim if (ULim != (UINT64_C(1) << b*8)) 5382280031Sdim return false; 5383280031Sdim 5384280031Sdim // Now we need to make sure that the upper bytes are known to be 5385280031Sdim // zero. 5386314564Sdim unsigned Bits = Op0.getValueSizeInBits(); 5387314564Sdim if (!CurDAG->MaskedValueIsZero( 5388314564Sdim Op0, APInt::getHighBitsSet(Bits, Bits - (b + 1) * 8))) 5389280031Sdim return false; 5390296417Sdim 5391280031Sdim LHS = Op0.getOperand(0); 5392280031Sdim RHS = Op0.getOperand(1); 5393280031Sdim return true; 5394280031Sdim } 5395280031Sdim 5396280031Sdim return false; 5397280031Sdim } 5398280031Sdim 5399280031Sdim if (CC != ISD::SETEQ) 5400280031Sdim return false; 5401280031Sdim 5402280031Sdim SDValue Op = O.getOperand(0); 5403280031Sdim if (Op.getOpcode() == ISD::AND) { 5404280031Sdim if (!isa<ConstantSDNode>(Op.getOperand(1))) 5405280031Sdim return false; 5406280031Sdim if (Op.getConstantOperandVal(1) != (UINT64_C(0xFF) << (8*b))) 5407280031Sdim return false; 5408280031Sdim 5409280031Sdim SDValue XOR = Op.getOperand(0); 5410280031Sdim if (XOR.getOpcode() == ISD::TRUNCATE) 5411280031Sdim XOR = XOR.getOperand(0); 5412280031Sdim if (XOR.getOpcode() != ISD::XOR) 5413280031Sdim return false; 5414280031Sdim 5415280031Sdim LHS = XOR.getOperand(0); 5416280031Sdim RHS = XOR.getOperand(1); 5417280031Sdim return true; 5418280031Sdim } else if (Op.getOpcode() == ISD::SRL) { 5419280031Sdim if (!isa<ConstantSDNode>(Op.getOperand(1))) 5420280031Sdim return false; 5421314564Sdim unsigned Bits = Op.getValueSizeInBits(); 5422280031Sdim if (b != Bits/8-1) 5423280031Sdim return false; 5424280031Sdim if (Op.getConstantOperandVal(1) != Bits-8) 5425280031Sdim return false; 5426280031Sdim 5427280031Sdim SDValue XOR = Op.getOperand(0); 5428280031Sdim if (XOR.getOpcode() == ISD::TRUNCATE) 5429280031Sdim XOR = XOR.getOperand(0); 5430280031Sdim if (XOR.getOpcode() != ISD::XOR) 5431280031Sdim return false; 5432280031Sdim 5433280031Sdim LHS = XOR.getOperand(0); 5434280031Sdim RHS = XOR.getOperand(1); 5435280031Sdim return true; 5436280031Sdim } 5437280031Sdim 5438280031Sdim return false; 5439280031Sdim }; 5440280031Sdim 5441280031Sdim SmallVector<SDValue, 8> Queue(1, SDValue(N, 0)); 5442280031Sdim while (!Queue.empty()) { 5443280031Sdim SDValue V = Queue.pop_back_val(); 5444280031Sdim 5445280031Sdim for (const SDValue &O : V.getNode()->ops()) { 5446353358Sdim unsigned b = 0; 5447280031Sdim uint64_t M = 0, A = 0; 5448280031Sdim SDValue OLHS, ORHS; 5449280031Sdim if (O.getOpcode() == ISD::OR) { 5450280031Sdim Queue.push_back(O); 5451280031Sdim } else if (IsByteSelectCC(O, b, M, A, OLHS, ORHS)) { 5452280031Sdim if (!LHS) { 5453280031Sdim LHS = OLHS; 5454280031Sdim RHS = ORHS; 5455280031Sdim BytesFound[b] = true; 5456280031Sdim Mask |= M; 5457280031Sdim Alt |= A; 5458280031Sdim } else if ((LHS == ORHS && RHS == OLHS) || 5459280031Sdim (RHS == ORHS && LHS == OLHS)) { 5460280031Sdim BytesFound[b] = true; 5461280031Sdim Mask |= M; 5462280031Sdim Alt |= A; 5463280031Sdim } else { 5464280031Sdim return Res; 5465280031Sdim } 5466280031Sdim } else { 5467280031Sdim return Res; 5468280031Sdim } 5469280031Sdim } 5470280031Sdim } 5471280031Sdim 5472280031Sdim unsigned LastB = 0, BCnt = 0; 5473280031Sdim for (unsigned i = 0; i < 8; ++i) 5474280031Sdim if (BytesFound[LastB]) { 5475280031Sdim ++BCnt; 5476280031Sdim LastB = i; 5477280031Sdim } 5478280031Sdim 5479280031Sdim if (!LastB || BCnt < 2) 5480280031Sdim return Res; 5481280031Sdim 5482280031Sdim // Because we'll be zero-extending the output anyway if don't have a specific 5483280031Sdim // value for each input byte (via the Mask), we can 'anyext' the inputs. 5484280031Sdim if (LHS.getValueType() != VT) { 5485280031Sdim LHS = CurDAG->getAnyExtOrTrunc(LHS, dl, VT); 5486280031Sdim RHS = CurDAG->getAnyExtOrTrunc(RHS, dl, VT); 5487280031Sdim } 5488280031Sdim 5489280031Sdim Res = CurDAG->getNode(PPCISD::CMPB, dl, VT, LHS, RHS); 5490280031Sdim 5491280031Sdim bool NonTrivialMask = ((int64_t) Mask) != INT64_C(-1); 5492280031Sdim if (NonTrivialMask && !Alt) { 5493280031Sdim // Res = Mask & CMPB 5494288943Sdim Res = CurDAG->getNode(ISD::AND, dl, VT, Res, 5495288943Sdim CurDAG->getConstant(Mask, dl, VT)); 5496280031Sdim } else if (Alt) { 5497280031Sdim // Res = (CMPB & Mask) | (~CMPB & Alt) 5498280031Sdim // Which, as suggested here: 5499280031Sdim // https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge 5500280031Sdim // can be written as: 5501280031Sdim // Res = Alt ^ ((Alt ^ Mask) & CMPB) 5502280031Sdim // useful because the (Alt ^ Mask) can be pre-computed. 5503280031Sdim Res = CurDAG->getNode(ISD::AND, dl, VT, Res, 5504288943Sdim CurDAG->getConstant(Mask ^ Alt, dl, VT)); 5505288943Sdim Res = CurDAG->getNode(ISD::XOR, dl, VT, Res, 5506288943Sdim CurDAG->getConstant(Alt, dl, VT)); 5507280031Sdim } 5508280031Sdim 5509280031Sdim return Res; 5510280031Sdim} 5511280031Sdim 5512280031Sdim// When CR bit registers are enabled, an extension of an i1 variable to a i32 5513280031Sdim// or i64 value is lowered in terms of a SELECT_I[48] operation, and thus 5514280031Sdim// involves constant materialization of a 0 or a 1 or both. If the result of 5515280031Sdim// the extension is then operated upon by some operator that can be constant 5516280031Sdim// folded with a constant 0 or 1, and that constant can be materialized using 5517280031Sdim// only one instruction (like a zero or one), then we should fold in those 5518280031Sdim// operations with the select. 5519280031Sdimvoid PPCDAGToDAGISel::foldBoolExts(SDValue &Res, SDNode *&N) { 5520280031Sdim if (!PPCSubTarget->useCRBits()) 5521280031Sdim return; 5522280031Sdim 5523280031Sdim if (N->getOpcode() != ISD::ZERO_EXTEND && 5524280031Sdim N->getOpcode() != ISD::SIGN_EXTEND && 5525280031Sdim N->getOpcode() != ISD::ANY_EXTEND) 5526280031Sdim return; 5527280031Sdim 5528280031Sdim if (N->getOperand(0).getValueType() != MVT::i1) 5529280031Sdim return; 5530280031Sdim 5531280031Sdim if (!N->hasOneUse()) 5532280031Sdim return; 5533280031Sdim 5534280031Sdim SDLoc dl(N); 5535280031Sdim EVT VT = N->getValueType(0); 5536280031Sdim SDValue Cond = N->getOperand(0); 5537280031Sdim SDValue ConstTrue = 5538288943Sdim CurDAG->getConstant(N->getOpcode() == ISD::SIGN_EXTEND ? -1 : 1, dl, VT); 5539288943Sdim SDValue ConstFalse = CurDAG->getConstant(0, dl, VT); 5540280031Sdim 5541280031Sdim do { 5542280031Sdim SDNode *User = *N->use_begin(); 5543280031Sdim if (User->getNumOperands() != 2) 5544280031Sdim break; 5545280031Sdim 5546288943Sdim auto TryFold = [this, N, User, dl](SDValue Val) { 5547280031Sdim SDValue UserO0 = User->getOperand(0), UserO1 = User->getOperand(1); 5548280031Sdim SDValue O0 = UserO0.getNode() == N ? Val : UserO0; 5549280031Sdim SDValue O1 = UserO1.getNode() == N ? Val : UserO1; 5550280031Sdim 5551288943Sdim return CurDAG->FoldConstantArithmetic(User->getOpcode(), dl, 5552280031Sdim User->getValueType(0), 5553280031Sdim O0.getNode(), O1.getNode()); 5554280031Sdim }; 5555280031Sdim 5556321369Sdim // FIXME: When the semantics of the interaction between select and undef 5557321369Sdim // are clearly defined, it may turn out to be unnecessary to break here. 5558280031Sdim SDValue TrueRes = TryFold(ConstTrue); 5559321369Sdim if (!TrueRes || TrueRes.isUndef()) 5560280031Sdim break; 5561280031Sdim SDValue FalseRes = TryFold(ConstFalse); 5562321369Sdim if (!FalseRes || FalseRes.isUndef()) 5563280031Sdim break; 5564280031Sdim 5565280031Sdim // For us to materialize these using one instruction, we must be able to 5566280031Sdim // represent them as signed 16-bit integers. 5567280031Sdim uint64_t True = cast<ConstantSDNode>(TrueRes)->getZExtValue(), 5568280031Sdim False = cast<ConstantSDNode>(FalseRes)->getZExtValue(); 5569280031Sdim if (!isInt<16>(True) || !isInt<16>(False)) 5570280031Sdim break; 5571280031Sdim 5572280031Sdim // We can replace User with a new SELECT node, and try again to see if we 5573280031Sdim // can fold the select with its user. 5574280031Sdim Res = CurDAG->getSelect(dl, User->getValueType(0), Cond, TrueRes, FalseRes); 5575280031Sdim N = User; 5576280031Sdim ConstTrue = TrueRes; 5577280031Sdim ConstFalse = FalseRes; 5578280031Sdim } while (N->hasOneUse()); 5579280031Sdim} 5580280031Sdim 5581280031Sdimvoid PPCDAGToDAGISel::PreprocessISelDAG() { 5582341825Sdim SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); 5583280031Sdim 5584280031Sdim bool MadeChange = false; 5585280031Sdim while (Position != CurDAG->allnodes_begin()) { 5586296417Sdim SDNode *N = &*--Position; 5587280031Sdim if (N->use_empty()) 5588280031Sdim continue; 5589280031Sdim 5590280031Sdim SDValue Res; 5591280031Sdim switch (N->getOpcode()) { 5592280031Sdim default: break; 5593280031Sdim case ISD::OR: 5594280031Sdim Res = combineToCMPB(N); 5595280031Sdim break; 5596280031Sdim } 5597280031Sdim 5598280031Sdim if (!Res) 5599280031Sdim foldBoolExts(Res, N); 5600280031Sdim 5601280031Sdim if (Res) { 5602341825Sdim LLVM_DEBUG(dbgs() << "PPC DAG preprocessing replacing:\nOld: "); 5603341825Sdim LLVM_DEBUG(N->dump(CurDAG)); 5604341825Sdim LLVM_DEBUG(dbgs() << "\nNew: "); 5605341825Sdim LLVM_DEBUG(Res.getNode()->dump(CurDAG)); 5606341825Sdim LLVM_DEBUG(dbgs() << "\n"); 5607280031Sdim 5608280031Sdim CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res); 5609280031Sdim MadeChange = true; 5610280031Sdim } 5611280031Sdim } 5612280031Sdim 5613280031Sdim if (MadeChange) 5614280031Sdim CurDAG->RemoveDeadNodes(); 5615280031Sdim} 5616280031Sdim 5617276479Sdim/// PostprocessISelDAG - Perform some late peephole optimizations 5618249423Sdim/// on the DAG representation. 5619249423Sdimvoid PPCDAGToDAGISel::PostprocessISelDAG() { 5620249423Sdim // Skip peepholes at -O0. 5621249423Sdim if (TM.getOptLevel() == CodeGenOpt::None) 5622249423Sdim return; 5623193323Sed 5624276479Sdim PeepholePPC64(); 5625276479Sdim PeepholeCROps(); 5626280031Sdim PeepholePPC64ZExt(); 5627276479Sdim} 5628276479Sdim 5629276479Sdim// Check if all users of this node will become isel where the second operand 5630276479Sdim// is the constant zero. If this is so, and if we can negate the condition, 5631276479Sdim// then we can flip the true and false operands. This will allow the zero to 5632276479Sdim// be folded with the isel so that we don't need to materialize a register 5633276479Sdim// containing zero. 5634276479Sdimbool PPCDAGToDAGISel::AllUsersSelectZero(SDNode *N) { 5635276479Sdim for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); 5636276479Sdim UI != UE; ++UI) { 5637276479Sdim SDNode *User = *UI; 5638276479Sdim if (!User->isMachineOpcode()) 5639276479Sdim return false; 5640276479Sdim if (User->getMachineOpcode() != PPC::SELECT_I4 && 5641276479Sdim User->getMachineOpcode() != PPC::SELECT_I8) 5642276479Sdim return false; 5643276479Sdim 5644276479Sdim SDNode *Op2 = User->getOperand(2).getNode(); 5645276479Sdim if (!Op2->isMachineOpcode()) 5646276479Sdim return false; 5647276479Sdim 5648276479Sdim if (Op2->getMachineOpcode() != PPC::LI && 5649276479Sdim Op2->getMachineOpcode() != PPC::LI8) 5650276479Sdim return false; 5651276479Sdim 5652276479Sdim ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op2->getOperand(0)); 5653276479Sdim if (!C) 5654276479Sdim return false; 5655276479Sdim 5656276479Sdim if (!C->isNullValue()) 5657276479Sdim return false; 5658276479Sdim } 5659276479Sdim 5660276479Sdim return true; 5661276479Sdim} 5662276479Sdim 5663276479Sdimvoid PPCDAGToDAGISel::SwapAllSelectUsers(SDNode *N) { 5664276479Sdim SmallVector<SDNode *, 4> ToReplace; 5665276479Sdim for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); 5666276479Sdim UI != UE; ++UI) { 5667276479Sdim SDNode *User = *UI; 5668276479Sdim assert((User->getMachineOpcode() == PPC::SELECT_I4 || 5669276479Sdim User->getMachineOpcode() == PPC::SELECT_I8) && 5670276479Sdim "Must have all select users"); 5671276479Sdim ToReplace.push_back(User); 5672276479Sdim } 5673276479Sdim 5674276479Sdim for (SmallVector<SDNode *, 4>::iterator UI = ToReplace.begin(), 5675276479Sdim UE = ToReplace.end(); UI != UE; ++UI) { 5676276479Sdim SDNode *User = *UI; 5677276479Sdim SDNode *ResNode = 5678276479Sdim CurDAG->getMachineNode(User->getMachineOpcode(), SDLoc(User), 5679276479Sdim User->getValueType(0), User->getOperand(0), 5680276479Sdim User->getOperand(2), 5681276479Sdim User->getOperand(1)); 5682276479Sdim 5683341825Sdim LLVM_DEBUG(dbgs() << "CR Peephole replacing:\nOld: "); 5684341825Sdim LLVM_DEBUG(User->dump(CurDAG)); 5685341825Sdim LLVM_DEBUG(dbgs() << "\nNew: "); 5686341825Sdim LLVM_DEBUG(ResNode->dump(CurDAG)); 5687341825Sdim LLVM_DEBUG(dbgs() << "\n"); 5688276479Sdim 5689341825Sdim ReplaceUses(User, ResNode); 5690276479Sdim } 5691276479Sdim} 5692276479Sdim 5693276479Sdimvoid PPCDAGToDAGISel::PeepholeCROps() { 5694276479Sdim bool IsModified; 5695276479Sdim do { 5696276479Sdim IsModified = false; 5697288943Sdim for (SDNode &Node : CurDAG->allnodes()) { 5698288943Sdim MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(&Node); 5699276479Sdim if (!MachineNode || MachineNode->use_empty()) 5700276479Sdim continue; 5701276479Sdim SDNode *ResNode = MachineNode; 5702276479Sdim 5703276479Sdim bool Op1Set = false, Op1Unset = false, 5704276479Sdim Op1Not = false, 5705276479Sdim Op2Set = false, Op2Unset = false, 5706276479Sdim Op2Not = false; 5707276479Sdim 5708276479Sdim unsigned Opcode = MachineNode->getMachineOpcode(); 5709276479Sdim switch (Opcode) { 5710276479Sdim default: break; 5711276479Sdim case PPC::CRAND: 5712276479Sdim case PPC::CRNAND: 5713276479Sdim case PPC::CROR: 5714276479Sdim case PPC::CRXOR: 5715276479Sdim case PPC::CRNOR: 5716276479Sdim case PPC::CREQV: 5717276479Sdim case PPC::CRANDC: 5718276479Sdim case PPC::CRORC: { 5719276479Sdim SDValue Op = MachineNode->getOperand(1); 5720276479Sdim if (Op.isMachineOpcode()) { 5721276479Sdim if (Op.getMachineOpcode() == PPC::CRSET) 5722276479Sdim Op2Set = true; 5723276479Sdim else if (Op.getMachineOpcode() == PPC::CRUNSET) 5724276479Sdim Op2Unset = true; 5725276479Sdim else if (Op.getMachineOpcode() == PPC::CRNOR && 5726276479Sdim Op.getOperand(0) == Op.getOperand(1)) 5727276479Sdim Op2Not = true; 5728276479Sdim } 5729314564Sdim LLVM_FALLTHROUGH; 5730314564Sdim } 5731276479Sdim case PPC::BC: 5732276479Sdim case PPC::BCn: 5733276479Sdim case PPC::SELECT_I4: 5734276479Sdim case PPC::SELECT_I8: 5735276479Sdim case PPC::SELECT_F4: 5736276479Sdim case PPC::SELECT_F8: 5737288943Sdim case PPC::SELECT_QFRC: 5738288943Sdim case PPC::SELECT_QSRC: 5739288943Sdim case PPC::SELECT_QBRC: 5740341825Sdim case PPC::SELECT_SPE: 5741341825Sdim case PPC::SELECT_SPE4: 5742280031Sdim case PPC::SELECT_VRRC: 5743280031Sdim case PPC::SELECT_VSFRC: 5744288943Sdim case PPC::SELECT_VSSRC: 5745280031Sdim case PPC::SELECT_VSRC: { 5746276479Sdim SDValue Op = MachineNode->getOperand(0); 5747276479Sdim if (Op.isMachineOpcode()) { 5748276479Sdim if (Op.getMachineOpcode() == PPC::CRSET) 5749276479Sdim Op1Set = true; 5750276479Sdim else if (Op.getMachineOpcode() == PPC::CRUNSET) 5751276479Sdim Op1Unset = true; 5752276479Sdim else if (Op.getMachineOpcode() == PPC::CRNOR && 5753276479Sdim Op.getOperand(0) == Op.getOperand(1)) 5754276479Sdim Op1Not = true; 5755276479Sdim } 5756276479Sdim } 5757276479Sdim break; 5758276479Sdim } 5759276479Sdim 5760276479Sdim bool SelectSwap = false; 5761276479Sdim switch (Opcode) { 5762276479Sdim default: break; 5763276479Sdim case PPC::CRAND: 5764276479Sdim if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) 5765276479Sdim // x & x = x 5766276479Sdim ResNode = MachineNode->getOperand(0).getNode(); 5767276479Sdim else if (Op1Set) 5768276479Sdim // 1 & y = y 5769276479Sdim ResNode = MachineNode->getOperand(1).getNode(); 5770276479Sdim else if (Op2Set) 5771276479Sdim // x & 1 = x 5772276479Sdim ResNode = MachineNode->getOperand(0).getNode(); 5773276479Sdim else if (Op1Unset || Op2Unset) 5774276479Sdim // x & 0 = 0 & y = 0 5775276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), 5776276479Sdim MVT::i1); 5777276479Sdim else if (Op1Not) 5778276479Sdim // ~x & y = andc(y, x) 5779276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), 5780276479Sdim MVT::i1, MachineNode->getOperand(1), 5781276479Sdim MachineNode->getOperand(0). 5782276479Sdim getOperand(0)); 5783276479Sdim else if (Op2Not) 5784276479Sdim // x & ~y = andc(x, y) 5785276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), 5786276479Sdim MVT::i1, MachineNode->getOperand(0), 5787276479Sdim MachineNode->getOperand(1). 5788276479Sdim getOperand(0)); 5789309124Sdim else if (AllUsersSelectZero(MachineNode)) { 5790276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRNAND, SDLoc(MachineNode), 5791276479Sdim MVT::i1, MachineNode->getOperand(0), 5792309124Sdim MachineNode->getOperand(1)); 5793276479Sdim SelectSwap = true; 5794309124Sdim } 5795276479Sdim break; 5796276479Sdim case PPC::CRNAND: 5797276479Sdim if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) 5798276479Sdim // nand(x, x) -> nor(x, x) 5799276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 5800276479Sdim MVT::i1, MachineNode->getOperand(0), 5801276479Sdim MachineNode->getOperand(0)); 5802276479Sdim else if (Op1Set) 5803276479Sdim // nand(1, y) -> nor(y, y) 5804276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 5805276479Sdim MVT::i1, MachineNode->getOperand(1), 5806276479Sdim MachineNode->getOperand(1)); 5807276479Sdim else if (Op2Set) 5808276479Sdim // nand(x, 1) -> nor(x, x) 5809276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 5810276479Sdim MVT::i1, MachineNode->getOperand(0), 5811276479Sdim MachineNode->getOperand(0)); 5812276479Sdim else if (Op1Unset || Op2Unset) 5813276479Sdim // nand(x, 0) = nand(0, y) = 1 5814276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), 5815276479Sdim MVT::i1); 5816276479Sdim else if (Op1Not) 5817276479Sdim // nand(~x, y) = ~(~x & y) = x | ~y = orc(x, y) 5818276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), 5819276479Sdim MVT::i1, MachineNode->getOperand(0). 5820276479Sdim getOperand(0), 5821276479Sdim MachineNode->getOperand(1)); 5822276479Sdim else if (Op2Not) 5823276479Sdim // nand(x, ~y) = ~x | y = orc(y, x) 5824276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), 5825276479Sdim MVT::i1, MachineNode->getOperand(1). 5826276479Sdim getOperand(0), 5827276479Sdim MachineNode->getOperand(0)); 5828309124Sdim else if (AllUsersSelectZero(MachineNode)) { 5829276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRAND, SDLoc(MachineNode), 5830276479Sdim MVT::i1, MachineNode->getOperand(0), 5831309124Sdim MachineNode->getOperand(1)); 5832276479Sdim SelectSwap = true; 5833309124Sdim } 5834276479Sdim break; 5835276479Sdim case PPC::CROR: 5836276479Sdim if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) 5837276479Sdim // x | x = x 5838276479Sdim ResNode = MachineNode->getOperand(0).getNode(); 5839276479Sdim else if (Op1Set || Op2Set) 5840276479Sdim // x | 1 = 1 | y = 1 5841276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), 5842276479Sdim MVT::i1); 5843276479Sdim else if (Op1Unset) 5844276479Sdim // 0 | y = y 5845276479Sdim ResNode = MachineNode->getOperand(1).getNode(); 5846276479Sdim else if (Op2Unset) 5847276479Sdim // x | 0 = x 5848276479Sdim ResNode = MachineNode->getOperand(0).getNode(); 5849276479Sdim else if (Op1Not) 5850276479Sdim // ~x | y = orc(y, x) 5851276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), 5852276479Sdim MVT::i1, MachineNode->getOperand(1), 5853276479Sdim MachineNode->getOperand(0). 5854276479Sdim getOperand(0)); 5855276479Sdim else if (Op2Not) 5856276479Sdim // x | ~y = orc(x, y) 5857276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), 5858276479Sdim MVT::i1, MachineNode->getOperand(0), 5859276479Sdim MachineNode->getOperand(1). 5860276479Sdim getOperand(0)); 5861309124Sdim else if (AllUsersSelectZero(MachineNode)) { 5862276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 5863276479Sdim MVT::i1, MachineNode->getOperand(0), 5864309124Sdim MachineNode->getOperand(1)); 5865276479Sdim SelectSwap = true; 5866309124Sdim } 5867276479Sdim break; 5868276479Sdim case PPC::CRXOR: 5869276479Sdim if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) 5870276479Sdim // xor(x, x) = 0 5871276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), 5872276479Sdim MVT::i1); 5873276479Sdim else if (Op1Set) 5874276479Sdim // xor(1, y) -> nor(y, y) 5875276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 5876276479Sdim MVT::i1, MachineNode->getOperand(1), 5877276479Sdim MachineNode->getOperand(1)); 5878276479Sdim else if (Op2Set) 5879276479Sdim // xor(x, 1) -> nor(x, x) 5880276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 5881276479Sdim MVT::i1, MachineNode->getOperand(0), 5882276479Sdim MachineNode->getOperand(0)); 5883276479Sdim else if (Op1Unset) 5884276479Sdim // xor(0, y) = y 5885276479Sdim ResNode = MachineNode->getOperand(1).getNode(); 5886276479Sdim else if (Op2Unset) 5887276479Sdim // xor(x, 0) = x 5888276479Sdim ResNode = MachineNode->getOperand(0).getNode(); 5889276479Sdim else if (Op1Not) 5890276479Sdim // xor(~x, y) = eqv(x, y) 5891276479Sdim ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode), 5892276479Sdim MVT::i1, MachineNode->getOperand(0). 5893276479Sdim getOperand(0), 5894276479Sdim MachineNode->getOperand(1)); 5895276479Sdim else if (Op2Not) 5896276479Sdim // xor(x, ~y) = eqv(x, y) 5897276479Sdim ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode), 5898276479Sdim MVT::i1, MachineNode->getOperand(0), 5899276479Sdim MachineNode->getOperand(1). 5900276479Sdim getOperand(0)); 5901309124Sdim else if (AllUsersSelectZero(MachineNode)) { 5902276479Sdim ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode), 5903276479Sdim MVT::i1, MachineNode->getOperand(0), 5904309124Sdim MachineNode->getOperand(1)); 5905276479Sdim SelectSwap = true; 5906309124Sdim } 5907276479Sdim break; 5908276479Sdim case PPC::CRNOR: 5909276479Sdim if (Op1Set || Op2Set) 5910276479Sdim // nor(1, y) -> 0 5911276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), 5912276479Sdim MVT::i1); 5913276479Sdim else if (Op1Unset) 5914276479Sdim // nor(0, y) = ~y -> nor(y, y) 5915276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 5916276479Sdim MVT::i1, MachineNode->getOperand(1), 5917276479Sdim MachineNode->getOperand(1)); 5918276479Sdim else if (Op2Unset) 5919276479Sdim // nor(x, 0) = ~x 5920276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 5921276479Sdim MVT::i1, MachineNode->getOperand(0), 5922276479Sdim MachineNode->getOperand(0)); 5923276479Sdim else if (Op1Not) 5924276479Sdim // nor(~x, y) = andc(x, y) 5925276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), 5926276479Sdim MVT::i1, MachineNode->getOperand(0). 5927276479Sdim getOperand(0), 5928276479Sdim MachineNode->getOperand(1)); 5929276479Sdim else if (Op2Not) 5930276479Sdim // nor(x, ~y) = andc(y, x) 5931276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), 5932276479Sdim MVT::i1, MachineNode->getOperand(1). 5933276479Sdim getOperand(0), 5934276479Sdim MachineNode->getOperand(0)); 5935309124Sdim else if (AllUsersSelectZero(MachineNode)) { 5936276479Sdim ResNode = CurDAG->getMachineNode(PPC::CROR, SDLoc(MachineNode), 5937276479Sdim MVT::i1, MachineNode->getOperand(0), 5938309124Sdim MachineNode->getOperand(1)); 5939276479Sdim SelectSwap = true; 5940309124Sdim } 5941276479Sdim break; 5942276479Sdim case PPC::CREQV: 5943276479Sdim if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) 5944276479Sdim // eqv(x, x) = 1 5945276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), 5946276479Sdim MVT::i1); 5947276479Sdim else if (Op1Set) 5948276479Sdim // eqv(1, y) = y 5949276479Sdim ResNode = MachineNode->getOperand(1).getNode(); 5950276479Sdim else if (Op2Set) 5951276479Sdim // eqv(x, 1) = x 5952276479Sdim ResNode = MachineNode->getOperand(0).getNode(); 5953276479Sdim else if (Op1Unset) 5954276479Sdim // eqv(0, y) = ~y -> nor(y, y) 5955276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 5956276479Sdim MVT::i1, MachineNode->getOperand(1), 5957276479Sdim MachineNode->getOperand(1)); 5958276479Sdim else if (Op2Unset) 5959276479Sdim // eqv(x, 0) = ~x 5960276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 5961276479Sdim MVT::i1, MachineNode->getOperand(0), 5962276479Sdim MachineNode->getOperand(0)); 5963276479Sdim else if (Op1Not) 5964276479Sdim // eqv(~x, y) = xor(x, y) 5965276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode), 5966276479Sdim MVT::i1, MachineNode->getOperand(0). 5967276479Sdim getOperand(0), 5968276479Sdim MachineNode->getOperand(1)); 5969276479Sdim else if (Op2Not) 5970276479Sdim // eqv(x, ~y) = xor(x, y) 5971276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode), 5972276479Sdim MVT::i1, MachineNode->getOperand(0), 5973276479Sdim MachineNode->getOperand(1). 5974276479Sdim getOperand(0)); 5975309124Sdim else if (AllUsersSelectZero(MachineNode)) { 5976276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode), 5977276479Sdim MVT::i1, MachineNode->getOperand(0), 5978309124Sdim MachineNode->getOperand(1)); 5979276479Sdim SelectSwap = true; 5980309124Sdim } 5981276479Sdim break; 5982276479Sdim case PPC::CRANDC: 5983276479Sdim if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) 5984276479Sdim // andc(x, x) = 0 5985276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), 5986276479Sdim MVT::i1); 5987276479Sdim else if (Op1Set) 5988276479Sdim // andc(1, y) = ~y 5989276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 5990276479Sdim MVT::i1, MachineNode->getOperand(1), 5991276479Sdim MachineNode->getOperand(1)); 5992276479Sdim else if (Op1Unset || Op2Set) 5993276479Sdim // andc(0, y) = andc(x, 1) = 0 5994276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), 5995276479Sdim MVT::i1); 5996276479Sdim else if (Op2Unset) 5997276479Sdim // andc(x, 0) = x 5998276479Sdim ResNode = MachineNode->getOperand(0).getNode(); 5999276479Sdim else if (Op1Not) 6000276479Sdim // andc(~x, y) = ~(x | y) = nor(x, y) 6001276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 6002276479Sdim MVT::i1, MachineNode->getOperand(0). 6003276479Sdim getOperand(0), 6004276479Sdim MachineNode->getOperand(1)); 6005276479Sdim else if (Op2Not) 6006276479Sdim // andc(x, ~y) = x & y 6007276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRAND, SDLoc(MachineNode), 6008276479Sdim MVT::i1, MachineNode->getOperand(0), 6009276479Sdim MachineNode->getOperand(1). 6010276479Sdim getOperand(0)); 6011309124Sdim else if (AllUsersSelectZero(MachineNode)) { 6012276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), 6013276479Sdim MVT::i1, MachineNode->getOperand(1), 6014309124Sdim MachineNode->getOperand(0)); 6015276479Sdim SelectSwap = true; 6016309124Sdim } 6017276479Sdim break; 6018276479Sdim case PPC::CRORC: 6019276479Sdim if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) 6020276479Sdim // orc(x, x) = 1 6021276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), 6022276479Sdim MVT::i1); 6023276479Sdim else if (Op1Set || Op2Unset) 6024276479Sdim // orc(1, y) = orc(x, 0) = 1 6025276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), 6026276479Sdim MVT::i1); 6027276479Sdim else if (Op2Set) 6028276479Sdim // orc(x, 1) = x 6029276479Sdim ResNode = MachineNode->getOperand(0).getNode(); 6030276479Sdim else if (Op1Unset) 6031276479Sdim // orc(0, y) = ~y 6032276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 6033276479Sdim MVT::i1, MachineNode->getOperand(1), 6034276479Sdim MachineNode->getOperand(1)); 6035276479Sdim else if (Op1Not) 6036276479Sdim // orc(~x, y) = ~(x & y) = nand(x, y) 6037276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRNAND, SDLoc(MachineNode), 6038276479Sdim MVT::i1, MachineNode->getOperand(0). 6039276479Sdim getOperand(0), 6040276479Sdim MachineNode->getOperand(1)); 6041276479Sdim else if (Op2Not) 6042276479Sdim // orc(x, ~y) = x | y 6043276479Sdim ResNode = CurDAG->getMachineNode(PPC::CROR, SDLoc(MachineNode), 6044276479Sdim MVT::i1, MachineNode->getOperand(0), 6045276479Sdim MachineNode->getOperand(1). 6046276479Sdim getOperand(0)); 6047309124Sdim else if (AllUsersSelectZero(MachineNode)) { 6048276479Sdim ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), 6049276479Sdim MVT::i1, MachineNode->getOperand(1), 6050309124Sdim MachineNode->getOperand(0)); 6051276479Sdim SelectSwap = true; 6052309124Sdim } 6053276479Sdim break; 6054276479Sdim case PPC::SELECT_I4: 6055276479Sdim case PPC::SELECT_I8: 6056276479Sdim case PPC::SELECT_F4: 6057276479Sdim case PPC::SELECT_F8: 6058288943Sdim case PPC::SELECT_QFRC: 6059288943Sdim case PPC::SELECT_QSRC: 6060288943Sdim case PPC::SELECT_QBRC: 6061341825Sdim case PPC::SELECT_SPE: 6062341825Sdim case PPC::SELECT_SPE4: 6063276479Sdim case PPC::SELECT_VRRC: 6064280031Sdim case PPC::SELECT_VSFRC: 6065288943Sdim case PPC::SELECT_VSSRC: 6066280031Sdim case PPC::SELECT_VSRC: 6067276479Sdim if (Op1Set) 6068276479Sdim ResNode = MachineNode->getOperand(1).getNode(); 6069276479Sdim else if (Op1Unset) 6070276479Sdim ResNode = MachineNode->getOperand(2).getNode(); 6071276479Sdim else if (Op1Not) 6072276479Sdim ResNode = CurDAG->getMachineNode(MachineNode->getMachineOpcode(), 6073276479Sdim SDLoc(MachineNode), 6074276479Sdim MachineNode->getValueType(0), 6075276479Sdim MachineNode->getOperand(0). 6076276479Sdim getOperand(0), 6077276479Sdim MachineNode->getOperand(2), 6078276479Sdim MachineNode->getOperand(1)); 6079276479Sdim break; 6080276479Sdim case PPC::BC: 6081276479Sdim case PPC::BCn: 6082276479Sdim if (Op1Not) 6083276479Sdim ResNode = CurDAG->getMachineNode(Opcode == PPC::BC ? PPC::BCn : 6084276479Sdim PPC::BC, 6085276479Sdim SDLoc(MachineNode), 6086276479Sdim MVT::Other, 6087276479Sdim MachineNode->getOperand(0). 6088276479Sdim getOperand(0), 6089276479Sdim MachineNode->getOperand(1), 6090276479Sdim MachineNode->getOperand(2)); 6091276479Sdim // FIXME: Handle Op1Set, Op1Unset here too. 6092276479Sdim break; 6093276479Sdim } 6094276479Sdim 6095276479Sdim // If we're inverting this node because it is used only by selects that 6096276479Sdim // we'd like to swap, then swap the selects before the node replacement. 6097276479Sdim if (SelectSwap) 6098276479Sdim SwapAllSelectUsers(MachineNode); 6099276479Sdim 6100276479Sdim if (ResNode != MachineNode) { 6101341825Sdim LLVM_DEBUG(dbgs() << "CR Peephole replacing:\nOld: "); 6102341825Sdim LLVM_DEBUG(MachineNode->dump(CurDAG)); 6103341825Sdim LLVM_DEBUG(dbgs() << "\nNew: "); 6104341825Sdim LLVM_DEBUG(ResNode->dump(CurDAG)); 6105341825Sdim LLVM_DEBUG(dbgs() << "\n"); 6106276479Sdim 6107276479Sdim ReplaceUses(MachineNode, ResNode); 6108276479Sdim IsModified = true; 6109276479Sdim } 6110276479Sdim } 6111276479Sdim if (IsModified) 6112276479Sdim CurDAG->RemoveDeadNodes(); 6113276479Sdim } while (IsModified); 6114276479Sdim} 6115276479Sdim 6116280031Sdim// Gather the set of 32-bit operations that are known to have their 6117280031Sdim// higher-order 32 bits zero, where ToPromote contains all such operations. 6118280031Sdimstatic bool PeepholePPC64ZExtGather(SDValue Op32, 6119280031Sdim SmallPtrSetImpl<SDNode *> &ToPromote) { 6120280031Sdim if (!Op32.isMachineOpcode()) 6121280031Sdim return false; 6122280031Sdim 6123280031Sdim // First, check for the "frontier" instructions (those that will clear the 6124280031Sdim // higher-order 32 bits. 6125280031Sdim 6126280031Sdim // For RLWINM and RLWNM, we need to make sure that the mask does not wrap 6127280031Sdim // around. If it does not, then these instructions will clear the 6128280031Sdim // higher-order bits. 6129280031Sdim if ((Op32.getMachineOpcode() == PPC::RLWINM || 6130280031Sdim Op32.getMachineOpcode() == PPC::RLWNM) && 6131280031Sdim Op32.getConstantOperandVal(2) <= Op32.getConstantOperandVal(3)) { 6132280031Sdim ToPromote.insert(Op32.getNode()); 6133280031Sdim return true; 6134280031Sdim } 6135280031Sdim 6136280031Sdim // SLW and SRW always clear the higher-order bits. 6137280031Sdim if (Op32.getMachineOpcode() == PPC::SLW || 6138280031Sdim Op32.getMachineOpcode() == PPC::SRW) { 6139280031Sdim ToPromote.insert(Op32.getNode()); 6140280031Sdim return true; 6141280031Sdim } 6142280031Sdim 6143280031Sdim // For LI and LIS, we need the immediate to be positive (so that it is not 6144280031Sdim // sign extended). 6145280031Sdim if (Op32.getMachineOpcode() == PPC::LI || 6146280031Sdim Op32.getMachineOpcode() == PPC::LIS) { 6147280031Sdim if (!isUInt<15>(Op32.getConstantOperandVal(0))) 6148280031Sdim return false; 6149280031Sdim 6150280031Sdim ToPromote.insert(Op32.getNode()); 6151280031Sdim return true; 6152280031Sdim } 6153280031Sdim 6154280031Sdim // LHBRX and LWBRX always clear the higher-order bits. 6155280031Sdim if (Op32.getMachineOpcode() == PPC::LHBRX || 6156280031Sdim Op32.getMachineOpcode() == PPC::LWBRX) { 6157280031Sdim ToPromote.insert(Op32.getNode()); 6158280031Sdim return true; 6159280031Sdim } 6160280031Sdim 6161314564Sdim // CNT[LT]ZW always produce a 64-bit value in [0,32], and so is zero extended. 6162314564Sdim if (Op32.getMachineOpcode() == PPC::CNTLZW || 6163314564Sdim Op32.getMachineOpcode() == PPC::CNTTZW) { 6164280031Sdim ToPromote.insert(Op32.getNode()); 6165280031Sdim return true; 6166280031Sdim } 6167280031Sdim 6168280031Sdim // Next, check for those instructions we can look through. 6169280031Sdim 6170280031Sdim // Assuming the mask does not wrap around, then the higher-order bits are 6171280031Sdim // taken directly from the first operand. 6172280031Sdim if (Op32.getMachineOpcode() == PPC::RLWIMI && 6173280031Sdim Op32.getConstantOperandVal(3) <= Op32.getConstantOperandVal(4)) { 6174280031Sdim SmallPtrSet<SDNode *, 16> ToPromote1; 6175280031Sdim if (!PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1)) 6176280031Sdim return false; 6177280031Sdim 6178280031Sdim ToPromote.insert(Op32.getNode()); 6179280031Sdim ToPromote.insert(ToPromote1.begin(), ToPromote1.end()); 6180280031Sdim return true; 6181280031Sdim } 6182280031Sdim 6183280031Sdim // For OR, the higher-order bits are zero if that is true for both operands. 6184280031Sdim // For SELECT_I4, the same is true (but the relevant operand numbers are 6185280031Sdim // shifted by 1). 6186280031Sdim if (Op32.getMachineOpcode() == PPC::OR || 6187280031Sdim Op32.getMachineOpcode() == PPC::SELECT_I4) { 6188280031Sdim unsigned B = Op32.getMachineOpcode() == PPC::SELECT_I4 ? 1 : 0; 6189280031Sdim SmallPtrSet<SDNode *, 16> ToPromote1; 6190280031Sdim if (!PeepholePPC64ZExtGather(Op32.getOperand(B+0), ToPromote1)) 6191280031Sdim return false; 6192280031Sdim if (!PeepholePPC64ZExtGather(Op32.getOperand(B+1), ToPromote1)) 6193280031Sdim return false; 6194280031Sdim 6195280031Sdim ToPromote.insert(Op32.getNode()); 6196280031Sdim ToPromote.insert(ToPromote1.begin(), ToPromote1.end()); 6197280031Sdim return true; 6198280031Sdim } 6199280031Sdim 6200280031Sdim // For ORI and ORIS, we need the higher-order bits of the first operand to be 6201280031Sdim // zero, and also for the constant to be positive (so that it is not sign 6202280031Sdim // extended). 6203280031Sdim if (Op32.getMachineOpcode() == PPC::ORI || 6204280031Sdim Op32.getMachineOpcode() == PPC::ORIS) { 6205280031Sdim SmallPtrSet<SDNode *, 16> ToPromote1; 6206280031Sdim if (!PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1)) 6207280031Sdim return false; 6208280031Sdim if (!isUInt<15>(Op32.getConstantOperandVal(1))) 6209280031Sdim return false; 6210280031Sdim 6211280031Sdim ToPromote.insert(Op32.getNode()); 6212280031Sdim ToPromote.insert(ToPromote1.begin(), ToPromote1.end()); 6213280031Sdim return true; 6214280031Sdim } 6215280031Sdim 6216280031Sdim // The higher-order bits of AND are zero if that is true for at least one of 6217280031Sdim // the operands. 6218280031Sdim if (Op32.getMachineOpcode() == PPC::AND) { 6219280031Sdim SmallPtrSet<SDNode *, 16> ToPromote1, ToPromote2; 6220280031Sdim bool Op0OK = 6221280031Sdim PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1); 6222280031Sdim bool Op1OK = 6223280031Sdim PeepholePPC64ZExtGather(Op32.getOperand(1), ToPromote2); 6224280031Sdim if (!Op0OK && !Op1OK) 6225280031Sdim return false; 6226280031Sdim 6227280031Sdim ToPromote.insert(Op32.getNode()); 6228280031Sdim 6229280031Sdim if (Op0OK) 6230280031Sdim ToPromote.insert(ToPromote1.begin(), ToPromote1.end()); 6231280031Sdim 6232280031Sdim if (Op1OK) 6233280031Sdim ToPromote.insert(ToPromote2.begin(), ToPromote2.end()); 6234280031Sdim 6235280031Sdim return true; 6236280031Sdim } 6237280031Sdim 6238280031Sdim // For ANDI and ANDIS, the higher-order bits are zero if either that is true 6239280031Sdim // of the first operand, or if the second operand is positive (so that it is 6240280031Sdim // not sign extended). 6241360784Sdim if (Op32.getMachineOpcode() == PPC::ANDI_rec || 6242360784Sdim Op32.getMachineOpcode() == PPC::ANDIS_rec) { 6243280031Sdim SmallPtrSet<SDNode *, 16> ToPromote1; 6244280031Sdim bool Op0OK = 6245280031Sdim PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1); 6246280031Sdim bool Op1OK = isUInt<15>(Op32.getConstantOperandVal(1)); 6247280031Sdim if (!Op0OK && !Op1OK) 6248280031Sdim return false; 6249280031Sdim 6250280031Sdim ToPromote.insert(Op32.getNode()); 6251280031Sdim 6252280031Sdim if (Op0OK) 6253280031Sdim ToPromote.insert(ToPromote1.begin(), ToPromote1.end()); 6254280031Sdim 6255280031Sdim return true; 6256280031Sdim } 6257280031Sdim 6258280031Sdim return false; 6259280031Sdim} 6260280031Sdim 6261280031Sdimvoid PPCDAGToDAGISel::PeepholePPC64ZExt() { 6262280031Sdim if (!PPCSubTarget->isPPC64()) 6263280031Sdim return; 6264280031Sdim 6265280031Sdim // When we zero-extend from i32 to i64, we use a pattern like this: 6266280031Sdim // def : Pat<(i64 (zext i32:$in)), 6267280031Sdim // (RLDICL (INSERT_SUBREG (i64 (IMPLICIT_DEF)), $in, sub_32), 6268280031Sdim // 0, 32)>; 6269280031Sdim // There are several 32-bit shift/rotate instructions, however, that will 6270280031Sdim // clear the higher-order bits of their output, rendering the RLDICL 6271280031Sdim // unnecessary. When that happens, we remove it here, and redefine the 6272280031Sdim // relevant 32-bit operation to be a 64-bit operation. 6273280031Sdim 6274341825Sdim SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); 6275280031Sdim 6276280031Sdim bool MadeChange = false; 6277280031Sdim while (Position != CurDAG->allnodes_begin()) { 6278296417Sdim SDNode *N = &*--Position; 6279280031Sdim // Skip dead nodes and any non-machine opcodes. 6280280031Sdim if (N->use_empty() || !N->isMachineOpcode()) 6281280031Sdim continue; 6282280031Sdim 6283280031Sdim if (N->getMachineOpcode() != PPC::RLDICL) 6284280031Sdim continue; 6285280031Sdim 6286280031Sdim if (N->getConstantOperandVal(1) != 0 || 6287280031Sdim N->getConstantOperandVal(2) != 32) 6288280031Sdim continue; 6289280031Sdim 6290280031Sdim SDValue ISR = N->getOperand(0); 6291280031Sdim if (!ISR.isMachineOpcode() || 6292280031Sdim ISR.getMachineOpcode() != TargetOpcode::INSERT_SUBREG) 6293280031Sdim continue; 6294280031Sdim 6295280031Sdim if (!ISR.hasOneUse()) 6296280031Sdim continue; 6297280031Sdim 6298280031Sdim if (ISR.getConstantOperandVal(2) != PPC::sub_32) 6299280031Sdim continue; 6300280031Sdim 6301280031Sdim SDValue IDef = ISR.getOperand(0); 6302280031Sdim if (!IDef.isMachineOpcode() || 6303280031Sdim IDef.getMachineOpcode() != TargetOpcode::IMPLICIT_DEF) 6304280031Sdim continue; 6305280031Sdim 6306280031Sdim // We now know that we're looking at a canonical i32 -> i64 zext. See if we 6307280031Sdim // can get rid of it. 6308280031Sdim 6309280031Sdim SDValue Op32 = ISR->getOperand(1); 6310280031Sdim if (!Op32.isMachineOpcode()) 6311280031Sdim continue; 6312280031Sdim 6313280031Sdim // There are some 32-bit instructions that always clear the high-order 32 6314280031Sdim // bits, there are also some instructions (like AND) that we can look 6315280031Sdim // through. 6316280031Sdim SmallPtrSet<SDNode *, 16> ToPromote; 6317280031Sdim if (!PeepholePPC64ZExtGather(Op32, ToPromote)) 6318280031Sdim continue; 6319280031Sdim 6320280031Sdim // If the ToPromote set contains nodes that have uses outside of the set 6321280031Sdim // (except for the original INSERT_SUBREG), then abort the transformation. 6322280031Sdim bool OutsideUse = false; 6323280031Sdim for (SDNode *PN : ToPromote) { 6324280031Sdim for (SDNode *UN : PN->uses()) { 6325280031Sdim if (!ToPromote.count(UN) && UN != ISR.getNode()) { 6326280031Sdim OutsideUse = true; 6327280031Sdim break; 6328280031Sdim } 6329280031Sdim } 6330280031Sdim 6331280031Sdim if (OutsideUse) 6332280031Sdim break; 6333280031Sdim } 6334280031Sdim if (OutsideUse) 6335280031Sdim continue; 6336280031Sdim 6337280031Sdim MadeChange = true; 6338280031Sdim 6339280031Sdim // We now know that this zero extension can be removed by promoting to 6340280031Sdim // nodes in ToPromote to 64-bit operations, where for operations in the 6341280031Sdim // frontier of the set, we need to insert INSERT_SUBREGs for their 6342280031Sdim // operands. 6343280031Sdim for (SDNode *PN : ToPromote) { 6344280031Sdim unsigned NewOpcode; 6345280031Sdim switch (PN->getMachineOpcode()) { 6346280031Sdim default: 6347280031Sdim llvm_unreachable("Don't know the 64-bit variant of this instruction"); 6348280031Sdim case PPC::RLWINM: NewOpcode = PPC::RLWINM8; break; 6349280031Sdim case PPC::RLWNM: NewOpcode = PPC::RLWNM8; break; 6350280031Sdim case PPC::SLW: NewOpcode = PPC::SLW8; break; 6351280031Sdim case PPC::SRW: NewOpcode = PPC::SRW8; break; 6352280031Sdim case PPC::LI: NewOpcode = PPC::LI8; break; 6353280031Sdim case PPC::LIS: NewOpcode = PPC::LIS8; break; 6354280031Sdim case PPC::LHBRX: NewOpcode = PPC::LHBRX8; break; 6355280031Sdim case PPC::LWBRX: NewOpcode = PPC::LWBRX8; break; 6356280031Sdim case PPC::CNTLZW: NewOpcode = PPC::CNTLZW8; break; 6357314564Sdim case PPC::CNTTZW: NewOpcode = PPC::CNTTZW8; break; 6358280031Sdim case PPC::RLWIMI: NewOpcode = PPC::RLWIMI8; break; 6359280031Sdim case PPC::OR: NewOpcode = PPC::OR8; break; 6360280031Sdim case PPC::SELECT_I4: NewOpcode = PPC::SELECT_I8; break; 6361280031Sdim case PPC::ORI: NewOpcode = PPC::ORI8; break; 6362280031Sdim case PPC::ORIS: NewOpcode = PPC::ORIS8; break; 6363280031Sdim case PPC::AND: NewOpcode = PPC::AND8; break; 6364360784Sdim case PPC::ANDI_rec: 6365360784Sdim NewOpcode = PPC::ANDI8_rec; 6366360784Sdim break; 6367360784Sdim case PPC::ANDIS_rec: 6368360784Sdim NewOpcode = PPC::ANDIS8_rec; 6369360784Sdim break; 6370280031Sdim } 6371280031Sdim 6372280031Sdim // Note: During the replacement process, the nodes will be in an 6373280031Sdim // inconsistent state (some instructions will have operands with values 6374280031Sdim // of the wrong type). Once done, however, everything should be right 6375280031Sdim // again. 6376280031Sdim 6377280031Sdim SmallVector<SDValue, 4> Ops; 6378280031Sdim for (const SDValue &V : PN->ops()) { 6379280031Sdim if (!ToPromote.count(V.getNode()) && V.getValueType() == MVT::i32 && 6380280031Sdim !isa<ConstantSDNode>(V)) { 6381280031Sdim SDValue ReplOpOps[] = { ISR.getOperand(0), V, ISR.getOperand(2) }; 6382280031Sdim SDNode *ReplOp = 6383280031Sdim CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG, SDLoc(V), 6384280031Sdim ISR.getNode()->getVTList(), ReplOpOps); 6385280031Sdim Ops.push_back(SDValue(ReplOp, 0)); 6386280031Sdim } else { 6387280031Sdim Ops.push_back(V); 6388280031Sdim } 6389280031Sdim } 6390280031Sdim 6391280031Sdim // Because all to-be-promoted nodes only have users that are other 6392280031Sdim // promoted nodes (or the original INSERT_SUBREG), we can safely replace 6393280031Sdim // the i32 result value type with i64. 6394280031Sdim 6395280031Sdim SmallVector<EVT, 2> NewVTs; 6396280031Sdim SDVTList VTs = PN->getVTList(); 6397280031Sdim for (unsigned i = 0, ie = VTs.NumVTs; i != ie; ++i) 6398280031Sdim if (VTs.VTs[i] == MVT::i32) 6399280031Sdim NewVTs.push_back(MVT::i64); 6400280031Sdim else 6401280031Sdim NewVTs.push_back(VTs.VTs[i]); 6402280031Sdim 6403341825Sdim LLVM_DEBUG(dbgs() << "PPC64 ZExt Peephole morphing:\nOld: "); 6404341825Sdim LLVM_DEBUG(PN->dump(CurDAG)); 6405280031Sdim 6406280031Sdim CurDAG->SelectNodeTo(PN, NewOpcode, CurDAG->getVTList(NewVTs), Ops); 6407280031Sdim 6408341825Sdim LLVM_DEBUG(dbgs() << "\nNew: "); 6409341825Sdim LLVM_DEBUG(PN->dump(CurDAG)); 6410341825Sdim LLVM_DEBUG(dbgs() << "\n"); 6411280031Sdim } 6412280031Sdim 6413280031Sdim // Now we replace the original zero extend and its associated INSERT_SUBREG 6414280031Sdim // with the value feeding the INSERT_SUBREG (which has now been promoted to 6415280031Sdim // return an i64). 6416280031Sdim 6417341825Sdim LLVM_DEBUG(dbgs() << "PPC64 ZExt Peephole replacing:\nOld: "); 6418341825Sdim LLVM_DEBUG(N->dump(CurDAG)); 6419341825Sdim LLVM_DEBUG(dbgs() << "\nNew: "); 6420341825Sdim LLVM_DEBUG(Op32.getNode()->dump(CurDAG)); 6421341825Sdim LLVM_DEBUG(dbgs() << "\n"); 6422280031Sdim 6423280031Sdim ReplaceUses(N, Op32.getNode()); 6424280031Sdim } 6425280031Sdim 6426280031Sdim if (MadeChange) 6427280031Sdim CurDAG->RemoveDeadNodes(); 6428280031Sdim} 6429280031Sdim 6430276479Sdimvoid PPCDAGToDAGISel::PeepholePPC64() { 6431249423Sdim // These optimizations are currently supported only for 64-bit SVR4. 6432276479Sdim if (PPCSubTarget->isDarwin() || !PPCSubTarget->isPPC64()) 6433249423Sdim return; 6434249423Sdim 6435341825Sdim SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); 6436249423Sdim 6437249423Sdim while (Position != CurDAG->allnodes_begin()) { 6438296417Sdim SDNode *N = &*--Position; 6439249423Sdim // Skip dead nodes and any non-machine opcodes. 6440249423Sdim if (N->use_empty() || !N->isMachineOpcode()) 6441249423Sdim continue; 6442249423Sdim 6443249423Sdim unsigned FirstOp; 6444249423Sdim unsigned StorageOpcode = N->getMachineOpcode(); 6445341825Sdim bool RequiresMod4Offset = false; 6446249423Sdim 6447249423Sdim switch (StorageOpcode) { 6448249423Sdim default: continue; 6449249423Sdim 6450341825Sdim case PPC::LWA: 6451341825Sdim case PPC::LD: 6452341825Sdim case PPC::DFLOADf64: 6453341825Sdim case PPC::DFLOADf32: 6454341825Sdim RequiresMod4Offset = true; 6455341825Sdim LLVM_FALLTHROUGH; 6456249423Sdim case PPC::LBZ: 6457249423Sdim case PPC::LBZ8: 6458249423Sdim case PPC::LFD: 6459249423Sdim case PPC::LFS: 6460249423Sdim case PPC::LHA: 6461249423Sdim case PPC::LHA8: 6462249423Sdim case PPC::LHZ: 6463249423Sdim case PPC::LHZ8: 6464249423Sdim case PPC::LWZ: 6465249423Sdim case PPC::LWZ8: 6466249423Sdim FirstOp = 0; 6467249423Sdim break; 6468249423Sdim 6469341825Sdim case PPC::STD: 6470341825Sdim case PPC::DFSTOREf64: 6471341825Sdim case PPC::DFSTOREf32: 6472341825Sdim RequiresMod4Offset = true; 6473341825Sdim LLVM_FALLTHROUGH; 6474249423Sdim case PPC::STB: 6475249423Sdim case PPC::STB8: 6476249423Sdim case PPC::STFD: 6477249423Sdim case PPC::STFS: 6478249423Sdim case PPC::STH: 6479249423Sdim case PPC::STH8: 6480249423Sdim case PPC::STW: 6481249423Sdim case PPC::STW8: 6482249423Sdim FirstOp = 1; 6483249423Sdim break; 6484249423Sdim } 6485249423Sdim 6486296417Sdim // If this is a load or store with a zero offset, or within the alignment, 6487296417Sdim // we may be able to fold an add-immediate into the memory operation. 6488296417Sdim // The check against alignment is below, as it can't occur until we check 6489296417Sdim // the arguments to N 6490296417Sdim if (!isa<ConstantSDNode>(N->getOperand(FirstOp))) 6491249423Sdim continue; 6492249423Sdim 6493249423Sdim SDValue Base = N->getOperand(FirstOp + 1); 6494249423Sdim if (!Base.isMachineOpcode()) 6495249423Sdim continue; 6496249423Sdim 6497249423Sdim unsigned Flags = 0; 6498249423Sdim bool ReplaceFlags = true; 6499249423Sdim 6500249423Sdim // When the feeding operation is an add-immediate of some sort, 6501249423Sdim // determine whether we need to add relocation information to the 6502249423Sdim // target flags on the immediate operand when we fold it into the 6503249423Sdim // load instruction. 6504249423Sdim // 6505249423Sdim // For something like ADDItocL, the relocation information is 6506249423Sdim // inferred from the opcode; when we process it in the AsmPrinter, 6507249423Sdim // we add the necessary relocation there. A load, though, can receive 6508249423Sdim // relocation from various flavors of ADDIxxx, so we need to carry 6509249423Sdim // the relocation information in the target flags. 6510249423Sdim switch (Base.getMachineOpcode()) { 6511249423Sdim default: continue; 6512249423Sdim 6513249423Sdim case PPC::ADDI8: 6514249423Sdim case PPC::ADDI: 6515249423Sdim // In some cases (such as TLS) the relocation information 6516249423Sdim // is already in place on the operand, so copying the operand 6517249423Sdim // is sufficient. 6518249423Sdim ReplaceFlags = false; 6519249423Sdim // For these cases, the immediate may not be divisible by 4, in 6520249423Sdim // which case the fold is illegal for DS-form instructions. (The 6521249423Sdim // other cases provide aligned addresses and are always safe.) 6522341825Sdim if (RequiresMod4Offset && 6523249423Sdim (!isa<ConstantSDNode>(Base.getOperand(1)) || 6524249423Sdim Base.getConstantOperandVal(1) % 4 != 0)) 6525249423Sdim continue; 6526249423Sdim break; 6527249423Sdim case PPC::ADDIdtprelL: 6528261991Sdim Flags = PPCII::MO_DTPREL_LO; 6529249423Sdim break; 6530249423Sdim case PPC::ADDItlsldL: 6531261991Sdim Flags = PPCII::MO_TLSLD_LO; 6532249423Sdim break; 6533249423Sdim case PPC::ADDItocL: 6534261991Sdim Flags = PPCII::MO_TOC_LO; 6535249423Sdim break; 6536249423Sdim } 6537249423Sdim 6538296417Sdim SDValue ImmOpnd = Base.getOperand(1); 6539314564Sdim 6540314564Sdim // On PPC64, the TOC base pointer is guaranteed by the ABI only to have 6541314564Sdim // 8-byte alignment, and so we can only use offsets less than 8 (otherwise, 6542314564Sdim // we might have needed different @ha relocation values for the offset 6543314564Sdim // pointers). 6544314564Sdim int MaxDisplacement = 7; 6545296417Sdim if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(ImmOpnd)) { 6546296417Sdim const GlobalValue *GV = GA->getGlobal(); 6547314564Sdim MaxDisplacement = std::min((int) GV->getAlignment() - 1, MaxDisplacement); 6548296417Sdim } 6549296417Sdim 6550314564Sdim bool UpdateHBase = false; 6551314564Sdim SDValue HBase = Base.getOperand(0); 6552314564Sdim 6553296417Sdim int Offset = N->getConstantOperandVal(FirstOp); 6554314564Sdim if (ReplaceFlags) { 6555314564Sdim if (Offset < 0 || Offset > MaxDisplacement) { 6556314564Sdim // If we have a addi(toc@l)/addis(toc@ha) pair, and the addis has only 6557314564Sdim // one use, then we can do this for any offset, we just need to also 6558314564Sdim // update the offset (i.e. the symbol addend) on the addis also. 6559314564Sdim if (Base.getMachineOpcode() != PPC::ADDItocL) 6560314564Sdim continue; 6561296417Sdim 6562314564Sdim if (!HBase.isMachineOpcode() || 6563360784Sdim HBase.getMachineOpcode() != PPC::ADDIStocHA8) 6564314564Sdim continue; 6565314564Sdim 6566314564Sdim if (!Base.hasOneUse() || !HBase.hasOneUse()) 6567314564Sdim continue; 6568314564Sdim 6569314564Sdim SDValue HImmOpnd = HBase.getOperand(1); 6570314564Sdim if (HImmOpnd != ImmOpnd) 6571314564Sdim continue; 6572314564Sdim 6573314564Sdim UpdateHBase = true; 6574314564Sdim } 6575314564Sdim } else { 6576314564Sdim // If we're directly folding the addend from an addi instruction, then: 6577314564Sdim // 1. In general, the offset on the memory access must be zero. 6578314564Sdim // 2. If the addend is a constant, then it can be combined with a 6579314564Sdim // non-zero offset, but only if the result meets the encoding 6580314564Sdim // requirements. 6581314564Sdim if (auto *C = dyn_cast<ConstantSDNode>(ImmOpnd)) { 6582314564Sdim Offset += C->getSExtValue(); 6583314564Sdim 6584341825Sdim if (RequiresMod4Offset && (Offset % 4) != 0) 6585314564Sdim continue; 6586314564Sdim 6587314564Sdim if (!isInt<16>(Offset)) 6588314564Sdim continue; 6589314564Sdim 6590314564Sdim ImmOpnd = CurDAG->getTargetConstant(Offset, SDLoc(ImmOpnd), 6591314564Sdim ImmOpnd.getValueType()); 6592314564Sdim } else if (Offset != 0) { 6593314564Sdim continue; 6594314564Sdim } 6595314564Sdim } 6596314564Sdim 6597249423Sdim // We found an opportunity. Reverse the operands from the add 6598249423Sdim // immediate and substitute them into the load or store. If 6599249423Sdim // needed, update the target flags for the immediate operand to 6600249423Sdim // reflect the necessary relocation information. 6601341825Sdim LLVM_DEBUG(dbgs() << "Folding add-immediate into mem-op:\nBase: "); 6602341825Sdim LLVM_DEBUG(Base->dump(CurDAG)); 6603341825Sdim LLVM_DEBUG(dbgs() << "\nN: "); 6604341825Sdim LLVM_DEBUG(N->dump(CurDAG)); 6605341825Sdim LLVM_DEBUG(dbgs() << "\n"); 6606249423Sdim 6607249423Sdim // If the relocation information isn't already present on the 6608249423Sdim // immediate operand, add it now. 6609249423Sdim if (ReplaceFlags) { 6610249423Sdim if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(ImmOpnd)) { 6611261991Sdim SDLoc dl(GA); 6612249423Sdim const GlobalValue *GV = GA->getGlobal(); 6613261991Sdim // We can't perform this optimization for data whose alignment 6614261991Sdim // is insufficient for the instruction encoding. 6615261991Sdim if (GV->getAlignment() < 4 && 6616341825Sdim (RequiresMod4Offset || (Offset % 4) != 0)) { 6617341825Sdim LLVM_DEBUG(dbgs() << "Rejected this candidate for alignment.\n\n"); 6618261991Sdim continue; 6619261991Sdim } 6620296417Sdim ImmOpnd = CurDAG->getTargetGlobalAddress(GV, dl, MVT::i64, Offset, Flags); 6621249423Sdim } else if (ConstantPoolSDNode *CP = 6622249423Sdim dyn_cast<ConstantPoolSDNode>(ImmOpnd)) { 6623249423Sdim const Constant *C = CP->getConstVal(); 6624249423Sdim ImmOpnd = CurDAG->getTargetConstantPool(C, MVT::i64, 6625249423Sdim CP->getAlignment(), 6626296417Sdim Offset, Flags); 6627249423Sdim } 6628249423Sdim } 6629249423Sdim 6630249423Sdim if (FirstOp == 1) // Store 6631249423Sdim (void)CurDAG->UpdateNodeOperands(N, N->getOperand(0), ImmOpnd, 6632249423Sdim Base.getOperand(0), N->getOperand(3)); 6633249423Sdim else // Load 6634249423Sdim (void)CurDAG->UpdateNodeOperands(N, ImmOpnd, Base.getOperand(0), 6635249423Sdim N->getOperand(2)); 6636249423Sdim 6637314564Sdim if (UpdateHBase) 6638314564Sdim (void)CurDAG->UpdateNodeOperands(HBase.getNode(), HBase.getOperand(0), 6639314564Sdim ImmOpnd); 6640314564Sdim 6641249423Sdim // The add-immediate may now be dead, in which case remove it. 6642249423Sdim if (Base.getNode()->use_empty()) 6643249423Sdim CurDAG->RemoveDeadNode(Base.getNode()); 6644249423Sdim } 6645249423Sdim} 6646249423Sdim 6647218893Sdim/// createPPCISelDag - This pass converts a legalized DAG into a 6648193323Sed/// PowerPC-specific DAG, ready for instruction scheduling. 6649193323Sed/// 6650321369SdimFunctionPass *llvm::createPPCISelDag(PPCTargetMachine &TM, 6651321369Sdim CodeGenOpt::Level OptLevel) { 6652321369Sdim return new PPCDAGToDAGISel(TM, OptLevel); 6653193323Sed} 6654