1327952Sdim//===- AArch6464FastISel.cpp - AArch64 FastISel implementation ------------===// 2274955Ssvnmir// 3353358Sdim// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4353358Sdim// See https://llvm.org/LICENSE.txt for license information. 5353358Sdim// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6274955Ssvnmir// 7274955Ssvnmir//===----------------------------------------------------------------------===// 8274955Ssvnmir// 9274955Ssvnmir// This file defines the AArch64-specific support for the FastISel class. Some 10274955Ssvnmir// of the target-specific code is generated by tablegen in the file 11274955Ssvnmir// AArch64GenFastISel.inc, which is #included here. 12274955Ssvnmir// 13274955Ssvnmir//===----------------------------------------------------------------------===// 14274955Ssvnmir 15274955Ssvnmir#include "AArch64.h" 16280031Sdim#include "AArch64CallingConvention.h" 17321369Sdim#include "AArch64RegisterInfo.h" 18280031Sdim#include "AArch64Subtarget.h" 19274955Ssvnmir#include "MCTargetDesc/AArch64AddressingModes.h" 20321369Sdim#include "Utils/AArch64BaseInfo.h" 21321369Sdim#include "llvm/ADT/APFloat.h" 22321369Sdim#include "llvm/ADT/APInt.h" 23321369Sdim#include "llvm/ADT/DenseMap.h" 24321369Sdim#include "llvm/ADT/SmallVector.h" 25280031Sdim#include "llvm/Analysis/BranchProbabilityInfo.h" 26274955Ssvnmir#include "llvm/CodeGen/CallingConvLower.h" 27274955Ssvnmir#include "llvm/CodeGen/FastISel.h" 28274955Ssvnmir#include "llvm/CodeGen/FunctionLoweringInfo.h" 29321369Sdim#include "llvm/CodeGen/ISDOpcodes.h" 30321369Sdim#include "llvm/CodeGen/MachineBasicBlock.h" 31274955Ssvnmir#include "llvm/CodeGen/MachineConstantPool.h" 32274955Ssvnmir#include "llvm/CodeGen/MachineFrameInfo.h" 33321369Sdim#include "llvm/CodeGen/MachineInstr.h" 34274955Ssvnmir#include "llvm/CodeGen/MachineInstrBuilder.h" 35321369Sdim#include "llvm/CodeGen/MachineMemOperand.h" 36274955Ssvnmir#include "llvm/CodeGen/MachineRegisterInfo.h" 37321369Sdim#include "llvm/CodeGen/RuntimeLibcalls.h" 38321369Sdim#include "llvm/CodeGen/ValueTypes.h" 39321369Sdim#include "llvm/IR/Argument.h" 40321369Sdim#include "llvm/IR/Attributes.h" 41321369Sdim#include "llvm/IR/BasicBlock.h" 42274955Ssvnmir#include "llvm/IR/CallingConv.h" 43321369Sdim#include "llvm/IR/Constant.h" 44321369Sdim#include "llvm/IR/Constants.h" 45274955Ssvnmir#include "llvm/IR/DataLayout.h" 46274955Ssvnmir#include "llvm/IR/DerivedTypes.h" 47274955Ssvnmir#include "llvm/IR/Function.h" 48274955Ssvnmir#include "llvm/IR/GetElementPtrTypeIterator.h" 49321369Sdim#include "llvm/IR/GlobalValue.h" 50321369Sdim#include "llvm/IR/InstrTypes.h" 51321369Sdim#include "llvm/IR/Instruction.h" 52274955Ssvnmir#include "llvm/IR/Instructions.h" 53274955Ssvnmir#include "llvm/IR/IntrinsicInst.h" 54327952Sdim#include "llvm/IR/Intrinsics.h" 55274955Ssvnmir#include "llvm/IR/Operator.h" 56321369Sdim#include "llvm/IR/Type.h" 57321369Sdim#include "llvm/IR/User.h" 58321369Sdim#include "llvm/IR/Value.h" 59321369Sdim#include "llvm/MC/MCInstrDesc.h" 60321369Sdim#include "llvm/MC/MCRegisterInfo.h" 61288943Sdim#include "llvm/MC/MCSymbol.h" 62321369Sdim#include "llvm/Support/AtomicOrdering.h" 63321369Sdim#include "llvm/Support/Casting.h" 64321369Sdim#include "llvm/Support/CodeGen.h" 65327952Sdim#include "llvm/Support/Compiler.h" 66321369Sdim#include "llvm/Support/ErrorHandling.h" 67341825Sdim#include "llvm/Support/MachineValueType.h" 68321369Sdim#include "llvm/Support/MathExtras.h" 69321369Sdim#include <algorithm> 70321369Sdim#include <cassert> 71321369Sdim#include <cstdint> 72321369Sdim#include <iterator> 73321369Sdim#include <utility> 74321369Sdim 75274955Ssvnmirusing namespace llvm; 76274955Ssvnmir 77274955Ssvnmirnamespace { 78274955Ssvnmir 79280031Sdimclass AArch64FastISel final : public FastISel { 80274955Ssvnmir class Address { 81274955Ssvnmir public: 82327952Sdim using BaseKind = enum { 83274955Ssvnmir RegBase, 84274955Ssvnmir FrameIndexBase 85327952Sdim }; 86274955Ssvnmir 87274955Ssvnmir private: 88321369Sdim BaseKind Kind = RegBase; 89321369Sdim AArch64_AM::ShiftExtendType ExtType = AArch64_AM::InvalidShiftExtend; 90274955Ssvnmir union { 91274955Ssvnmir unsigned Reg; 92274955Ssvnmir int FI; 93274955Ssvnmir } Base; 94321369Sdim unsigned OffsetReg = 0; 95321369Sdim unsigned Shift = 0; 96321369Sdim int64_t Offset = 0; 97321369Sdim const GlobalValue *GV = nullptr; 98274955Ssvnmir 99274955Ssvnmir public: 100321369Sdim Address() { Base.Reg = 0; } 101321369Sdim 102274955Ssvnmir void setKind(BaseKind K) { Kind = K; } 103274955Ssvnmir BaseKind getKind() const { return Kind; } 104280031Sdim void setExtendType(AArch64_AM::ShiftExtendType E) { ExtType = E; } 105280031Sdim AArch64_AM::ShiftExtendType getExtendType() const { return ExtType; } 106274955Ssvnmir bool isRegBase() const { return Kind == RegBase; } 107274955Ssvnmir bool isFIBase() const { return Kind == FrameIndexBase; } 108321369Sdim 109274955Ssvnmir void setReg(unsigned Reg) { 110274955Ssvnmir assert(isRegBase() && "Invalid base register access!"); 111274955Ssvnmir Base.Reg = Reg; 112274955Ssvnmir } 113321369Sdim 114274955Ssvnmir unsigned getReg() const { 115274955Ssvnmir assert(isRegBase() && "Invalid base register access!"); 116274955Ssvnmir return Base.Reg; 117274955Ssvnmir } 118321369Sdim 119280031Sdim void setOffsetReg(unsigned Reg) { 120280031Sdim OffsetReg = Reg; 121280031Sdim } 122321369Sdim 123280031Sdim unsigned getOffsetReg() const { 124280031Sdim return OffsetReg; 125280031Sdim } 126321369Sdim 127274955Ssvnmir void setFI(unsigned FI) { 128274955Ssvnmir assert(isFIBase() && "Invalid base frame index access!"); 129274955Ssvnmir Base.FI = FI; 130274955Ssvnmir } 131321369Sdim 132274955Ssvnmir unsigned getFI() const { 133274955Ssvnmir assert(isFIBase() && "Invalid base frame index access!"); 134274955Ssvnmir return Base.FI; 135274955Ssvnmir } 136321369Sdim 137274955Ssvnmir void setOffset(int64_t O) { Offset = O; } 138274955Ssvnmir int64_t getOffset() { return Offset; } 139280031Sdim void setShift(unsigned S) { Shift = S; } 140280031Sdim unsigned getShift() { return Shift; } 141274955Ssvnmir 142280031Sdim void setGlobalValue(const GlobalValue *G) { GV = G; } 143280031Sdim const GlobalValue *getGlobalValue() { return GV; } 144274955Ssvnmir }; 145274955Ssvnmir 146274955Ssvnmir /// Subtarget - Keep a pointer to the AArch64Subtarget around so that we can 147274955Ssvnmir /// make the right decision when generating code for different targets. 148274955Ssvnmir const AArch64Subtarget *Subtarget; 149274955Ssvnmir LLVMContext *Context; 150274955Ssvnmir 151280031Sdim bool fastLowerArguments() override; 152280031Sdim bool fastLowerCall(CallLoweringInfo &CLI) override; 153280031Sdim bool fastLowerIntrinsicCall(const IntrinsicInst *II) override; 154280031Sdim 155274955Ssvnmirprivate: 156274955Ssvnmir // Selection routines. 157280031Sdim bool selectAddSub(const Instruction *I); 158280031Sdim bool selectLogicalOp(const Instruction *I); 159280031Sdim bool selectLoad(const Instruction *I); 160280031Sdim bool selectStore(const Instruction *I); 161280031Sdim bool selectBranch(const Instruction *I); 162280031Sdim bool selectIndirectBr(const Instruction *I); 163280031Sdim bool selectCmp(const Instruction *I); 164280031Sdim bool selectSelect(const Instruction *I); 165280031Sdim bool selectFPExt(const Instruction *I); 166280031Sdim bool selectFPTrunc(const Instruction *I); 167280031Sdim bool selectFPToInt(const Instruction *I, bool Signed); 168280031Sdim bool selectIntToFP(const Instruction *I, bool Signed); 169280031Sdim bool selectRem(const Instruction *I, unsigned ISDOpcode); 170280031Sdim bool selectRet(const Instruction *I); 171280031Sdim bool selectTrunc(const Instruction *I); 172280031Sdim bool selectIntExt(const Instruction *I); 173280031Sdim bool selectMul(const Instruction *I); 174280031Sdim bool selectShift(const Instruction *I); 175280031Sdim bool selectBitCast(const Instruction *I); 176280031Sdim bool selectFRem(const Instruction *I); 177280031Sdim bool selectSDiv(const Instruction *I); 178280031Sdim bool selectGetElementPtr(const Instruction *I); 179314564Sdim bool selectAtomicCmpXchg(const AtomicCmpXchgInst *I); 180274955Ssvnmir 181274955Ssvnmir // Utility helper routines. 182274955Ssvnmir bool isTypeLegal(Type *Ty, MVT &VT); 183280031Sdim bool isTypeSupported(Type *Ty, MVT &VT, bool IsVectorAllowed = false); 184280031Sdim bool isValueAvailable(const Value *V) const; 185280031Sdim bool computeAddress(const Value *Obj, Address &Addr, Type *Ty = nullptr); 186280031Sdim bool computeCallAddress(const Value *V, Address &Addr); 187280031Sdim bool simplifyAddress(Address &Addr, MVT VT); 188280031Sdim void addLoadStoreOperands(Address &Addr, const MachineInstrBuilder &MIB, 189309124Sdim MachineMemOperand::Flags Flags, 190309124Sdim unsigned ScaleFactor, MachineMemOperand *MMO); 191280031Sdim bool isMemCpySmall(uint64_t Len, unsigned Alignment); 192280031Sdim bool tryEmitSmallMemCpy(Address Dest, Address Src, uint64_t Len, 193274955Ssvnmir unsigned Alignment); 194280031Sdim bool foldXALUIntrinsic(AArch64CC::CondCode &CC, const Instruction *I, 195280031Sdim const Value *Cond); 196280031Sdim bool optimizeIntExtLoad(const Instruction *I, MVT RetVT, MVT SrcVT); 197280031Sdim bool optimizeSelect(const SelectInst *SI); 198280031Sdim std::pair<unsigned, bool> getRegForGEPIndex(const Value *Idx); 199280031Sdim 200280031Sdim // Emit helper routines. 201280031Sdim unsigned emitAddSub(bool UseAdd, MVT RetVT, const Value *LHS, 202280031Sdim const Value *RHS, bool SetFlags = false, 203280031Sdim bool WantResult = true, bool IsZExt = false); 204280031Sdim unsigned emitAddSub_rr(bool UseAdd, MVT RetVT, unsigned LHSReg, 205280031Sdim bool LHSIsKill, unsigned RHSReg, bool RHSIsKill, 206280031Sdim bool SetFlags = false, bool WantResult = true); 207280031Sdim unsigned emitAddSub_ri(bool UseAdd, MVT RetVT, unsigned LHSReg, 208280031Sdim bool LHSIsKill, uint64_t Imm, bool SetFlags = false, 209280031Sdim bool WantResult = true); 210280031Sdim unsigned emitAddSub_rs(bool UseAdd, MVT RetVT, unsigned LHSReg, 211280031Sdim bool LHSIsKill, unsigned RHSReg, bool RHSIsKill, 212280031Sdim AArch64_AM::ShiftExtendType ShiftType, 213280031Sdim uint64_t ShiftImm, bool SetFlags = false, 214280031Sdim bool WantResult = true); 215280031Sdim unsigned emitAddSub_rx(bool UseAdd, MVT RetVT, unsigned LHSReg, 216280031Sdim bool LHSIsKill, unsigned RHSReg, bool RHSIsKill, 217280031Sdim AArch64_AM::ShiftExtendType ExtType, 218280031Sdim uint64_t ShiftImm, bool SetFlags = false, 219280031Sdim bool WantResult = true); 220280031Sdim 221274955Ssvnmir // Emit functions. 222280031Sdim bool emitCompareAndBranch(const BranchInst *BI); 223280031Sdim bool emitCmp(const Value *LHS, const Value *RHS, bool IsZExt); 224280031Sdim bool emitICmp(MVT RetVT, const Value *LHS, const Value *RHS, bool IsZExt); 225280031Sdim bool emitICmp_ri(MVT RetVT, unsigned LHSReg, bool LHSIsKill, uint64_t Imm); 226280031Sdim bool emitFCmp(MVT RetVT, const Value *LHS, const Value *RHS); 227280031Sdim unsigned emitLoad(MVT VT, MVT ResultVT, Address Addr, bool WantZExt = true, 228280031Sdim MachineMemOperand *MMO = nullptr); 229280031Sdim bool emitStore(MVT VT, unsigned SrcReg, Address Addr, 230280031Sdim MachineMemOperand *MMO = nullptr); 231314564Sdim bool emitStoreRelease(MVT VT, unsigned SrcReg, unsigned AddrReg, 232314564Sdim MachineMemOperand *MMO = nullptr); 233280031Sdim unsigned emitIntExt(MVT SrcVT, unsigned SrcReg, MVT DestVT, bool isZExt); 234280031Sdim unsigned emiti1Ext(unsigned SrcReg, MVT DestVT, bool isZExt); 235280031Sdim unsigned emitAdd(MVT RetVT, const Value *LHS, const Value *RHS, 236280031Sdim bool SetFlags = false, bool WantResult = true, 237280031Sdim bool IsZExt = false); 238280031Sdim unsigned emitAdd_ri_(MVT VT, unsigned Op0, bool Op0IsKill, int64_t Imm); 239280031Sdim unsigned emitSub(MVT RetVT, const Value *LHS, const Value *RHS, 240280031Sdim bool SetFlags = false, bool WantResult = true, 241280031Sdim bool IsZExt = false); 242280031Sdim unsigned emitSubs_rr(MVT RetVT, unsigned LHSReg, bool LHSIsKill, 243280031Sdim unsigned RHSReg, bool RHSIsKill, bool WantResult = true); 244280031Sdim unsigned emitSubs_rs(MVT RetVT, unsigned LHSReg, bool LHSIsKill, 245280031Sdim unsigned RHSReg, bool RHSIsKill, 246280031Sdim AArch64_AM::ShiftExtendType ShiftType, uint64_t ShiftImm, 247280031Sdim bool WantResult = true); 248280031Sdim unsigned emitLogicalOp(unsigned ISDOpc, MVT RetVT, const Value *LHS, 249280031Sdim const Value *RHS); 250280031Sdim unsigned emitLogicalOp_ri(unsigned ISDOpc, MVT RetVT, unsigned LHSReg, 251280031Sdim bool LHSIsKill, uint64_t Imm); 252280031Sdim unsigned emitLogicalOp_rs(unsigned ISDOpc, MVT RetVT, unsigned LHSReg, 253280031Sdim bool LHSIsKill, unsigned RHSReg, bool RHSIsKill, 254280031Sdim uint64_t ShiftImm); 255280031Sdim unsigned emitAnd_ri(MVT RetVT, unsigned LHSReg, bool LHSIsKill, uint64_t Imm); 256280031Sdim unsigned emitMul_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, 257280031Sdim unsigned Op1, bool Op1IsKill); 258280031Sdim unsigned emitSMULL_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, 259280031Sdim unsigned Op1, bool Op1IsKill); 260280031Sdim unsigned emitUMULL_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, 261280031Sdim unsigned Op1, bool Op1IsKill); 262280031Sdim unsigned emitLSL_rr(MVT RetVT, unsigned Op0Reg, bool Op0IsKill, 263280031Sdim unsigned Op1Reg, bool Op1IsKill); 264280031Sdim unsigned emitLSL_ri(MVT RetVT, MVT SrcVT, unsigned Op0Reg, bool Op0IsKill, 265280031Sdim uint64_t Imm, bool IsZExt = true); 266280031Sdim unsigned emitLSR_rr(MVT RetVT, unsigned Op0Reg, bool Op0IsKill, 267280031Sdim unsigned Op1Reg, bool Op1IsKill); 268280031Sdim unsigned emitLSR_ri(MVT RetVT, MVT SrcVT, unsigned Op0Reg, bool Op0IsKill, 269280031Sdim uint64_t Imm, bool IsZExt = true); 270280031Sdim unsigned emitASR_rr(MVT RetVT, unsigned Op0Reg, bool Op0IsKill, 271280031Sdim unsigned Op1Reg, bool Op1IsKill); 272280031Sdim unsigned emitASR_ri(MVT RetVT, MVT SrcVT, unsigned Op0Reg, bool Op0IsKill, 273280031Sdim uint64_t Imm, bool IsZExt = false); 274274955Ssvnmir 275280031Sdim unsigned materializeInt(const ConstantInt *CI, MVT VT); 276280031Sdim unsigned materializeFP(const ConstantFP *CFP, MVT VT); 277280031Sdim unsigned materializeGV(const GlobalValue *GV); 278274955Ssvnmir 279274955Ssvnmir // Call handling routines. 280274955Ssvnmirprivate: 281274955Ssvnmir CCAssignFn *CCAssignFnForCall(CallingConv::ID CC) const; 282280031Sdim bool processCallArgs(CallLoweringInfo &CLI, SmallVectorImpl<MVT> &ArgVTs, 283274955Ssvnmir unsigned &NumBytes); 284280031Sdim bool finishCall(CallLoweringInfo &CLI, MVT RetVT, unsigned NumBytes); 285274955Ssvnmir 286274955Ssvnmirpublic: 287274955Ssvnmir // Backend specific FastISel code. 288280031Sdim unsigned fastMaterializeAlloca(const AllocaInst *AI) override; 289280031Sdim unsigned fastMaterializeConstant(const Constant *C) override; 290280031Sdim unsigned fastMaterializeFloatZero(const ConstantFP* CF) override; 291274955Ssvnmir 292280031Sdim explicit AArch64FastISel(FunctionLoweringInfo &FuncInfo, 293288943Sdim const TargetLibraryInfo *LibInfo) 294280031Sdim : FastISel(FuncInfo, LibInfo, /*SkipTargetIndependentISel=*/true) { 295288943Sdim Subtarget = 296288943Sdim &static_cast<const AArch64Subtarget &>(FuncInfo.MF->getSubtarget()); 297280031Sdim Context = &FuncInfo.Fn->getContext(); 298274955Ssvnmir } 299274955Ssvnmir 300280031Sdim bool fastSelectInstruction(const Instruction *I) override; 301274955Ssvnmir 302274955Ssvnmir#include "AArch64GenFastISel.inc" 303274955Ssvnmir}; 304274955Ssvnmir 305274955Ssvnmir} // end anonymous namespace 306274955Ssvnmir 307341825Sdim/// Check if the sign-/zero-extend will be a noop. 308280031Sdimstatic bool isIntExtFree(const Instruction *I) { 309280031Sdim assert((isa<ZExtInst>(I) || isa<SExtInst>(I)) && 310280031Sdim "Unexpected integer extend instruction."); 311280031Sdim assert(!I->getType()->isVectorTy() && I->getType()->isIntegerTy() && 312280031Sdim "Unexpected value type."); 313280031Sdim bool IsZExt = isa<ZExtInst>(I); 314280031Sdim 315280031Sdim if (const auto *LI = dyn_cast<LoadInst>(I->getOperand(0))) 316280031Sdim if (LI->hasOneUse()) 317280031Sdim return true; 318280031Sdim 319280031Sdim if (const auto *Arg = dyn_cast<Argument>(I->getOperand(0))) 320280031Sdim if ((IsZExt && Arg->hasZExtAttr()) || (!IsZExt && Arg->hasSExtAttr())) 321280031Sdim return true; 322280031Sdim 323280031Sdim return false; 324280031Sdim} 325280031Sdim 326341825Sdim/// Determine the implicit scale factor that is applied by a memory 327280031Sdim/// operation for a given value type. 328280031Sdimstatic unsigned getImplicitScaleFactor(MVT VT) { 329280031Sdim switch (VT.SimpleTy) { 330280031Sdim default: 331280031Sdim return 0; // invalid 332280031Sdim case MVT::i1: // fall-through 333280031Sdim case MVT::i8: 334280031Sdim return 1; 335280031Sdim case MVT::i16: 336280031Sdim return 2; 337280031Sdim case MVT::i32: // fall-through 338280031Sdim case MVT::f32: 339280031Sdim return 4; 340280031Sdim case MVT::i64: // fall-through 341280031Sdim case MVT::f64: 342280031Sdim return 8; 343280031Sdim } 344280031Sdim} 345280031Sdim 346274955SsvnmirCCAssignFn *AArch64FastISel::CCAssignFnForCall(CallingConv::ID CC) const { 347274955Ssvnmir if (CC == CallingConv::WebKit_JS) 348274955Ssvnmir return CC_AArch64_WebKit_JS; 349280031Sdim if (CC == CallingConv::GHC) 350280031Sdim return CC_AArch64_GHC; 351360784Sdim if (CC == CallingConv::CFGuard_Check) 352360784Sdim return CC_AArch64_Win64_CFGuard_Check; 353274955Ssvnmir return Subtarget->isTargetDarwin() ? CC_AArch64_DarwinPCS : CC_AArch64_AAPCS; 354274955Ssvnmir} 355274955Ssvnmir 356280031Sdimunsigned AArch64FastISel::fastMaterializeAlloca(const AllocaInst *AI) { 357288943Sdim assert(TLI.getValueType(DL, AI->getType(), true) == MVT::i64 && 358274955Ssvnmir "Alloca should always return a pointer."); 359274955Ssvnmir 360274955Ssvnmir // Don't handle dynamic allocas. 361274955Ssvnmir if (!FuncInfo.StaticAllocaMap.count(AI)) 362274955Ssvnmir return 0; 363274955Ssvnmir 364274955Ssvnmir DenseMap<const AllocaInst *, int>::iterator SI = 365274955Ssvnmir FuncInfo.StaticAllocaMap.find(AI); 366274955Ssvnmir 367274955Ssvnmir if (SI != FuncInfo.StaticAllocaMap.end()) { 368280031Sdim unsigned ResultReg = createResultReg(&AArch64::GPR64spRegClass); 369274955Ssvnmir BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::ADDXri), 370274955Ssvnmir ResultReg) 371274955Ssvnmir .addFrameIndex(SI->second) 372274955Ssvnmir .addImm(0) 373274955Ssvnmir .addImm(0); 374274955Ssvnmir return ResultReg; 375274955Ssvnmir } 376274955Ssvnmir 377274955Ssvnmir return 0; 378274955Ssvnmir} 379274955Ssvnmir 380280031Sdimunsigned AArch64FastISel::materializeInt(const ConstantInt *CI, MVT VT) { 381280031Sdim if (VT > MVT::i64) 382280031Sdim return 0; 383280031Sdim 384280031Sdim if (!CI->isZero()) 385280031Sdim return fastEmit_i(VT, VT, ISD::Constant, CI->getZExtValue()); 386280031Sdim 387280031Sdim // Create a copy from the zero register to materialize a "0" value. 388280031Sdim const TargetRegisterClass *RC = (VT == MVT::i64) ? &AArch64::GPR64RegClass 389280031Sdim : &AArch64::GPR32RegClass; 390280031Sdim unsigned ZeroReg = (VT == MVT::i64) ? AArch64::XZR : AArch64::WZR; 391280031Sdim unsigned ResultReg = createResultReg(RC); 392280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), 393280031Sdim ResultReg).addReg(ZeroReg, getKillRegState(true)); 394280031Sdim return ResultReg; 395280031Sdim} 396280031Sdim 397280031Sdimunsigned AArch64FastISel::materializeFP(const ConstantFP *CFP, MVT VT) { 398280031Sdim // Positive zero (+0.0) has to be materialized with a fmov from the zero 399280031Sdim // register, because the immediate version of fmov cannot encode zero. 400280031Sdim if (CFP->isNullValue()) 401280031Sdim return fastMaterializeFloatZero(CFP); 402280031Sdim 403274955Ssvnmir if (VT != MVT::f32 && VT != MVT::f64) 404274955Ssvnmir return 0; 405274955Ssvnmir 406274955Ssvnmir const APFloat Val = CFP->getValueAPF(); 407280031Sdim bool Is64Bit = (VT == MVT::f64); 408274955Ssvnmir // This checks to see if we can use FMOV instructions to materialize 409274955Ssvnmir // a constant, otherwise we have to materialize via the constant pool. 410353358Sdim int Imm = 411353358Sdim Is64Bit ? AArch64_AM::getFP64Imm(Val) : AArch64_AM::getFP32Imm(Val); 412353358Sdim if (Imm != -1) { 413280031Sdim unsigned Opc = Is64Bit ? AArch64::FMOVDi : AArch64::FMOVSi; 414280031Sdim return fastEmitInst_i(Opc, TLI.getRegClassFor(VT), Imm); 415280031Sdim } 416280031Sdim 417280031Sdim // For the MachO large code model materialize the FP constant in code. 418280031Sdim if (Subtarget->isTargetMachO() && TM.getCodeModel() == CodeModel::Large) { 419280031Sdim unsigned Opc1 = Is64Bit ? AArch64::MOVi64imm : AArch64::MOVi32imm; 420280031Sdim const TargetRegisterClass *RC = Is64Bit ? 421280031Sdim &AArch64::GPR64RegClass : &AArch64::GPR32RegClass; 422280031Sdim 423280031Sdim unsigned TmpReg = createResultReg(RC); 424280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc1), TmpReg) 425280031Sdim .addImm(CFP->getValueAPF().bitcastToAPInt().getZExtValue()); 426280031Sdim 427274955Ssvnmir unsigned ResultReg = createResultReg(TLI.getRegClassFor(VT)); 428280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, 429280031Sdim TII.get(TargetOpcode::COPY), ResultReg) 430280031Sdim .addReg(TmpReg, getKillRegState(true)); 431280031Sdim 432274955Ssvnmir return ResultReg; 433274955Ssvnmir } 434274955Ssvnmir 435274955Ssvnmir // Materialize via constant pool. MachineConstantPool wants an explicit 436274955Ssvnmir // alignment. 437274955Ssvnmir unsigned Align = DL.getPrefTypeAlignment(CFP->getType()); 438274955Ssvnmir if (Align == 0) 439274955Ssvnmir Align = DL.getTypeAllocSize(CFP->getType()); 440274955Ssvnmir 441280031Sdim unsigned CPI = MCP.getConstantPoolIndex(cast<Constant>(CFP), Align); 442274955Ssvnmir unsigned ADRPReg = createResultReg(&AArch64::GPR64commonRegClass); 443274955Ssvnmir BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::ADRP), 444280031Sdim ADRPReg).addConstantPoolIndex(CPI, 0, AArch64II::MO_PAGE); 445274955Ssvnmir 446280031Sdim unsigned Opc = Is64Bit ? AArch64::LDRDui : AArch64::LDRSui; 447274955Ssvnmir unsigned ResultReg = createResultReg(TLI.getRegClassFor(VT)); 448274955Ssvnmir BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg) 449274955Ssvnmir .addReg(ADRPReg) 450280031Sdim .addConstantPoolIndex(CPI, 0, AArch64II::MO_PAGEOFF | AArch64II::MO_NC); 451274955Ssvnmir return ResultReg; 452274955Ssvnmir} 453274955Ssvnmir 454280031Sdimunsigned AArch64FastISel::materializeGV(const GlobalValue *GV) { 455274955Ssvnmir // We can't handle thread-local variables quickly yet. 456274955Ssvnmir if (GV->isThreadLocal()) 457274955Ssvnmir return 0; 458274955Ssvnmir 459274955Ssvnmir // MachO still uses GOT for large code-model accesses, but ELF requires 460274955Ssvnmir // movz/movk sequences, which FastISel doesn't handle yet. 461321369Sdim if (!Subtarget->useSmallAddressing() && !Subtarget->isTargetMachO()) 462274955Ssvnmir return 0; 463274955Ssvnmir 464360784Sdim unsigned OpFlags = Subtarget->ClassifyGlobalReference(GV, TM); 465274955Ssvnmir 466288943Sdim EVT DestEVT = TLI.getValueType(DL, GV->getType(), true); 467274955Ssvnmir if (!DestEVT.isSimple()) 468274955Ssvnmir return 0; 469274955Ssvnmir 470274955Ssvnmir unsigned ADRPReg = createResultReg(&AArch64::GPR64commonRegClass); 471274955Ssvnmir unsigned ResultReg; 472274955Ssvnmir 473274955Ssvnmir if (OpFlags & AArch64II::MO_GOT) { 474274955Ssvnmir // ADRP + LDRX 475274955Ssvnmir BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::ADRP), 476274955Ssvnmir ADRPReg) 477328753Sdim .addGlobalAddress(GV, 0, AArch64II::MO_PAGE | OpFlags); 478274955Ssvnmir 479360784Sdim unsigned LdrOpc; 480360784Sdim if (Subtarget->isTargetILP32()) { 481360784Sdim ResultReg = createResultReg(&AArch64::GPR32RegClass); 482360784Sdim LdrOpc = AArch64::LDRWui; 483360784Sdim } else { 484360784Sdim ResultReg = createResultReg(&AArch64::GPR64RegClass); 485360784Sdim LdrOpc = AArch64::LDRXui; 486360784Sdim } 487360784Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(LdrOpc), 488274955Ssvnmir ResultReg) 489360784Sdim .addReg(ADRPReg) 490360784Sdim .addGlobalAddress(GV, 0, AArch64II::MO_GOT | AArch64II::MO_PAGEOFF | 491360784Sdim AArch64II::MO_NC | OpFlags); 492360784Sdim if (!Subtarget->isTargetILP32()) 493360784Sdim return ResultReg; 494360784Sdim 495360784Sdim // LDRWui produces a 32-bit register, but pointers in-register are 64-bits 496360784Sdim // so we must extend the result on ILP32. 497360784Sdim unsigned Result64 = createResultReg(&AArch64::GPR64RegClass); 498360784Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, 499360784Sdim TII.get(TargetOpcode::SUBREG_TO_REG)) 500360784Sdim .addDef(Result64) 501360784Sdim .addImm(0) 502360784Sdim .addReg(ResultReg, RegState::Kill) 503360784Sdim .addImm(AArch64::sub_32); 504360784Sdim return Result64; 505274955Ssvnmir } else { 506274955Ssvnmir // ADRP + ADDX 507274955Ssvnmir BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::ADRP), 508280031Sdim ADRPReg) 509328753Sdim .addGlobalAddress(GV, 0, AArch64II::MO_PAGE | OpFlags); 510274955Ssvnmir 511274955Ssvnmir ResultReg = createResultReg(&AArch64::GPR64spRegClass); 512274955Ssvnmir BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::ADDXri), 513274955Ssvnmir ResultReg) 514328753Sdim .addReg(ADRPReg) 515328753Sdim .addGlobalAddress(GV, 0, 516328753Sdim AArch64II::MO_PAGEOFF | AArch64II::MO_NC | OpFlags) 517328753Sdim .addImm(0); 518274955Ssvnmir } 519274955Ssvnmir return ResultReg; 520274955Ssvnmir} 521274955Ssvnmir 522280031Sdimunsigned AArch64FastISel::fastMaterializeConstant(const Constant *C) { 523288943Sdim EVT CEVT = TLI.getValueType(DL, C->getType(), true); 524274955Ssvnmir 525274955Ssvnmir // Only handle simple types. 526274955Ssvnmir if (!CEVT.isSimple()) 527274955Ssvnmir return 0; 528274955Ssvnmir MVT VT = CEVT.getSimpleVT(); 529360784Sdim // arm64_32 has 32-bit pointers held in 64-bit registers. Because of that, 530360784Sdim // 'null' pointers need to have a somewhat special treatment. 531360784Sdim if (const auto *CPN = dyn_cast<ConstantPointerNull>(C)) { 532360784Sdim (void)CPN; 533360784Sdim assert(CPN->getType()->getPointerAddressSpace() == 0 && 534360784Sdim "Unexpected address space"); 535360784Sdim assert(VT == MVT::i64 && "Expected 64-bit pointers"); 536360784Sdim return materializeInt(ConstantInt::get(Type::getInt64Ty(*Context), 0), VT); 537360784Sdim } 538274955Ssvnmir 539280031Sdim if (const auto *CI = dyn_cast<ConstantInt>(C)) 540280031Sdim return materializeInt(CI, VT); 541280031Sdim else if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C)) 542280031Sdim return materializeFP(CFP, VT); 543274955Ssvnmir else if (const GlobalValue *GV = dyn_cast<GlobalValue>(C)) 544280031Sdim return materializeGV(GV); 545274955Ssvnmir 546274955Ssvnmir return 0; 547274955Ssvnmir} 548274955Ssvnmir 549280031Sdimunsigned AArch64FastISel::fastMaterializeFloatZero(const ConstantFP* CFP) { 550280031Sdim assert(CFP->isNullValue() && 551280031Sdim "Floating-point constant is not a positive zero."); 552280031Sdim MVT VT; 553280031Sdim if (!isTypeLegal(CFP->getType(), VT)) 554280031Sdim return 0; 555280031Sdim 556280031Sdim if (VT != MVT::f32 && VT != MVT::f64) 557280031Sdim return 0; 558280031Sdim 559280031Sdim bool Is64Bit = (VT == MVT::f64); 560280031Sdim unsigned ZReg = Is64Bit ? AArch64::XZR : AArch64::WZR; 561280031Sdim unsigned Opc = Is64Bit ? AArch64::FMOVXDr : AArch64::FMOVWSr; 562280031Sdim return fastEmitInst_r(Opc, TLI.getRegClassFor(VT), ZReg, /*IsKill=*/true); 563280031Sdim} 564280031Sdim 565341825Sdim/// Check if the multiply is by a power-of-2 constant. 566280031Sdimstatic bool isMulPowOf2(const Value *I) { 567280031Sdim if (const auto *MI = dyn_cast<MulOperator>(I)) { 568280031Sdim if (const auto *C = dyn_cast<ConstantInt>(MI->getOperand(0))) 569280031Sdim if (C->getValue().isPowerOf2()) 570280031Sdim return true; 571280031Sdim if (const auto *C = dyn_cast<ConstantInt>(MI->getOperand(1))) 572280031Sdim if (C->getValue().isPowerOf2()) 573280031Sdim return true; 574280031Sdim } 575280031Sdim return false; 576280031Sdim} 577280031Sdim 578274955Ssvnmir// Computes the address to get to an object. 579280031Sdimbool AArch64FastISel::computeAddress(const Value *Obj, Address &Addr, Type *Ty) 580280031Sdim{ 581274955Ssvnmir const User *U = nullptr; 582274955Ssvnmir unsigned Opcode = Instruction::UserOp1; 583274955Ssvnmir if (const Instruction *I = dyn_cast<Instruction>(Obj)) { 584274955Ssvnmir // Don't walk into other basic blocks unless the object is an alloca from 585274955Ssvnmir // another block, otherwise it may not have a virtual register assigned. 586274955Ssvnmir if (FuncInfo.StaticAllocaMap.count(static_cast<const AllocaInst *>(Obj)) || 587274955Ssvnmir FuncInfo.MBBMap[I->getParent()] == FuncInfo.MBB) { 588274955Ssvnmir Opcode = I->getOpcode(); 589274955Ssvnmir U = I; 590274955Ssvnmir } 591274955Ssvnmir } else if (const ConstantExpr *C = dyn_cast<ConstantExpr>(Obj)) { 592274955Ssvnmir Opcode = C->getOpcode(); 593274955Ssvnmir U = C; 594274955Ssvnmir } 595274955Ssvnmir 596296417Sdim if (auto *Ty = dyn_cast<PointerType>(Obj->getType())) 597274955Ssvnmir if (Ty->getAddressSpace() > 255) 598274955Ssvnmir // Fast instruction selection doesn't support the special 599274955Ssvnmir // address spaces. 600274955Ssvnmir return false; 601274955Ssvnmir 602274955Ssvnmir switch (Opcode) { 603274955Ssvnmir default: 604274955Ssvnmir break; 605321369Sdim case Instruction::BitCast: 606274955Ssvnmir // Look through bitcasts. 607280031Sdim return computeAddress(U->getOperand(0), Addr, Ty); 608321369Sdim 609321369Sdim case Instruction::IntToPtr: 610274955Ssvnmir // Look past no-op inttoptrs. 611288943Sdim if (TLI.getValueType(DL, U->getOperand(0)->getType()) == 612288943Sdim TLI.getPointerTy(DL)) 613280031Sdim return computeAddress(U->getOperand(0), Addr, Ty); 614274955Ssvnmir break; 615321369Sdim 616321369Sdim case Instruction::PtrToInt: 617274955Ssvnmir // Look past no-op ptrtoints. 618288943Sdim if (TLI.getValueType(DL, U->getType()) == TLI.getPointerTy(DL)) 619280031Sdim return computeAddress(U->getOperand(0), Addr, Ty); 620274955Ssvnmir break; 621321369Sdim 622274955Ssvnmir case Instruction::GetElementPtr: { 623274955Ssvnmir Address SavedAddr = Addr; 624274955Ssvnmir uint64_t TmpOffset = Addr.getOffset(); 625274955Ssvnmir 626274955Ssvnmir // Iterate through the GEP folding the constants into offsets where 627274955Ssvnmir // we can. 628309124Sdim for (gep_type_iterator GTI = gep_type_begin(U), E = gep_type_end(U); 629309124Sdim GTI != E; ++GTI) { 630309124Sdim const Value *Op = GTI.getOperand(); 631314564Sdim if (StructType *STy = GTI.getStructTypeOrNull()) { 632274955Ssvnmir const StructLayout *SL = DL.getStructLayout(STy); 633274955Ssvnmir unsigned Idx = cast<ConstantInt>(Op)->getZExtValue(); 634274955Ssvnmir TmpOffset += SL->getElementOffset(Idx); 635274955Ssvnmir } else { 636274955Ssvnmir uint64_t S = DL.getTypeAllocSize(GTI.getIndexedType()); 637321369Sdim while (true) { 638274955Ssvnmir if (const ConstantInt *CI = dyn_cast<ConstantInt>(Op)) { 639274955Ssvnmir // Constant-offset addressing. 640274955Ssvnmir TmpOffset += CI->getSExtValue() * S; 641274955Ssvnmir break; 642274955Ssvnmir } 643274955Ssvnmir if (canFoldAddIntoGEP(U, Op)) { 644274955Ssvnmir // A compatible add with a constant operand. Fold the constant. 645274955Ssvnmir ConstantInt *CI = 646274955Ssvnmir cast<ConstantInt>(cast<AddOperator>(Op)->getOperand(1)); 647274955Ssvnmir TmpOffset += CI->getSExtValue() * S; 648274955Ssvnmir // Iterate on the other operand. 649274955Ssvnmir Op = cast<AddOperator>(Op)->getOperand(0); 650274955Ssvnmir continue; 651274955Ssvnmir } 652274955Ssvnmir // Unsupported 653274955Ssvnmir goto unsupported_gep; 654274955Ssvnmir } 655274955Ssvnmir } 656274955Ssvnmir } 657274955Ssvnmir 658274955Ssvnmir // Try to grab the base operand now. 659274955Ssvnmir Addr.setOffset(TmpOffset); 660280031Sdim if (computeAddress(U->getOperand(0), Addr, Ty)) 661274955Ssvnmir return true; 662274955Ssvnmir 663274955Ssvnmir // We failed, restore everything and try the other options. 664274955Ssvnmir Addr = SavedAddr; 665274955Ssvnmir 666274955Ssvnmir unsupported_gep: 667274955Ssvnmir break; 668274955Ssvnmir } 669274955Ssvnmir case Instruction::Alloca: { 670274955Ssvnmir const AllocaInst *AI = cast<AllocaInst>(Obj); 671274955Ssvnmir DenseMap<const AllocaInst *, int>::iterator SI = 672274955Ssvnmir FuncInfo.StaticAllocaMap.find(AI); 673274955Ssvnmir if (SI != FuncInfo.StaticAllocaMap.end()) { 674274955Ssvnmir Addr.setKind(Address::FrameIndexBase); 675274955Ssvnmir Addr.setFI(SI->second); 676274955Ssvnmir return true; 677274955Ssvnmir } 678274955Ssvnmir break; 679274955Ssvnmir } 680280031Sdim case Instruction::Add: { 681280031Sdim // Adds of constants are common and easy enough. 682280031Sdim const Value *LHS = U->getOperand(0); 683280031Sdim const Value *RHS = U->getOperand(1); 684280031Sdim 685280031Sdim if (isa<ConstantInt>(LHS)) 686280031Sdim std::swap(LHS, RHS); 687280031Sdim 688280031Sdim if (const ConstantInt *CI = dyn_cast<ConstantInt>(RHS)) { 689280031Sdim Addr.setOffset(Addr.getOffset() + CI->getSExtValue()); 690280031Sdim return computeAddress(LHS, Addr, Ty); 691280031Sdim } 692280031Sdim 693280031Sdim Address Backup = Addr; 694280031Sdim if (computeAddress(LHS, Addr, Ty) && computeAddress(RHS, Addr, Ty)) 695280031Sdim return true; 696280031Sdim Addr = Backup; 697280031Sdim 698280031Sdim break; 699274955Ssvnmir } 700280031Sdim case Instruction::Sub: { 701280031Sdim // Subs of constants are common and easy enough. 702280031Sdim const Value *LHS = U->getOperand(0); 703280031Sdim const Value *RHS = U->getOperand(1); 704274955Ssvnmir 705280031Sdim if (const ConstantInt *CI = dyn_cast<ConstantInt>(RHS)) { 706280031Sdim Addr.setOffset(Addr.getOffset() - CI->getSExtValue()); 707280031Sdim return computeAddress(LHS, Addr, Ty); 708280031Sdim } 709280031Sdim break; 710280031Sdim } 711280031Sdim case Instruction::Shl: { 712280031Sdim if (Addr.getOffsetReg()) 713280031Sdim break; 714280031Sdim 715280031Sdim const auto *CI = dyn_cast<ConstantInt>(U->getOperand(1)); 716280031Sdim if (!CI) 717280031Sdim break; 718280031Sdim 719280031Sdim unsigned Val = CI->getZExtValue(); 720280031Sdim if (Val < 1 || Val > 3) 721280031Sdim break; 722280031Sdim 723280031Sdim uint64_t NumBytes = 0; 724280031Sdim if (Ty && Ty->isSized()) { 725280031Sdim uint64_t NumBits = DL.getTypeSizeInBits(Ty); 726280031Sdim NumBytes = NumBits / 8; 727280031Sdim if (!isPowerOf2_64(NumBits)) 728280031Sdim NumBytes = 0; 729280031Sdim } 730280031Sdim 731280031Sdim if (NumBytes != (1ULL << Val)) 732280031Sdim break; 733280031Sdim 734280031Sdim Addr.setShift(Val); 735280031Sdim Addr.setExtendType(AArch64_AM::LSL); 736280031Sdim 737280031Sdim const Value *Src = U->getOperand(0); 738288943Sdim if (const auto *I = dyn_cast<Instruction>(Src)) { 739288943Sdim if (FuncInfo.MBBMap[I->getParent()] == FuncInfo.MBB) { 740288943Sdim // Fold the zext or sext when it won't become a noop. 741288943Sdim if (const auto *ZE = dyn_cast<ZExtInst>(I)) { 742288943Sdim if (!isIntExtFree(ZE) && 743288943Sdim ZE->getOperand(0)->getType()->isIntegerTy(32)) { 744288943Sdim Addr.setExtendType(AArch64_AM::UXTW); 745288943Sdim Src = ZE->getOperand(0); 746288943Sdim } 747288943Sdim } else if (const auto *SE = dyn_cast<SExtInst>(I)) { 748288943Sdim if (!isIntExtFree(SE) && 749288943Sdim SE->getOperand(0)->getType()->isIntegerTy(32)) { 750288943Sdim Addr.setExtendType(AArch64_AM::SXTW); 751288943Sdim Src = SE->getOperand(0); 752288943Sdim } 753288943Sdim } 754280031Sdim } 755280031Sdim } 756280031Sdim 757280031Sdim if (const auto *AI = dyn_cast<BinaryOperator>(Src)) 758280031Sdim if (AI->getOpcode() == Instruction::And) { 759280031Sdim const Value *LHS = AI->getOperand(0); 760280031Sdim const Value *RHS = AI->getOperand(1); 761280031Sdim 762280031Sdim if (const auto *C = dyn_cast<ConstantInt>(LHS)) 763280031Sdim if (C->getValue() == 0xffffffff) 764280031Sdim std::swap(LHS, RHS); 765280031Sdim 766280031Sdim if (const auto *C = dyn_cast<ConstantInt>(RHS)) 767280031Sdim if (C->getValue() == 0xffffffff) { 768280031Sdim Addr.setExtendType(AArch64_AM::UXTW); 769280031Sdim unsigned Reg = getRegForValue(LHS); 770280031Sdim if (!Reg) 771280031Sdim return false; 772280031Sdim bool RegIsKill = hasTrivialKill(LHS); 773280031Sdim Reg = fastEmitInst_extractsubreg(MVT::i32, Reg, RegIsKill, 774280031Sdim AArch64::sub_32); 775280031Sdim Addr.setOffsetReg(Reg); 776280031Sdim return true; 777280031Sdim } 778280031Sdim } 779280031Sdim 780280031Sdim unsigned Reg = getRegForValue(Src); 781280031Sdim if (!Reg) 782280031Sdim return false; 783280031Sdim Addr.setOffsetReg(Reg); 784280031Sdim return true; 785280031Sdim } 786280031Sdim case Instruction::Mul: { 787280031Sdim if (Addr.getOffsetReg()) 788280031Sdim break; 789280031Sdim 790280031Sdim if (!isMulPowOf2(U)) 791280031Sdim break; 792280031Sdim 793280031Sdim const Value *LHS = U->getOperand(0); 794280031Sdim const Value *RHS = U->getOperand(1); 795280031Sdim 796280031Sdim // Canonicalize power-of-2 value to the RHS. 797280031Sdim if (const auto *C = dyn_cast<ConstantInt>(LHS)) 798280031Sdim if (C->getValue().isPowerOf2()) 799280031Sdim std::swap(LHS, RHS); 800280031Sdim 801280031Sdim assert(isa<ConstantInt>(RHS) && "Expected an ConstantInt."); 802280031Sdim const auto *C = cast<ConstantInt>(RHS); 803280031Sdim unsigned Val = C->getValue().logBase2(); 804280031Sdim if (Val < 1 || Val > 3) 805280031Sdim break; 806280031Sdim 807280031Sdim uint64_t NumBytes = 0; 808280031Sdim if (Ty && Ty->isSized()) { 809280031Sdim uint64_t NumBits = DL.getTypeSizeInBits(Ty); 810280031Sdim NumBytes = NumBits / 8; 811280031Sdim if (!isPowerOf2_64(NumBits)) 812280031Sdim NumBytes = 0; 813280031Sdim } 814280031Sdim 815280031Sdim if (NumBytes != (1ULL << Val)) 816280031Sdim break; 817280031Sdim 818280031Sdim Addr.setShift(Val); 819280031Sdim Addr.setExtendType(AArch64_AM::LSL); 820280031Sdim 821280031Sdim const Value *Src = LHS; 822288943Sdim if (const auto *I = dyn_cast<Instruction>(Src)) { 823288943Sdim if (FuncInfo.MBBMap[I->getParent()] == FuncInfo.MBB) { 824288943Sdim // Fold the zext or sext when it won't become a noop. 825288943Sdim if (const auto *ZE = dyn_cast<ZExtInst>(I)) { 826288943Sdim if (!isIntExtFree(ZE) && 827288943Sdim ZE->getOperand(0)->getType()->isIntegerTy(32)) { 828288943Sdim Addr.setExtendType(AArch64_AM::UXTW); 829288943Sdim Src = ZE->getOperand(0); 830288943Sdim } 831288943Sdim } else if (const auto *SE = dyn_cast<SExtInst>(I)) { 832288943Sdim if (!isIntExtFree(SE) && 833288943Sdim SE->getOperand(0)->getType()->isIntegerTy(32)) { 834288943Sdim Addr.setExtendType(AArch64_AM::SXTW); 835288943Sdim Src = SE->getOperand(0); 836288943Sdim } 837288943Sdim } 838280031Sdim } 839280031Sdim } 840280031Sdim 841280031Sdim unsigned Reg = getRegForValue(Src); 842280031Sdim if (!Reg) 843280031Sdim return false; 844280031Sdim Addr.setOffsetReg(Reg); 845280031Sdim return true; 846280031Sdim } 847280031Sdim case Instruction::And: { 848280031Sdim if (Addr.getOffsetReg()) 849280031Sdim break; 850280031Sdim 851280031Sdim if (!Ty || DL.getTypeSizeInBits(Ty) != 8) 852280031Sdim break; 853280031Sdim 854280031Sdim const Value *LHS = U->getOperand(0); 855280031Sdim const Value *RHS = U->getOperand(1); 856280031Sdim 857280031Sdim if (const auto *C = dyn_cast<ConstantInt>(LHS)) 858280031Sdim if (C->getValue() == 0xffffffff) 859280031Sdim std::swap(LHS, RHS); 860280031Sdim 861280031Sdim if (const auto *C = dyn_cast<ConstantInt>(RHS)) 862280031Sdim if (C->getValue() == 0xffffffff) { 863280031Sdim Addr.setShift(0); 864280031Sdim Addr.setExtendType(AArch64_AM::LSL); 865280031Sdim Addr.setExtendType(AArch64_AM::UXTW); 866280031Sdim 867280031Sdim unsigned Reg = getRegForValue(LHS); 868280031Sdim if (!Reg) 869280031Sdim return false; 870280031Sdim bool RegIsKill = hasTrivialKill(LHS); 871280031Sdim Reg = fastEmitInst_extractsubreg(MVT::i32, Reg, RegIsKill, 872280031Sdim AArch64::sub_32); 873280031Sdim Addr.setOffsetReg(Reg); 874280031Sdim return true; 875280031Sdim } 876280031Sdim break; 877280031Sdim } 878280031Sdim case Instruction::SExt: 879280031Sdim case Instruction::ZExt: { 880280031Sdim if (!Addr.getReg() || Addr.getOffsetReg()) 881280031Sdim break; 882280031Sdim 883280031Sdim const Value *Src = nullptr; 884280031Sdim // Fold the zext or sext when it won't become a noop. 885280031Sdim if (const auto *ZE = dyn_cast<ZExtInst>(U)) { 886280031Sdim if (!isIntExtFree(ZE) && ZE->getOperand(0)->getType()->isIntegerTy(32)) { 887280031Sdim Addr.setExtendType(AArch64_AM::UXTW); 888280031Sdim Src = ZE->getOperand(0); 889280031Sdim } 890280031Sdim } else if (const auto *SE = dyn_cast<SExtInst>(U)) { 891280031Sdim if (!isIntExtFree(SE) && SE->getOperand(0)->getType()->isIntegerTy(32)) { 892280031Sdim Addr.setExtendType(AArch64_AM::SXTW); 893280031Sdim Src = SE->getOperand(0); 894280031Sdim } 895280031Sdim } 896280031Sdim 897280031Sdim if (!Src) 898280031Sdim break; 899280031Sdim 900280031Sdim Addr.setShift(0); 901280031Sdim unsigned Reg = getRegForValue(Src); 902280031Sdim if (!Reg) 903280031Sdim return false; 904280031Sdim Addr.setOffsetReg(Reg); 905280031Sdim return true; 906280031Sdim } 907280031Sdim } // end switch 908280031Sdim 909280031Sdim if (Addr.isRegBase() && !Addr.getReg()) { 910280031Sdim unsigned Reg = getRegForValue(Obj); 911280031Sdim if (!Reg) 912280031Sdim return false; 913280031Sdim Addr.setReg(Reg); 914280031Sdim return true; 915280031Sdim } 916280031Sdim 917280031Sdim if (!Addr.getOffsetReg()) { 918280031Sdim unsigned Reg = getRegForValue(Obj); 919280031Sdim if (!Reg) 920280031Sdim return false; 921280031Sdim Addr.setOffsetReg(Reg); 922280031Sdim return true; 923280031Sdim } 924280031Sdim 925280031Sdim return false; 926274955Ssvnmir} 927274955Ssvnmir 928280031Sdimbool AArch64FastISel::computeCallAddress(const Value *V, Address &Addr) { 929280031Sdim const User *U = nullptr; 930280031Sdim unsigned Opcode = Instruction::UserOp1; 931280031Sdim bool InMBB = true; 932280031Sdim 933280031Sdim if (const auto *I = dyn_cast<Instruction>(V)) { 934280031Sdim Opcode = I->getOpcode(); 935280031Sdim U = I; 936280031Sdim InMBB = I->getParent() == FuncInfo.MBB->getBasicBlock(); 937280031Sdim } else if (const auto *C = dyn_cast<ConstantExpr>(V)) { 938280031Sdim Opcode = C->getOpcode(); 939280031Sdim U = C; 940280031Sdim } 941280031Sdim 942280031Sdim switch (Opcode) { 943280031Sdim default: break; 944280031Sdim case Instruction::BitCast: 945280031Sdim // Look past bitcasts if its operand is in the same BB. 946280031Sdim if (InMBB) 947280031Sdim return computeCallAddress(U->getOperand(0), Addr); 948280031Sdim break; 949280031Sdim case Instruction::IntToPtr: 950280031Sdim // Look past no-op inttoptrs if its operand is in the same BB. 951280031Sdim if (InMBB && 952288943Sdim TLI.getValueType(DL, U->getOperand(0)->getType()) == 953288943Sdim TLI.getPointerTy(DL)) 954280031Sdim return computeCallAddress(U->getOperand(0), Addr); 955280031Sdim break; 956280031Sdim case Instruction::PtrToInt: 957280031Sdim // Look past no-op ptrtoints if its operand is in the same BB. 958288943Sdim if (InMBB && TLI.getValueType(DL, U->getType()) == TLI.getPointerTy(DL)) 959280031Sdim return computeCallAddress(U->getOperand(0), Addr); 960280031Sdim break; 961280031Sdim } 962280031Sdim 963280031Sdim if (const GlobalValue *GV = dyn_cast<GlobalValue>(V)) { 964280031Sdim Addr.setGlobalValue(GV); 965280031Sdim return true; 966280031Sdim } 967280031Sdim 968280031Sdim // If all else fails, try to materialize the value in a register. 969280031Sdim if (!Addr.getGlobalValue()) { 970280031Sdim Addr.setReg(getRegForValue(V)); 971280031Sdim return Addr.getReg() != 0; 972280031Sdim } 973280031Sdim 974280031Sdim return false; 975280031Sdim} 976280031Sdim 977274955Ssvnmirbool AArch64FastISel::isTypeLegal(Type *Ty, MVT &VT) { 978288943Sdim EVT evt = TLI.getValueType(DL, Ty, true); 979274955Ssvnmir 980360784Sdim if (Subtarget->isTargetILP32() && Ty->isPointerTy()) 981360784Sdim return false; 982360784Sdim 983274955Ssvnmir // Only handle simple types. 984274955Ssvnmir if (evt == MVT::Other || !evt.isSimple()) 985274955Ssvnmir return false; 986274955Ssvnmir VT = evt.getSimpleVT(); 987274955Ssvnmir 988274955Ssvnmir // This is a legal type, but it's not something we handle in fast-isel. 989274955Ssvnmir if (VT == MVT::f128) 990274955Ssvnmir return false; 991274955Ssvnmir 992274955Ssvnmir // Handle all other legal types, i.e. a register that will directly hold this 993274955Ssvnmir // value. 994274955Ssvnmir return TLI.isTypeLegal(VT); 995274955Ssvnmir} 996274955Ssvnmir 997341825Sdim/// Determine if the value type is supported by FastISel. 998280031Sdim/// 999280031Sdim/// FastISel for AArch64 can handle more value types than are legal. This adds 1000280031Sdim/// simple value type such as i1, i8, and i16. 1001280031Sdimbool AArch64FastISel::isTypeSupported(Type *Ty, MVT &VT, bool IsVectorAllowed) { 1002280031Sdim if (Ty->isVectorTy() && !IsVectorAllowed) 1003280031Sdim return false; 1004280031Sdim 1005274955Ssvnmir if (isTypeLegal(Ty, VT)) 1006274955Ssvnmir return true; 1007274955Ssvnmir 1008274955Ssvnmir // If this is a type than can be sign or zero-extended to a basic operation 1009280031Sdim // go ahead and accept it now. 1010274955Ssvnmir if (VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16) 1011274955Ssvnmir return true; 1012274955Ssvnmir 1013274955Ssvnmir return false; 1014274955Ssvnmir} 1015274955Ssvnmir 1016280031Sdimbool AArch64FastISel::isValueAvailable(const Value *V) const { 1017280031Sdim if (!isa<Instruction>(V)) 1018280031Sdim return true; 1019280031Sdim 1020280031Sdim const auto *I = cast<Instruction>(V); 1021309124Sdim return FuncInfo.MBBMap[I->getParent()] == FuncInfo.MBB; 1022280031Sdim} 1023280031Sdim 1024280031Sdimbool AArch64FastISel::simplifyAddress(Address &Addr, MVT VT) { 1025360784Sdim if (Subtarget->isTargetILP32()) 1026360784Sdim return false; 1027360784Sdim 1028280031Sdim unsigned ScaleFactor = getImplicitScaleFactor(VT); 1029280031Sdim if (!ScaleFactor) 1030274955Ssvnmir return false; 1031274955Ssvnmir 1032280031Sdim bool ImmediateOffsetNeedsLowering = false; 1033280031Sdim bool RegisterOffsetNeedsLowering = false; 1034280031Sdim int64_t Offset = Addr.getOffset(); 1035280031Sdim if (((Offset < 0) || (Offset & (ScaleFactor - 1))) && !isInt<9>(Offset)) 1036280031Sdim ImmediateOffsetNeedsLowering = true; 1037280031Sdim else if (Offset > 0 && !(Offset & (ScaleFactor - 1)) && 1038280031Sdim !isUInt<12>(Offset / ScaleFactor)) 1039280031Sdim ImmediateOffsetNeedsLowering = true; 1040280031Sdim 1041280031Sdim // Cannot encode an offset register and an immediate offset in the same 1042280031Sdim // instruction. Fold the immediate offset into the load/store instruction and 1043296417Sdim // emit an additional add to take care of the offset register. 1044280031Sdim if (!ImmediateOffsetNeedsLowering && Addr.getOffset() && Addr.getOffsetReg()) 1045280031Sdim RegisterOffsetNeedsLowering = true; 1046280031Sdim 1047280031Sdim // Cannot encode zero register as base. 1048280031Sdim if (Addr.isRegBase() && Addr.getOffsetReg() && !Addr.getReg()) 1049280031Sdim RegisterOffsetNeedsLowering = true; 1050280031Sdim 1051280031Sdim // If this is a stack pointer and the offset needs to be simplified then put 1052274955Ssvnmir // the alloca address into a register, set the base type back to register and 1053274955Ssvnmir // continue. This should almost never happen. 1054280031Sdim if ((ImmediateOffsetNeedsLowering || Addr.getOffsetReg()) && Addr.isFIBase()) 1055280031Sdim { 1056280031Sdim unsigned ResultReg = createResultReg(&AArch64::GPR64spRegClass); 1057274955Ssvnmir BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::ADDXri), 1058274955Ssvnmir ResultReg) 1059280031Sdim .addFrameIndex(Addr.getFI()) 1060280031Sdim .addImm(0) 1061280031Sdim .addImm(0); 1062274955Ssvnmir Addr.setKind(Address::RegBase); 1063274955Ssvnmir Addr.setReg(ResultReg); 1064274955Ssvnmir } 1065274955Ssvnmir 1066280031Sdim if (RegisterOffsetNeedsLowering) { 1067280031Sdim unsigned ResultReg = 0; 1068280031Sdim if (Addr.getReg()) { 1069280031Sdim if (Addr.getExtendType() == AArch64_AM::SXTW || 1070280031Sdim Addr.getExtendType() == AArch64_AM::UXTW ) 1071280031Sdim ResultReg = emitAddSub_rx(/*UseAdd=*/true, MVT::i64, Addr.getReg(), 1072280031Sdim /*TODO:IsKill=*/false, Addr.getOffsetReg(), 1073280031Sdim /*TODO:IsKill=*/false, Addr.getExtendType(), 1074280031Sdim Addr.getShift()); 1075280031Sdim else 1076280031Sdim ResultReg = emitAddSub_rs(/*UseAdd=*/true, MVT::i64, Addr.getReg(), 1077280031Sdim /*TODO:IsKill=*/false, Addr.getOffsetReg(), 1078280031Sdim /*TODO:IsKill=*/false, AArch64_AM::LSL, 1079280031Sdim Addr.getShift()); 1080280031Sdim } else { 1081280031Sdim if (Addr.getExtendType() == AArch64_AM::UXTW) 1082280031Sdim ResultReg = emitLSL_ri(MVT::i64, MVT::i32, Addr.getOffsetReg(), 1083280031Sdim /*Op0IsKill=*/false, Addr.getShift(), 1084280031Sdim /*IsZExt=*/true); 1085280031Sdim else if (Addr.getExtendType() == AArch64_AM::SXTW) 1086280031Sdim ResultReg = emitLSL_ri(MVT::i64, MVT::i32, Addr.getOffsetReg(), 1087280031Sdim /*Op0IsKill=*/false, Addr.getShift(), 1088280031Sdim /*IsZExt=*/false); 1089280031Sdim else 1090280031Sdim ResultReg = emitLSL_ri(MVT::i64, MVT::i64, Addr.getOffsetReg(), 1091280031Sdim /*Op0IsKill=*/false, Addr.getShift()); 1092280031Sdim } 1093280031Sdim if (!ResultReg) 1094280031Sdim return false; 1095280031Sdim 1096280031Sdim Addr.setReg(ResultReg); 1097280031Sdim Addr.setOffsetReg(0); 1098280031Sdim Addr.setShift(0); 1099280031Sdim Addr.setExtendType(AArch64_AM::InvalidShiftExtend); 1100280031Sdim } 1101280031Sdim 1102274955Ssvnmir // Since the offset is too large for the load/store instruction get the 1103274955Ssvnmir // reg+offset into a register. 1104280031Sdim if (ImmediateOffsetNeedsLowering) { 1105280031Sdim unsigned ResultReg; 1106280031Sdim if (Addr.getReg()) 1107280031Sdim // Try to fold the immediate into the add instruction. 1108280031Sdim ResultReg = emitAdd_ri_(MVT::i64, Addr.getReg(), /*IsKill=*/false, Offset); 1109280031Sdim else 1110280031Sdim ResultReg = fastEmit_i(MVT::i64, MVT::i64, ISD::Constant, Offset); 1111280031Sdim 1112280031Sdim if (!ResultReg) 1113274955Ssvnmir return false; 1114274955Ssvnmir Addr.setReg(ResultReg); 1115274955Ssvnmir Addr.setOffset(0); 1116274955Ssvnmir } 1117274955Ssvnmir return true; 1118274955Ssvnmir} 1119274955Ssvnmir 1120280031Sdimvoid AArch64FastISel::addLoadStoreOperands(Address &Addr, 1121274955Ssvnmir const MachineInstrBuilder &MIB, 1122309124Sdim MachineMemOperand::Flags Flags, 1123280031Sdim unsigned ScaleFactor, 1124280031Sdim MachineMemOperand *MMO) { 1125280031Sdim int64_t Offset = Addr.getOffset() / ScaleFactor; 1126274955Ssvnmir // Frame base works a bit differently. Handle it separately. 1127280031Sdim if (Addr.isFIBase()) { 1128274955Ssvnmir int FI = Addr.getFI(); 1129274955Ssvnmir // FIXME: We shouldn't be using getObjectSize/getObjectAlignment. The size 1130274955Ssvnmir // and alignment should be based on the VT. 1131280031Sdim MMO = FuncInfo.MF->getMachineMemOperand( 1132296417Sdim MachinePointerInfo::getFixedStack(*FuncInfo.MF, FI, Offset), Flags, 1133296417Sdim MFI.getObjectSize(FI), MFI.getObjectAlignment(FI)); 1134274955Ssvnmir // Now add the rest of the operands. 1135280031Sdim MIB.addFrameIndex(FI).addImm(Offset); 1136274955Ssvnmir } else { 1137280031Sdim assert(Addr.isRegBase() && "Unexpected address kind."); 1138280031Sdim const MCInstrDesc &II = MIB->getDesc(); 1139280031Sdim unsigned Idx = (Flags & MachineMemOperand::MOStore) ? 1 : 0; 1140280031Sdim Addr.setReg( 1141280031Sdim constrainOperandRegClass(II, Addr.getReg(), II.getNumDefs()+Idx)); 1142280031Sdim Addr.setOffsetReg( 1143280031Sdim constrainOperandRegClass(II, Addr.getOffsetReg(), II.getNumDefs()+Idx+1)); 1144280031Sdim if (Addr.getOffsetReg()) { 1145280031Sdim assert(Addr.getOffset() == 0 && "Unexpected offset"); 1146280031Sdim bool IsSigned = Addr.getExtendType() == AArch64_AM::SXTW || 1147280031Sdim Addr.getExtendType() == AArch64_AM::SXTX; 1148280031Sdim MIB.addReg(Addr.getReg()); 1149280031Sdim MIB.addReg(Addr.getOffsetReg()); 1150280031Sdim MIB.addImm(IsSigned); 1151280031Sdim MIB.addImm(Addr.getShift() != 0); 1152280031Sdim } else 1153280031Sdim MIB.addReg(Addr.getReg()).addImm(Offset); 1154274955Ssvnmir } 1155280031Sdim 1156280031Sdim if (MMO) 1157280031Sdim MIB.addMemOperand(MMO); 1158274955Ssvnmir} 1159274955Ssvnmir 1160280031Sdimunsigned AArch64FastISel::emitAddSub(bool UseAdd, MVT RetVT, const Value *LHS, 1161280031Sdim const Value *RHS, bool SetFlags, 1162280031Sdim bool WantResult, bool IsZExt) { 1163280031Sdim AArch64_AM::ShiftExtendType ExtendType = AArch64_AM::InvalidShiftExtend; 1164280031Sdim bool NeedExtend = false; 1165280031Sdim switch (RetVT.SimpleTy) { 1166280031Sdim default: 1167280031Sdim return 0; 1168280031Sdim case MVT::i1: 1169280031Sdim NeedExtend = true; 1170280031Sdim break; 1171280031Sdim case MVT::i8: 1172280031Sdim NeedExtend = true; 1173280031Sdim ExtendType = IsZExt ? AArch64_AM::UXTB : AArch64_AM::SXTB; 1174280031Sdim break; 1175280031Sdim case MVT::i16: 1176280031Sdim NeedExtend = true; 1177280031Sdim ExtendType = IsZExt ? AArch64_AM::UXTH : AArch64_AM::SXTH; 1178280031Sdim break; 1179280031Sdim case MVT::i32: // fall-through 1180280031Sdim case MVT::i64: 1181280031Sdim break; 1182280031Sdim } 1183280031Sdim MVT SrcVT = RetVT; 1184280031Sdim RetVT.SimpleTy = std::max(RetVT.SimpleTy, MVT::i32); 1185274955Ssvnmir 1186280031Sdim // Canonicalize immediates to the RHS first. 1187280031Sdim if (UseAdd && isa<Constant>(LHS) && !isa<Constant>(RHS)) 1188280031Sdim std::swap(LHS, RHS); 1189280031Sdim 1190280031Sdim // Canonicalize mul by power of 2 to the RHS. 1191280031Sdim if (UseAdd && LHS->hasOneUse() && isValueAvailable(LHS)) 1192280031Sdim if (isMulPowOf2(LHS)) 1193280031Sdim std::swap(LHS, RHS); 1194280031Sdim 1195280031Sdim // Canonicalize shift immediate to the RHS. 1196280031Sdim if (UseAdd && LHS->hasOneUse() && isValueAvailable(LHS)) 1197280031Sdim if (const auto *SI = dyn_cast<BinaryOperator>(LHS)) 1198280031Sdim if (isa<ConstantInt>(SI->getOperand(1))) 1199280031Sdim if (SI->getOpcode() == Instruction::Shl || 1200280031Sdim SI->getOpcode() == Instruction::LShr || 1201280031Sdim SI->getOpcode() == Instruction::AShr ) 1202280031Sdim std::swap(LHS, RHS); 1203280031Sdim 1204280031Sdim unsigned LHSReg = getRegForValue(LHS); 1205280031Sdim if (!LHSReg) 1206280031Sdim return 0; 1207280031Sdim bool LHSIsKill = hasTrivialKill(LHS); 1208280031Sdim 1209280031Sdim if (NeedExtend) 1210280031Sdim LHSReg = emitIntExt(SrcVT, LHSReg, RetVT, IsZExt); 1211280031Sdim 1212280031Sdim unsigned ResultReg = 0; 1213280031Sdim if (const auto *C = dyn_cast<ConstantInt>(RHS)) { 1214280031Sdim uint64_t Imm = IsZExt ? C->getZExtValue() : C->getSExtValue(); 1215280031Sdim if (C->isNegative()) 1216280031Sdim ResultReg = emitAddSub_ri(!UseAdd, RetVT, LHSReg, LHSIsKill, -Imm, 1217280031Sdim SetFlags, WantResult); 1218280031Sdim else 1219280031Sdim ResultReg = emitAddSub_ri(UseAdd, RetVT, LHSReg, LHSIsKill, Imm, SetFlags, 1220280031Sdim WantResult); 1221280031Sdim } else if (const auto *C = dyn_cast<Constant>(RHS)) 1222280031Sdim if (C->isNullValue()) 1223280031Sdim ResultReg = emitAddSub_ri(UseAdd, RetVT, LHSReg, LHSIsKill, 0, SetFlags, 1224280031Sdim WantResult); 1225280031Sdim 1226280031Sdim if (ResultReg) 1227280031Sdim return ResultReg; 1228280031Sdim 1229280031Sdim // Only extend the RHS within the instruction if there is a valid extend type. 1230280031Sdim if (ExtendType != AArch64_AM::InvalidShiftExtend && RHS->hasOneUse() && 1231280031Sdim isValueAvailable(RHS)) { 1232280031Sdim if (const auto *SI = dyn_cast<BinaryOperator>(RHS)) 1233280031Sdim if (const auto *C = dyn_cast<ConstantInt>(SI->getOperand(1))) 1234280031Sdim if ((SI->getOpcode() == Instruction::Shl) && (C->getZExtValue() < 4)) { 1235280031Sdim unsigned RHSReg = getRegForValue(SI->getOperand(0)); 1236280031Sdim if (!RHSReg) 1237280031Sdim return 0; 1238280031Sdim bool RHSIsKill = hasTrivialKill(SI->getOperand(0)); 1239280031Sdim return emitAddSub_rx(UseAdd, RetVT, LHSReg, LHSIsKill, RHSReg, 1240280031Sdim RHSIsKill, ExtendType, C->getZExtValue(), 1241280031Sdim SetFlags, WantResult); 1242280031Sdim } 1243280031Sdim unsigned RHSReg = getRegForValue(RHS); 1244280031Sdim if (!RHSReg) 1245280031Sdim return 0; 1246280031Sdim bool RHSIsKill = hasTrivialKill(RHS); 1247280031Sdim return emitAddSub_rx(UseAdd, RetVT, LHSReg, LHSIsKill, RHSReg, RHSIsKill, 1248280031Sdim ExtendType, 0, SetFlags, WantResult); 1249280031Sdim } 1250280031Sdim 1251280031Sdim // Check if the mul can be folded into the instruction. 1252296417Sdim if (RHS->hasOneUse() && isValueAvailable(RHS)) { 1253280031Sdim if (isMulPowOf2(RHS)) { 1254280031Sdim const Value *MulLHS = cast<MulOperator>(RHS)->getOperand(0); 1255280031Sdim const Value *MulRHS = cast<MulOperator>(RHS)->getOperand(1); 1256280031Sdim 1257280031Sdim if (const auto *C = dyn_cast<ConstantInt>(MulLHS)) 1258280031Sdim if (C->getValue().isPowerOf2()) 1259280031Sdim std::swap(MulLHS, MulRHS); 1260280031Sdim 1261280031Sdim assert(isa<ConstantInt>(MulRHS) && "Expected a ConstantInt."); 1262280031Sdim uint64_t ShiftVal = cast<ConstantInt>(MulRHS)->getValue().logBase2(); 1263280031Sdim unsigned RHSReg = getRegForValue(MulLHS); 1264280031Sdim if (!RHSReg) 1265280031Sdim return 0; 1266280031Sdim bool RHSIsKill = hasTrivialKill(MulLHS); 1267296417Sdim ResultReg = emitAddSub_rs(UseAdd, RetVT, LHSReg, LHSIsKill, RHSReg, 1268296417Sdim RHSIsKill, AArch64_AM::LSL, ShiftVal, SetFlags, 1269296417Sdim WantResult); 1270296417Sdim if (ResultReg) 1271296417Sdim return ResultReg; 1272280031Sdim } 1273296417Sdim } 1274280031Sdim 1275280031Sdim // Check if the shift can be folded into the instruction. 1276296417Sdim if (RHS->hasOneUse() && isValueAvailable(RHS)) { 1277280031Sdim if (const auto *SI = dyn_cast<BinaryOperator>(RHS)) { 1278280031Sdim if (const auto *C = dyn_cast<ConstantInt>(SI->getOperand(1))) { 1279280031Sdim AArch64_AM::ShiftExtendType ShiftType = AArch64_AM::InvalidShiftExtend; 1280280031Sdim switch (SI->getOpcode()) { 1281280031Sdim default: break; 1282280031Sdim case Instruction::Shl: ShiftType = AArch64_AM::LSL; break; 1283280031Sdim case Instruction::LShr: ShiftType = AArch64_AM::LSR; break; 1284280031Sdim case Instruction::AShr: ShiftType = AArch64_AM::ASR; break; 1285280031Sdim } 1286280031Sdim uint64_t ShiftVal = C->getZExtValue(); 1287280031Sdim if (ShiftType != AArch64_AM::InvalidShiftExtend) { 1288280031Sdim unsigned RHSReg = getRegForValue(SI->getOperand(0)); 1289280031Sdim if (!RHSReg) 1290280031Sdim return 0; 1291280031Sdim bool RHSIsKill = hasTrivialKill(SI->getOperand(0)); 1292296417Sdim ResultReg = emitAddSub_rs(UseAdd, RetVT, LHSReg, LHSIsKill, RHSReg, 1293296417Sdim RHSIsKill, ShiftType, ShiftVal, SetFlags, 1294296417Sdim WantResult); 1295296417Sdim if (ResultReg) 1296296417Sdim return ResultReg; 1297280031Sdim } 1298280031Sdim } 1299280031Sdim } 1300296417Sdim } 1301280031Sdim 1302280031Sdim unsigned RHSReg = getRegForValue(RHS); 1303280031Sdim if (!RHSReg) 1304280031Sdim return 0; 1305280031Sdim bool RHSIsKill = hasTrivialKill(RHS); 1306280031Sdim 1307280031Sdim if (NeedExtend) 1308280031Sdim RHSReg = emitIntExt(SrcVT, RHSReg, RetVT, IsZExt); 1309280031Sdim 1310280031Sdim return emitAddSub_rr(UseAdd, RetVT, LHSReg, LHSIsKill, RHSReg, RHSIsKill, 1311280031Sdim SetFlags, WantResult); 1312280031Sdim} 1313280031Sdim 1314280031Sdimunsigned AArch64FastISel::emitAddSub_rr(bool UseAdd, MVT RetVT, unsigned LHSReg, 1315280031Sdim bool LHSIsKill, unsigned RHSReg, 1316280031Sdim bool RHSIsKill, bool SetFlags, 1317280031Sdim bool WantResult) { 1318280031Sdim assert(LHSReg && RHSReg && "Invalid register number."); 1319280031Sdim 1320321369Sdim if (LHSReg == AArch64::SP || LHSReg == AArch64::WSP || 1321321369Sdim RHSReg == AArch64::SP || RHSReg == AArch64::WSP) 1322321369Sdim return 0; 1323321369Sdim 1324280031Sdim if (RetVT != MVT::i32 && RetVT != MVT::i64) 1325280031Sdim return 0; 1326280031Sdim 1327280031Sdim static const unsigned OpcTable[2][2][2] = { 1328280031Sdim { { AArch64::SUBWrr, AArch64::SUBXrr }, 1329280031Sdim { AArch64::ADDWrr, AArch64::ADDXrr } }, 1330280031Sdim { { AArch64::SUBSWrr, AArch64::SUBSXrr }, 1331280031Sdim { AArch64::ADDSWrr, AArch64::ADDSXrr } } 1332280031Sdim }; 1333280031Sdim bool Is64Bit = RetVT == MVT::i64; 1334280031Sdim unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit]; 1335280031Sdim const TargetRegisterClass *RC = 1336280031Sdim Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass; 1337280031Sdim unsigned ResultReg; 1338280031Sdim if (WantResult) 1339280031Sdim ResultReg = createResultReg(RC); 1340280031Sdim else 1341280031Sdim ResultReg = Is64Bit ? AArch64::XZR : AArch64::WZR; 1342280031Sdim 1343280031Sdim const MCInstrDesc &II = TII.get(Opc); 1344280031Sdim LHSReg = constrainOperandRegClass(II, LHSReg, II.getNumDefs()); 1345280031Sdim RHSReg = constrainOperandRegClass(II, RHSReg, II.getNumDefs() + 1); 1346280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg) 1347280031Sdim .addReg(LHSReg, getKillRegState(LHSIsKill)) 1348280031Sdim .addReg(RHSReg, getKillRegState(RHSIsKill)); 1349280031Sdim return ResultReg; 1350280031Sdim} 1351280031Sdim 1352280031Sdimunsigned AArch64FastISel::emitAddSub_ri(bool UseAdd, MVT RetVT, unsigned LHSReg, 1353280031Sdim bool LHSIsKill, uint64_t Imm, 1354280031Sdim bool SetFlags, bool WantResult) { 1355280031Sdim assert(LHSReg && "Invalid register number."); 1356280031Sdim 1357280031Sdim if (RetVT != MVT::i32 && RetVT != MVT::i64) 1358280031Sdim return 0; 1359280031Sdim 1360280031Sdim unsigned ShiftImm; 1361280031Sdim if (isUInt<12>(Imm)) 1362280031Sdim ShiftImm = 0; 1363280031Sdim else if ((Imm & 0xfff000) == Imm) { 1364280031Sdim ShiftImm = 12; 1365280031Sdim Imm >>= 12; 1366280031Sdim } else 1367280031Sdim return 0; 1368280031Sdim 1369280031Sdim static const unsigned OpcTable[2][2][2] = { 1370280031Sdim { { AArch64::SUBWri, AArch64::SUBXri }, 1371280031Sdim { AArch64::ADDWri, AArch64::ADDXri } }, 1372280031Sdim { { AArch64::SUBSWri, AArch64::SUBSXri }, 1373280031Sdim { AArch64::ADDSWri, AArch64::ADDSXri } } 1374280031Sdim }; 1375280031Sdim bool Is64Bit = RetVT == MVT::i64; 1376280031Sdim unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit]; 1377274955Ssvnmir const TargetRegisterClass *RC; 1378280031Sdim if (SetFlags) 1379280031Sdim RC = Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass; 1380280031Sdim else 1381280031Sdim RC = Is64Bit ? &AArch64::GPR64spRegClass : &AArch64::GPR32spRegClass; 1382280031Sdim unsigned ResultReg; 1383280031Sdim if (WantResult) 1384280031Sdim ResultReg = createResultReg(RC); 1385280031Sdim else 1386280031Sdim ResultReg = Is64Bit ? AArch64::XZR : AArch64::WZR; 1387280031Sdim 1388280031Sdim const MCInstrDesc &II = TII.get(Opc); 1389280031Sdim LHSReg = constrainOperandRegClass(II, LHSReg, II.getNumDefs()); 1390280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg) 1391280031Sdim .addReg(LHSReg, getKillRegState(LHSIsKill)) 1392280031Sdim .addImm(Imm) 1393280031Sdim .addImm(getShifterImm(AArch64_AM::LSL, ShiftImm)); 1394280031Sdim return ResultReg; 1395280031Sdim} 1396280031Sdim 1397280031Sdimunsigned AArch64FastISel::emitAddSub_rs(bool UseAdd, MVT RetVT, unsigned LHSReg, 1398280031Sdim bool LHSIsKill, unsigned RHSReg, 1399280031Sdim bool RHSIsKill, 1400280031Sdim AArch64_AM::ShiftExtendType ShiftType, 1401280031Sdim uint64_t ShiftImm, bool SetFlags, 1402280031Sdim bool WantResult) { 1403280031Sdim assert(LHSReg && RHSReg && "Invalid register number."); 1404321369Sdim assert(LHSReg != AArch64::SP && LHSReg != AArch64::WSP && 1405321369Sdim RHSReg != AArch64::SP && RHSReg != AArch64::WSP); 1406280031Sdim 1407280031Sdim if (RetVT != MVT::i32 && RetVT != MVT::i64) 1408280031Sdim return 0; 1409280031Sdim 1410296417Sdim // Don't deal with undefined shifts. 1411296417Sdim if (ShiftImm >= RetVT.getSizeInBits()) 1412296417Sdim return 0; 1413296417Sdim 1414280031Sdim static const unsigned OpcTable[2][2][2] = { 1415280031Sdim { { AArch64::SUBWrs, AArch64::SUBXrs }, 1416280031Sdim { AArch64::ADDWrs, AArch64::ADDXrs } }, 1417280031Sdim { { AArch64::SUBSWrs, AArch64::SUBSXrs }, 1418280031Sdim { AArch64::ADDSWrs, AArch64::ADDSXrs } } 1419280031Sdim }; 1420280031Sdim bool Is64Bit = RetVT == MVT::i64; 1421280031Sdim unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit]; 1422280031Sdim const TargetRegisterClass *RC = 1423280031Sdim Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass; 1424280031Sdim unsigned ResultReg; 1425280031Sdim if (WantResult) 1426280031Sdim ResultReg = createResultReg(RC); 1427280031Sdim else 1428280031Sdim ResultReg = Is64Bit ? AArch64::XZR : AArch64::WZR; 1429280031Sdim 1430280031Sdim const MCInstrDesc &II = TII.get(Opc); 1431280031Sdim LHSReg = constrainOperandRegClass(II, LHSReg, II.getNumDefs()); 1432280031Sdim RHSReg = constrainOperandRegClass(II, RHSReg, II.getNumDefs() + 1); 1433280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg) 1434280031Sdim .addReg(LHSReg, getKillRegState(LHSIsKill)) 1435280031Sdim .addReg(RHSReg, getKillRegState(RHSIsKill)) 1436280031Sdim .addImm(getShifterImm(ShiftType, ShiftImm)); 1437280031Sdim return ResultReg; 1438280031Sdim} 1439280031Sdim 1440280031Sdimunsigned AArch64FastISel::emitAddSub_rx(bool UseAdd, MVT RetVT, unsigned LHSReg, 1441280031Sdim bool LHSIsKill, unsigned RHSReg, 1442280031Sdim bool RHSIsKill, 1443280031Sdim AArch64_AM::ShiftExtendType ExtType, 1444280031Sdim uint64_t ShiftImm, bool SetFlags, 1445280031Sdim bool WantResult) { 1446280031Sdim assert(LHSReg && RHSReg && "Invalid register number."); 1447321369Sdim assert(LHSReg != AArch64::XZR && LHSReg != AArch64::WZR && 1448321369Sdim RHSReg != AArch64::XZR && RHSReg != AArch64::WZR); 1449280031Sdim 1450280031Sdim if (RetVT != MVT::i32 && RetVT != MVT::i64) 1451280031Sdim return 0; 1452280031Sdim 1453296417Sdim if (ShiftImm >= 4) 1454296417Sdim return 0; 1455296417Sdim 1456280031Sdim static const unsigned OpcTable[2][2][2] = { 1457280031Sdim { { AArch64::SUBWrx, AArch64::SUBXrx }, 1458280031Sdim { AArch64::ADDWrx, AArch64::ADDXrx } }, 1459280031Sdim { { AArch64::SUBSWrx, AArch64::SUBSXrx }, 1460280031Sdim { AArch64::ADDSWrx, AArch64::ADDSXrx } } 1461280031Sdim }; 1462280031Sdim bool Is64Bit = RetVT == MVT::i64; 1463280031Sdim unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit]; 1464280031Sdim const TargetRegisterClass *RC = nullptr; 1465280031Sdim if (SetFlags) 1466280031Sdim RC = Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass; 1467280031Sdim else 1468280031Sdim RC = Is64Bit ? &AArch64::GPR64spRegClass : &AArch64::GPR32spRegClass; 1469280031Sdim unsigned ResultReg; 1470280031Sdim if (WantResult) 1471280031Sdim ResultReg = createResultReg(RC); 1472280031Sdim else 1473280031Sdim ResultReg = Is64Bit ? AArch64::XZR : AArch64::WZR; 1474280031Sdim 1475280031Sdim const MCInstrDesc &II = TII.get(Opc); 1476280031Sdim LHSReg = constrainOperandRegClass(II, LHSReg, II.getNumDefs()); 1477280031Sdim RHSReg = constrainOperandRegClass(II, RHSReg, II.getNumDefs() + 1); 1478280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg) 1479280031Sdim .addReg(LHSReg, getKillRegState(LHSIsKill)) 1480280031Sdim .addReg(RHSReg, getKillRegState(RHSIsKill)) 1481280031Sdim .addImm(getArithExtendImm(ExtType, ShiftImm)); 1482280031Sdim return ResultReg; 1483280031Sdim} 1484280031Sdim 1485280031Sdimbool AArch64FastISel::emitCmp(const Value *LHS, const Value *RHS, bool IsZExt) { 1486280031Sdim Type *Ty = LHS->getType(); 1487288943Sdim EVT EVT = TLI.getValueType(DL, Ty, true); 1488280031Sdim if (!EVT.isSimple()) 1489280031Sdim return false; 1490280031Sdim MVT VT = EVT.getSimpleVT(); 1491280031Sdim 1492274955Ssvnmir switch (VT.SimpleTy) { 1493274955Ssvnmir default: 1494274955Ssvnmir return false; 1495274955Ssvnmir case MVT::i1: 1496274955Ssvnmir case MVT::i8: 1497280031Sdim case MVT::i16: 1498280031Sdim case MVT::i32: 1499280031Sdim case MVT::i64: 1500280031Sdim return emitICmp(VT, LHS, RHS, IsZExt); 1501280031Sdim case MVT::f32: 1502280031Sdim case MVT::f64: 1503280031Sdim return emitFCmp(VT, LHS, RHS); 1504280031Sdim } 1505280031Sdim} 1506280031Sdim 1507280031Sdimbool AArch64FastISel::emitICmp(MVT RetVT, const Value *LHS, const Value *RHS, 1508280031Sdim bool IsZExt) { 1509280031Sdim return emitSub(RetVT, LHS, RHS, /*SetFlags=*/true, /*WantResult=*/false, 1510280031Sdim IsZExt) != 0; 1511280031Sdim} 1512280031Sdim 1513280031Sdimbool AArch64FastISel::emitICmp_ri(MVT RetVT, unsigned LHSReg, bool LHSIsKill, 1514280031Sdim uint64_t Imm) { 1515280031Sdim return emitAddSub_ri(/*UseAdd=*/false, RetVT, LHSReg, LHSIsKill, Imm, 1516280031Sdim /*SetFlags=*/true, /*WantResult=*/false) != 0; 1517280031Sdim} 1518280031Sdim 1519280031Sdimbool AArch64FastISel::emitFCmp(MVT RetVT, const Value *LHS, const Value *RHS) { 1520280031Sdim if (RetVT != MVT::f32 && RetVT != MVT::f64) 1521280031Sdim return false; 1522280031Sdim 1523280031Sdim // Check to see if the 2nd operand is a constant that we can encode directly 1524280031Sdim // in the compare. 1525280031Sdim bool UseImm = false; 1526280031Sdim if (const auto *CFP = dyn_cast<ConstantFP>(RHS)) 1527280031Sdim if (CFP->isZero() && !CFP->isNegative()) 1528280031Sdim UseImm = true; 1529280031Sdim 1530280031Sdim unsigned LHSReg = getRegForValue(LHS); 1531280031Sdim if (!LHSReg) 1532280031Sdim return false; 1533280031Sdim bool LHSIsKill = hasTrivialKill(LHS); 1534280031Sdim 1535280031Sdim if (UseImm) { 1536280031Sdim unsigned Opc = (RetVT == MVT::f64) ? AArch64::FCMPDri : AArch64::FCMPSri; 1537280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc)) 1538280031Sdim .addReg(LHSReg, getKillRegState(LHSIsKill)); 1539280031Sdim return true; 1540280031Sdim } 1541280031Sdim 1542280031Sdim unsigned RHSReg = getRegForValue(RHS); 1543280031Sdim if (!RHSReg) 1544280031Sdim return false; 1545280031Sdim bool RHSIsKill = hasTrivialKill(RHS); 1546280031Sdim 1547280031Sdim unsigned Opc = (RetVT == MVT::f64) ? AArch64::FCMPDrr : AArch64::FCMPSrr; 1548280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc)) 1549280031Sdim .addReg(LHSReg, getKillRegState(LHSIsKill)) 1550280031Sdim .addReg(RHSReg, getKillRegState(RHSIsKill)); 1551280031Sdim return true; 1552280031Sdim} 1553280031Sdim 1554280031Sdimunsigned AArch64FastISel::emitAdd(MVT RetVT, const Value *LHS, const Value *RHS, 1555280031Sdim bool SetFlags, bool WantResult, bool IsZExt) { 1556280031Sdim return emitAddSub(/*UseAdd=*/true, RetVT, LHS, RHS, SetFlags, WantResult, 1557280031Sdim IsZExt); 1558280031Sdim} 1559280031Sdim 1560341825Sdim/// This method is a wrapper to simplify add emission. 1561280031Sdim/// 1562280031Sdim/// First try to emit an add with an immediate operand using emitAddSub_ri. If 1563280031Sdim/// that fails, then try to materialize the immediate into a register and use 1564280031Sdim/// emitAddSub_rr instead. 1565280031Sdimunsigned AArch64FastISel::emitAdd_ri_(MVT VT, unsigned Op0, bool Op0IsKill, 1566280031Sdim int64_t Imm) { 1567280031Sdim unsigned ResultReg; 1568280031Sdim if (Imm < 0) 1569280031Sdim ResultReg = emitAddSub_ri(false, VT, Op0, Op0IsKill, -Imm); 1570280031Sdim else 1571280031Sdim ResultReg = emitAddSub_ri(true, VT, Op0, Op0IsKill, Imm); 1572280031Sdim 1573280031Sdim if (ResultReg) 1574280031Sdim return ResultReg; 1575280031Sdim 1576280031Sdim unsigned CReg = fastEmit_i(VT, VT, ISD::Constant, Imm); 1577280031Sdim if (!CReg) 1578280031Sdim return 0; 1579280031Sdim 1580280031Sdim ResultReg = emitAddSub_rr(true, VT, Op0, Op0IsKill, CReg, true); 1581280031Sdim return ResultReg; 1582280031Sdim} 1583280031Sdim 1584280031Sdimunsigned AArch64FastISel::emitSub(MVT RetVT, const Value *LHS, const Value *RHS, 1585280031Sdim bool SetFlags, bool WantResult, bool IsZExt) { 1586280031Sdim return emitAddSub(/*UseAdd=*/false, RetVT, LHS, RHS, SetFlags, WantResult, 1587280031Sdim IsZExt); 1588280031Sdim} 1589280031Sdim 1590280031Sdimunsigned AArch64FastISel::emitSubs_rr(MVT RetVT, unsigned LHSReg, 1591280031Sdim bool LHSIsKill, unsigned RHSReg, 1592280031Sdim bool RHSIsKill, bool WantResult) { 1593280031Sdim return emitAddSub_rr(/*UseAdd=*/false, RetVT, LHSReg, LHSIsKill, RHSReg, 1594280031Sdim RHSIsKill, /*SetFlags=*/true, WantResult); 1595280031Sdim} 1596280031Sdim 1597280031Sdimunsigned AArch64FastISel::emitSubs_rs(MVT RetVT, unsigned LHSReg, 1598280031Sdim bool LHSIsKill, unsigned RHSReg, 1599280031Sdim bool RHSIsKill, 1600280031Sdim AArch64_AM::ShiftExtendType ShiftType, 1601280031Sdim uint64_t ShiftImm, bool WantResult) { 1602280031Sdim return emitAddSub_rs(/*UseAdd=*/false, RetVT, LHSReg, LHSIsKill, RHSReg, 1603280031Sdim RHSIsKill, ShiftType, ShiftImm, /*SetFlags=*/true, 1604280031Sdim WantResult); 1605280031Sdim} 1606280031Sdim 1607280031Sdimunsigned AArch64FastISel::emitLogicalOp(unsigned ISDOpc, MVT RetVT, 1608280031Sdim const Value *LHS, const Value *RHS) { 1609280031Sdim // Canonicalize immediates to the RHS first. 1610280031Sdim if (isa<ConstantInt>(LHS) && !isa<ConstantInt>(RHS)) 1611280031Sdim std::swap(LHS, RHS); 1612280031Sdim 1613280031Sdim // Canonicalize mul by power-of-2 to the RHS. 1614280031Sdim if (LHS->hasOneUse() && isValueAvailable(LHS)) 1615280031Sdim if (isMulPowOf2(LHS)) 1616280031Sdim std::swap(LHS, RHS); 1617280031Sdim 1618280031Sdim // Canonicalize shift immediate to the RHS. 1619280031Sdim if (LHS->hasOneUse() && isValueAvailable(LHS)) 1620280031Sdim if (const auto *SI = dyn_cast<ShlOperator>(LHS)) 1621280031Sdim if (isa<ConstantInt>(SI->getOperand(1))) 1622280031Sdim std::swap(LHS, RHS); 1623280031Sdim 1624280031Sdim unsigned LHSReg = getRegForValue(LHS); 1625280031Sdim if (!LHSReg) 1626280031Sdim return 0; 1627280031Sdim bool LHSIsKill = hasTrivialKill(LHS); 1628280031Sdim 1629280031Sdim unsigned ResultReg = 0; 1630280031Sdim if (const auto *C = dyn_cast<ConstantInt>(RHS)) { 1631280031Sdim uint64_t Imm = C->getZExtValue(); 1632280031Sdim ResultReg = emitLogicalOp_ri(ISDOpc, RetVT, LHSReg, LHSIsKill, Imm); 1633280031Sdim } 1634280031Sdim if (ResultReg) 1635280031Sdim return ResultReg; 1636280031Sdim 1637280031Sdim // Check if the mul can be folded into the instruction. 1638296417Sdim if (RHS->hasOneUse() && isValueAvailable(RHS)) { 1639280031Sdim if (isMulPowOf2(RHS)) { 1640280031Sdim const Value *MulLHS = cast<MulOperator>(RHS)->getOperand(0); 1641280031Sdim const Value *MulRHS = cast<MulOperator>(RHS)->getOperand(1); 1642280031Sdim 1643280031Sdim if (const auto *C = dyn_cast<ConstantInt>(MulLHS)) 1644280031Sdim if (C->getValue().isPowerOf2()) 1645280031Sdim std::swap(MulLHS, MulRHS); 1646280031Sdim 1647280031Sdim assert(isa<ConstantInt>(MulRHS) && "Expected a ConstantInt."); 1648280031Sdim uint64_t ShiftVal = cast<ConstantInt>(MulRHS)->getValue().logBase2(); 1649280031Sdim 1650280031Sdim unsigned RHSReg = getRegForValue(MulLHS); 1651280031Sdim if (!RHSReg) 1652280031Sdim return 0; 1653280031Sdim bool RHSIsKill = hasTrivialKill(MulLHS); 1654296417Sdim ResultReg = emitLogicalOp_rs(ISDOpc, RetVT, LHSReg, LHSIsKill, RHSReg, 1655296417Sdim RHSIsKill, ShiftVal); 1656296417Sdim if (ResultReg) 1657296417Sdim return ResultReg; 1658280031Sdim } 1659296417Sdim } 1660280031Sdim 1661280031Sdim // Check if the shift can be folded into the instruction. 1662296417Sdim if (RHS->hasOneUse() && isValueAvailable(RHS)) { 1663280031Sdim if (const auto *SI = dyn_cast<ShlOperator>(RHS)) 1664280031Sdim if (const auto *C = dyn_cast<ConstantInt>(SI->getOperand(1))) { 1665280031Sdim uint64_t ShiftVal = C->getZExtValue(); 1666280031Sdim unsigned RHSReg = getRegForValue(SI->getOperand(0)); 1667280031Sdim if (!RHSReg) 1668280031Sdim return 0; 1669280031Sdim bool RHSIsKill = hasTrivialKill(SI->getOperand(0)); 1670296417Sdim ResultReg = emitLogicalOp_rs(ISDOpc, RetVT, LHSReg, LHSIsKill, RHSReg, 1671296417Sdim RHSIsKill, ShiftVal); 1672296417Sdim if (ResultReg) 1673296417Sdim return ResultReg; 1674280031Sdim } 1675296417Sdim } 1676280031Sdim 1677280031Sdim unsigned RHSReg = getRegForValue(RHS); 1678280031Sdim if (!RHSReg) 1679280031Sdim return 0; 1680280031Sdim bool RHSIsKill = hasTrivialKill(RHS); 1681280031Sdim 1682280031Sdim MVT VT = std::max(MVT::i32, RetVT.SimpleTy); 1683280031Sdim ResultReg = fastEmit_rr(VT, VT, ISDOpc, LHSReg, LHSIsKill, RHSReg, RHSIsKill); 1684280031Sdim if (RetVT >= MVT::i8 && RetVT <= MVT::i16) { 1685280031Sdim uint64_t Mask = (RetVT == MVT::i8) ? 0xff : 0xffff; 1686280031Sdim ResultReg = emitAnd_ri(MVT::i32, ResultReg, /*IsKill=*/true, Mask); 1687280031Sdim } 1688280031Sdim return ResultReg; 1689280031Sdim} 1690280031Sdim 1691280031Sdimunsigned AArch64FastISel::emitLogicalOp_ri(unsigned ISDOpc, MVT RetVT, 1692280031Sdim unsigned LHSReg, bool LHSIsKill, 1693280031Sdim uint64_t Imm) { 1694309124Sdim static_assert((ISD::AND + 1 == ISD::OR) && (ISD::AND + 2 == ISD::XOR), 1695309124Sdim "ISD nodes are not consecutive!"); 1696280031Sdim static const unsigned OpcTable[3][2] = { 1697280031Sdim { AArch64::ANDWri, AArch64::ANDXri }, 1698280031Sdim { AArch64::ORRWri, AArch64::ORRXri }, 1699280031Sdim { AArch64::EORWri, AArch64::EORXri } 1700280031Sdim }; 1701280031Sdim const TargetRegisterClass *RC; 1702280031Sdim unsigned Opc; 1703280031Sdim unsigned RegSize; 1704280031Sdim switch (RetVT.SimpleTy) { 1705280031Sdim default: 1706280031Sdim return 0; 1707280031Sdim case MVT::i1: 1708280031Sdim case MVT::i8: 1709280031Sdim case MVT::i16: 1710280031Sdim case MVT::i32: { 1711280031Sdim unsigned Idx = ISDOpc - ISD::AND; 1712280031Sdim Opc = OpcTable[Idx][0]; 1713280031Sdim RC = &AArch64::GPR32spRegClass; 1714280031Sdim RegSize = 32; 1715280031Sdim break; 1716280031Sdim } 1717280031Sdim case MVT::i64: 1718280031Sdim Opc = OpcTable[ISDOpc - ISD::AND][1]; 1719280031Sdim RC = &AArch64::GPR64spRegClass; 1720280031Sdim RegSize = 64; 1721280031Sdim break; 1722280031Sdim } 1723280031Sdim 1724280031Sdim if (!AArch64_AM::isLogicalImmediate(Imm, RegSize)) 1725280031Sdim return 0; 1726280031Sdim 1727280031Sdim unsigned ResultReg = 1728280031Sdim fastEmitInst_ri(Opc, RC, LHSReg, LHSIsKill, 1729280031Sdim AArch64_AM::encodeLogicalImmediate(Imm, RegSize)); 1730280031Sdim if (RetVT >= MVT::i8 && RetVT <= MVT::i16 && ISDOpc != ISD::AND) { 1731280031Sdim uint64_t Mask = (RetVT == MVT::i8) ? 0xff : 0xffff; 1732280031Sdim ResultReg = emitAnd_ri(MVT::i32, ResultReg, /*IsKill=*/true, Mask); 1733280031Sdim } 1734280031Sdim return ResultReg; 1735280031Sdim} 1736280031Sdim 1737280031Sdimunsigned AArch64FastISel::emitLogicalOp_rs(unsigned ISDOpc, MVT RetVT, 1738280031Sdim unsigned LHSReg, bool LHSIsKill, 1739280031Sdim unsigned RHSReg, bool RHSIsKill, 1740280031Sdim uint64_t ShiftImm) { 1741309124Sdim static_assert((ISD::AND + 1 == ISD::OR) && (ISD::AND + 2 == ISD::XOR), 1742309124Sdim "ISD nodes are not consecutive!"); 1743280031Sdim static const unsigned OpcTable[3][2] = { 1744280031Sdim { AArch64::ANDWrs, AArch64::ANDXrs }, 1745280031Sdim { AArch64::ORRWrs, AArch64::ORRXrs }, 1746280031Sdim { AArch64::EORWrs, AArch64::EORXrs } 1747280031Sdim }; 1748296417Sdim 1749296417Sdim // Don't deal with undefined shifts. 1750296417Sdim if (ShiftImm >= RetVT.getSizeInBits()) 1751296417Sdim return 0; 1752296417Sdim 1753280031Sdim const TargetRegisterClass *RC; 1754280031Sdim unsigned Opc; 1755280031Sdim switch (RetVT.SimpleTy) { 1756280031Sdim default: 1757280031Sdim return 0; 1758280031Sdim case MVT::i1: 1759280031Sdim case MVT::i8: 1760280031Sdim case MVT::i16: 1761280031Sdim case MVT::i32: 1762280031Sdim Opc = OpcTable[ISDOpc - ISD::AND][0]; 1763274955Ssvnmir RC = &AArch64::GPR32RegClass; 1764280031Sdim break; 1765280031Sdim case MVT::i64: 1766280031Sdim Opc = OpcTable[ISDOpc - ISD::AND][1]; 1767280031Sdim RC = &AArch64::GPR64RegClass; 1768280031Sdim break; 1769280031Sdim } 1770280031Sdim unsigned ResultReg = 1771280031Sdim fastEmitInst_rri(Opc, RC, LHSReg, LHSIsKill, RHSReg, RHSIsKill, 1772280031Sdim AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftImm)); 1773280031Sdim if (RetVT >= MVT::i8 && RetVT <= MVT::i16) { 1774280031Sdim uint64_t Mask = (RetVT == MVT::i8) ? 0xff : 0xffff; 1775280031Sdim ResultReg = emitAnd_ri(MVT::i32, ResultReg, /*IsKill=*/true, Mask); 1776280031Sdim } 1777280031Sdim return ResultReg; 1778280031Sdim} 1779280031Sdim 1780280031Sdimunsigned AArch64FastISel::emitAnd_ri(MVT RetVT, unsigned LHSReg, bool LHSIsKill, 1781280031Sdim uint64_t Imm) { 1782280031Sdim return emitLogicalOp_ri(ISD::AND, RetVT, LHSReg, LHSIsKill, Imm); 1783280031Sdim} 1784280031Sdim 1785280031Sdimunsigned AArch64FastISel::emitLoad(MVT VT, MVT RetVT, Address Addr, 1786280031Sdim bool WantZExt, MachineMemOperand *MMO) { 1787288943Sdim if (!TLI.allowsMisalignedMemoryAccesses(VT)) 1788288943Sdim return 0; 1789288943Sdim 1790280031Sdim // Simplify this down to something we can handle. 1791280031Sdim if (!simplifyAddress(Addr, VT)) 1792280031Sdim return 0; 1793280031Sdim 1794280031Sdim unsigned ScaleFactor = getImplicitScaleFactor(VT); 1795280031Sdim if (!ScaleFactor) 1796280031Sdim llvm_unreachable("Unexpected value type."); 1797280031Sdim 1798280031Sdim // Negative offsets require unscaled, 9-bit, signed immediate offsets. 1799280031Sdim // Otherwise, we try using scaled, 12-bit, unsigned immediate offsets. 1800280031Sdim bool UseScaled = true; 1801280031Sdim if ((Addr.getOffset() < 0) || (Addr.getOffset() & (ScaleFactor - 1))) { 1802280031Sdim UseScaled = false; 1803274955Ssvnmir ScaleFactor = 1; 1804280031Sdim } 1805280031Sdim 1806280031Sdim static const unsigned GPOpcTable[2][8][4] = { 1807280031Sdim // Sign-extend. 1808280031Sdim { { AArch64::LDURSBWi, AArch64::LDURSHWi, AArch64::LDURWi, 1809280031Sdim AArch64::LDURXi }, 1810280031Sdim { AArch64::LDURSBXi, AArch64::LDURSHXi, AArch64::LDURSWi, 1811280031Sdim AArch64::LDURXi }, 1812280031Sdim { AArch64::LDRSBWui, AArch64::LDRSHWui, AArch64::LDRWui, 1813280031Sdim AArch64::LDRXui }, 1814280031Sdim { AArch64::LDRSBXui, AArch64::LDRSHXui, AArch64::LDRSWui, 1815280031Sdim AArch64::LDRXui }, 1816280031Sdim { AArch64::LDRSBWroX, AArch64::LDRSHWroX, AArch64::LDRWroX, 1817280031Sdim AArch64::LDRXroX }, 1818280031Sdim { AArch64::LDRSBXroX, AArch64::LDRSHXroX, AArch64::LDRSWroX, 1819280031Sdim AArch64::LDRXroX }, 1820280031Sdim { AArch64::LDRSBWroW, AArch64::LDRSHWroW, AArch64::LDRWroW, 1821280031Sdim AArch64::LDRXroW }, 1822280031Sdim { AArch64::LDRSBXroW, AArch64::LDRSHXroW, AArch64::LDRSWroW, 1823280031Sdim AArch64::LDRXroW } 1824280031Sdim }, 1825280031Sdim // Zero-extend. 1826280031Sdim { { AArch64::LDURBBi, AArch64::LDURHHi, AArch64::LDURWi, 1827280031Sdim AArch64::LDURXi }, 1828280031Sdim { AArch64::LDURBBi, AArch64::LDURHHi, AArch64::LDURWi, 1829280031Sdim AArch64::LDURXi }, 1830280031Sdim { AArch64::LDRBBui, AArch64::LDRHHui, AArch64::LDRWui, 1831280031Sdim AArch64::LDRXui }, 1832280031Sdim { AArch64::LDRBBui, AArch64::LDRHHui, AArch64::LDRWui, 1833280031Sdim AArch64::LDRXui }, 1834280031Sdim { AArch64::LDRBBroX, AArch64::LDRHHroX, AArch64::LDRWroX, 1835280031Sdim AArch64::LDRXroX }, 1836280031Sdim { AArch64::LDRBBroX, AArch64::LDRHHroX, AArch64::LDRWroX, 1837280031Sdim AArch64::LDRXroX }, 1838280031Sdim { AArch64::LDRBBroW, AArch64::LDRHHroW, AArch64::LDRWroW, 1839280031Sdim AArch64::LDRXroW }, 1840280031Sdim { AArch64::LDRBBroW, AArch64::LDRHHroW, AArch64::LDRWroW, 1841280031Sdim AArch64::LDRXroW } 1842280031Sdim } 1843280031Sdim }; 1844280031Sdim 1845280031Sdim static const unsigned FPOpcTable[4][2] = { 1846280031Sdim { AArch64::LDURSi, AArch64::LDURDi }, 1847280031Sdim { AArch64::LDRSui, AArch64::LDRDui }, 1848280031Sdim { AArch64::LDRSroX, AArch64::LDRDroX }, 1849280031Sdim { AArch64::LDRSroW, AArch64::LDRDroW } 1850280031Sdim }; 1851280031Sdim 1852280031Sdim unsigned Opc; 1853280031Sdim const TargetRegisterClass *RC; 1854280031Sdim bool UseRegOffset = Addr.isRegBase() && !Addr.getOffset() && Addr.getReg() && 1855280031Sdim Addr.getOffsetReg(); 1856280031Sdim unsigned Idx = UseRegOffset ? 2 : UseScaled ? 1 : 0; 1857280031Sdim if (Addr.getExtendType() == AArch64_AM::UXTW || 1858280031Sdim Addr.getExtendType() == AArch64_AM::SXTW) 1859280031Sdim Idx++; 1860280031Sdim 1861280031Sdim bool IsRet64Bit = RetVT == MVT::i64; 1862280031Sdim switch (VT.SimpleTy) { 1863280031Sdim default: 1864280031Sdim llvm_unreachable("Unexpected value type."); 1865280031Sdim case MVT::i1: // Intentional fall-through. 1866280031Sdim case MVT::i8: 1867280031Sdim Opc = GPOpcTable[WantZExt][2 * Idx + IsRet64Bit][0]; 1868280031Sdim RC = (IsRet64Bit && !WantZExt) ? 1869280031Sdim &AArch64::GPR64RegClass: &AArch64::GPR32RegClass; 1870274955Ssvnmir break; 1871274955Ssvnmir case MVT::i16: 1872280031Sdim Opc = GPOpcTable[WantZExt][2 * Idx + IsRet64Bit][1]; 1873280031Sdim RC = (IsRet64Bit && !WantZExt) ? 1874280031Sdim &AArch64::GPR64RegClass: &AArch64::GPR32RegClass; 1875274955Ssvnmir break; 1876274955Ssvnmir case MVT::i32: 1877280031Sdim Opc = GPOpcTable[WantZExt][2 * Idx + IsRet64Bit][2]; 1878280031Sdim RC = (IsRet64Bit && !WantZExt) ? 1879280031Sdim &AArch64::GPR64RegClass: &AArch64::GPR32RegClass; 1880274955Ssvnmir break; 1881274955Ssvnmir case MVT::i64: 1882280031Sdim Opc = GPOpcTable[WantZExt][2 * Idx + IsRet64Bit][3]; 1883274955Ssvnmir RC = &AArch64::GPR64RegClass; 1884274955Ssvnmir break; 1885274955Ssvnmir case MVT::f32: 1886280031Sdim Opc = FPOpcTable[Idx][0]; 1887280031Sdim RC = &AArch64::FPR32RegClass; 1888274955Ssvnmir break; 1889274955Ssvnmir case MVT::f64: 1890280031Sdim Opc = FPOpcTable[Idx][1]; 1891280031Sdim RC = &AArch64::FPR64RegClass; 1892274955Ssvnmir break; 1893274955Ssvnmir } 1894274955Ssvnmir 1895274955Ssvnmir // Create the base instruction, then add the operands. 1896280031Sdim unsigned ResultReg = createResultReg(RC); 1897274955Ssvnmir MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, 1898274955Ssvnmir TII.get(Opc), ResultReg); 1899280031Sdim addLoadStoreOperands(Addr, MIB, MachineMemOperand::MOLoad, ScaleFactor, MMO); 1900274955Ssvnmir 1901274955Ssvnmir // Loading an i1 requires special handling. 1902280031Sdim if (VT == MVT::i1) { 1903280031Sdim unsigned ANDReg = emitAnd_ri(MVT::i32, ResultReg, /*IsKill=*/true, 1); 1904280031Sdim assert(ANDReg && "Unexpected AND instruction emission failure."); 1905274955Ssvnmir ResultReg = ANDReg; 1906274955Ssvnmir } 1907280031Sdim 1908280031Sdim // For zero-extending loads to 64bit we emit a 32bit load and then convert 1909280031Sdim // the 32bit reg to a 64bit reg. 1910280031Sdim if (WantZExt && RetVT == MVT::i64 && VT <= MVT::i32) { 1911280031Sdim unsigned Reg64 = createResultReg(&AArch64::GPR64RegClass); 1912280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, 1913280031Sdim TII.get(AArch64::SUBREG_TO_REG), Reg64) 1914280031Sdim .addImm(0) 1915280031Sdim .addReg(ResultReg, getKillRegState(true)) 1916280031Sdim .addImm(AArch64::sub_32); 1917280031Sdim ResultReg = Reg64; 1918280031Sdim } 1919280031Sdim return ResultReg; 1920280031Sdim} 1921280031Sdim 1922280031Sdimbool AArch64FastISel::selectAddSub(const Instruction *I) { 1923280031Sdim MVT VT; 1924280031Sdim if (!isTypeSupported(I->getType(), VT, /*IsVectorAllowed=*/true)) 1925280031Sdim return false; 1926280031Sdim 1927280031Sdim if (VT.isVector()) 1928280031Sdim return selectOperator(I, I->getOpcode()); 1929280031Sdim 1930280031Sdim unsigned ResultReg; 1931280031Sdim switch (I->getOpcode()) { 1932280031Sdim default: 1933280031Sdim llvm_unreachable("Unexpected instruction."); 1934280031Sdim case Instruction::Add: 1935280031Sdim ResultReg = emitAdd(VT, I->getOperand(0), I->getOperand(1)); 1936280031Sdim break; 1937280031Sdim case Instruction::Sub: 1938280031Sdim ResultReg = emitSub(VT, I->getOperand(0), I->getOperand(1)); 1939280031Sdim break; 1940280031Sdim } 1941280031Sdim if (!ResultReg) 1942280031Sdim return false; 1943280031Sdim 1944280031Sdim updateValueMap(I, ResultReg); 1945274955Ssvnmir return true; 1946274955Ssvnmir} 1947274955Ssvnmir 1948280031Sdimbool AArch64FastISel::selectLogicalOp(const Instruction *I) { 1949274955Ssvnmir MVT VT; 1950280031Sdim if (!isTypeSupported(I->getType(), VT, /*IsVectorAllowed=*/true)) 1951280031Sdim return false; 1952280031Sdim 1953280031Sdim if (VT.isVector()) 1954280031Sdim return selectOperator(I, I->getOpcode()); 1955280031Sdim 1956280031Sdim unsigned ResultReg; 1957280031Sdim switch (I->getOpcode()) { 1958280031Sdim default: 1959280031Sdim llvm_unreachable("Unexpected instruction."); 1960280031Sdim case Instruction::And: 1961280031Sdim ResultReg = emitLogicalOp(ISD::AND, VT, I->getOperand(0), I->getOperand(1)); 1962280031Sdim break; 1963280031Sdim case Instruction::Or: 1964280031Sdim ResultReg = emitLogicalOp(ISD::OR, VT, I->getOperand(0), I->getOperand(1)); 1965280031Sdim break; 1966280031Sdim case Instruction::Xor: 1967280031Sdim ResultReg = emitLogicalOp(ISD::XOR, VT, I->getOperand(0), I->getOperand(1)); 1968280031Sdim break; 1969280031Sdim } 1970280031Sdim if (!ResultReg) 1971280031Sdim return false; 1972280031Sdim 1973280031Sdim updateValueMap(I, ResultReg); 1974280031Sdim return true; 1975280031Sdim} 1976280031Sdim 1977280031Sdimbool AArch64FastISel::selectLoad(const Instruction *I) { 1978280031Sdim MVT VT; 1979274955Ssvnmir // Verify we have a legal type before going any further. Currently, we handle 1980274955Ssvnmir // simple types that will directly fit in a register (i32/f32/i64/f64) or 1981274955Ssvnmir // those that can be sign or zero-extended to a basic operation (i1/i8/i16). 1982280031Sdim if (!isTypeSupported(I->getType(), VT, /*IsVectorAllowed=*/true) || 1983280031Sdim cast<LoadInst>(I)->isAtomic()) 1984274955Ssvnmir return false; 1985274955Ssvnmir 1986309124Sdim const Value *SV = I->getOperand(0); 1987309124Sdim if (TLI.supportSwiftError()) { 1988309124Sdim // Swifterror values can come from either a function parameter with 1989309124Sdim // swifterror attribute or an alloca with swifterror attribute. 1990309124Sdim if (const Argument *Arg = dyn_cast<Argument>(SV)) { 1991309124Sdim if (Arg->hasSwiftErrorAttr()) 1992309124Sdim return false; 1993309124Sdim } 1994309124Sdim 1995309124Sdim if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) { 1996309124Sdim if (Alloca->isSwiftError()) 1997309124Sdim return false; 1998309124Sdim } 1999309124Sdim } 2000309124Sdim 2001274955Ssvnmir // See if we can handle this address. 2002274955Ssvnmir Address Addr; 2003280031Sdim if (!computeAddress(I->getOperand(0), Addr, I->getType())) 2004274955Ssvnmir return false; 2005274955Ssvnmir 2006280031Sdim // Fold the following sign-/zero-extend into the load instruction. 2007280031Sdim bool WantZExt = true; 2008280031Sdim MVT RetVT = VT; 2009280031Sdim const Value *IntExtVal = nullptr; 2010280031Sdim if (I->hasOneUse()) { 2011280031Sdim if (const auto *ZE = dyn_cast<ZExtInst>(I->use_begin()->getUser())) { 2012280031Sdim if (isTypeSupported(ZE->getType(), RetVT)) 2013280031Sdim IntExtVal = ZE; 2014280031Sdim else 2015280031Sdim RetVT = VT; 2016280031Sdim } else if (const auto *SE = dyn_cast<SExtInst>(I->use_begin()->getUser())) { 2017280031Sdim if (isTypeSupported(SE->getType(), RetVT)) 2018280031Sdim IntExtVal = SE; 2019280031Sdim else 2020280031Sdim RetVT = VT; 2021280031Sdim WantZExt = false; 2022280031Sdim } 2023280031Sdim } 2024280031Sdim 2025280031Sdim unsigned ResultReg = 2026280031Sdim emitLoad(VT, RetVT, Addr, WantZExt, createMachineMemOperandFor(I)); 2027280031Sdim if (!ResultReg) 2028274955Ssvnmir return false; 2029274955Ssvnmir 2030280031Sdim // There are a few different cases we have to handle, because the load or the 2031280031Sdim // sign-/zero-extend might not be selected by FastISel if we fall-back to 2032280031Sdim // SelectionDAG. There is also an ordering issue when both instructions are in 2033280031Sdim // different basic blocks. 2034280031Sdim // 1.) The load instruction is selected by FastISel, but the integer extend 2035280031Sdim // not. This usually happens when the integer extend is in a different 2036280031Sdim // basic block and SelectionDAG took over for that basic block. 2037280031Sdim // 2.) The load instruction is selected before the integer extend. This only 2038280031Sdim // happens when the integer extend is in a different basic block. 2039280031Sdim // 3.) The load instruction is selected by SelectionDAG and the integer extend 2040280031Sdim // by FastISel. This happens if there are instructions between the load 2041280031Sdim // and the integer extend that couldn't be selected by FastISel. 2042280031Sdim if (IntExtVal) { 2043280031Sdim // The integer extend hasn't been emitted yet. FastISel or SelectionDAG 2044280031Sdim // could select it. Emit a copy to subreg if necessary. FastISel will remove 2045280031Sdim // it when it selects the integer extend. 2046280031Sdim unsigned Reg = lookUpRegForValue(IntExtVal); 2047288943Sdim auto *MI = MRI.getUniqueVRegDef(Reg); 2048288943Sdim if (!MI) { 2049280031Sdim if (RetVT == MVT::i64 && VT <= MVT::i32) { 2050280031Sdim if (WantZExt) { 2051280031Sdim // Delete the last emitted instruction from emitLoad (SUBREG_TO_REG). 2052344779Sdim MachineBasicBlock::iterator I(std::prev(FuncInfo.InsertPt)); 2053344779Sdim ResultReg = std::prev(I)->getOperand(0).getReg(); 2054344779Sdim removeDeadCode(I, std::next(I)); 2055280031Sdim } else 2056280031Sdim ResultReg = fastEmitInst_extractsubreg(MVT::i32, ResultReg, 2057280031Sdim /*IsKill=*/true, 2058280031Sdim AArch64::sub_32); 2059280031Sdim } 2060280031Sdim updateValueMap(I, ResultReg); 2061280031Sdim return true; 2062280031Sdim } 2063280031Sdim 2064280031Sdim // The integer extend has already been emitted - delete all the instructions 2065280031Sdim // that have been emitted by the integer extend lowering code and use the 2066280031Sdim // result from the load instruction directly. 2067288943Sdim while (MI) { 2068280031Sdim Reg = 0; 2069280031Sdim for (auto &Opnd : MI->uses()) { 2070280031Sdim if (Opnd.isReg()) { 2071280031Sdim Reg = Opnd.getReg(); 2072280031Sdim break; 2073280031Sdim } 2074280031Sdim } 2075344779Sdim MachineBasicBlock::iterator I(MI); 2076344779Sdim removeDeadCode(I, std::next(I)); 2077288943Sdim MI = nullptr; 2078288943Sdim if (Reg) 2079288943Sdim MI = MRI.getUniqueVRegDef(Reg); 2080280031Sdim } 2081280031Sdim updateValueMap(IntExtVal, ResultReg); 2082280031Sdim return true; 2083280031Sdim } 2084280031Sdim 2085280031Sdim updateValueMap(I, ResultReg); 2086274955Ssvnmir return true; 2087274955Ssvnmir} 2088274955Ssvnmir 2089314564Sdimbool AArch64FastISel::emitStoreRelease(MVT VT, unsigned SrcReg, 2090314564Sdim unsigned AddrReg, 2091314564Sdim MachineMemOperand *MMO) { 2092314564Sdim unsigned Opc; 2093314564Sdim switch (VT.SimpleTy) { 2094314564Sdim default: return false; 2095314564Sdim case MVT::i8: Opc = AArch64::STLRB; break; 2096314564Sdim case MVT::i16: Opc = AArch64::STLRH; break; 2097314564Sdim case MVT::i32: Opc = AArch64::STLRW; break; 2098314564Sdim case MVT::i64: Opc = AArch64::STLRX; break; 2099314564Sdim } 2100314564Sdim 2101314564Sdim const MCInstrDesc &II = TII.get(Opc); 2102314564Sdim SrcReg = constrainOperandRegClass(II, SrcReg, 0); 2103314564Sdim AddrReg = constrainOperandRegClass(II, AddrReg, 1); 2104314564Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II) 2105314564Sdim .addReg(SrcReg) 2106314564Sdim .addReg(AddrReg) 2107314564Sdim .addMemOperand(MMO); 2108314564Sdim return true; 2109314564Sdim} 2110314564Sdim 2111280031Sdimbool AArch64FastISel::emitStore(MVT VT, unsigned SrcReg, Address Addr, 2112280031Sdim MachineMemOperand *MMO) { 2113288943Sdim if (!TLI.allowsMisalignedMemoryAccesses(VT)) 2114288943Sdim return false; 2115288943Sdim 2116280031Sdim // Simplify this down to something we can handle. 2117280031Sdim if (!simplifyAddress(Addr, VT)) 2118280031Sdim return false; 2119280031Sdim 2120280031Sdim unsigned ScaleFactor = getImplicitScaleFactor(VT); 2121280031Sdim if (!ScaleFactor) 2122280031Sdim llvm_unreachable("Unexpected value type."); 2123280031Sdim 2124274955Ssvnmir // Negative offsets require unscaled, 9-bit, signed immediate offsets. 2125274955Ssvnmir // Otherwise, we try using scaled, 12-bit, unsigned immediate offsets. 2126280031Sdim bool UseScaled = true; 2127280031Sdim if ((Addr.getOffset() < 0) || (Addr.getOffset() & (ScaleFactor - 1))) { 2128280031Sdim UseScaled = false; 2129280031Sdim ScaleFactor = 1; 2130280031Sdim } 2131274955Ssvnmir 2132280031Sdim static const unsigned OpcTable[4][6] = { 2133280031Sdim { AArch64::STURBBi, AArch64::STURHHi, AArch64::STURWi, AArch64::STURXi, 2134280031Sdim AArch64::STURSi, AArch64::STURDi }, 2135280031Sdim { AArch64::STRBBui, AArch64::STRHHui, AArch64::STRWui, AArch64::STRXui, 2136280031Sdim AArch64::STRSui, AArch64::STRDui }, 2137280031Sdim { AArch64::STRBBroX, AArch64::STRHHroX, AArch64::STRWroX, AArch64::STRXroX, 2138280031Sdim AArch64::STRSroX, AArch64::STRDroX }, 2139280031Sdim { AArch64::STRBBroW, AArch64::STRHHroW, AArch64::STRWroW, AArch64::STRXroW, 2140280031Sdim AArch64::STRSroW, AArch64::STRDroW } 2141280031Sdim }; 2142280031Sdim 2143280031Sdim unsigned Opc; 2144274955Ssvnmir bool VTIsi1 = false; 2145280031Sdim bool UseRegOffset = Addr.isRegBase() && !Addr.getOffset() && Addr.getReg() && 2146280031Sdim Addr.getOffsetReg(); 2147280031Sdim unsigned Idx = UseRegOffset ? 2 : UseScaled ? 1 : 0; 2148280031Sdim if (Addr.getExtendType() == AArch64_AM::UXTW || 2149280031Sdim Addr.getExtendType() == AArch64_AM::SXTW) 2150280031Sdim Idx++; 2151280031Sdim 2152274955Ssvnmir switch (VT.SimpleTy) { 2153280031Sdim default: llvm_unreachable("Unexpected value type."); 2154321369Sdim case MVT::i1: VTIsi1 = true; LLVM_FALLTHROUGH; 2155280031Sdim case MVT::i8: Opc = OpcTable[Idx][0]; break; 2156280031Sdim case MVT::i16: Opc = OpcTable[Idx][1]; break; 2157280031Sdim case MVT::i32: Opc = OpcTable[Idx][2]; break; 2158280031Sdim case MVT::i64: Opc = OpcTable[Idx][3]; break; 2159280031Sdim case MVT::f32: Opc = OpcTable[Idx][4]; break; 2160280031Sdim case MVT::f64: Opc = OpcTable[Idx][5]; break; 2161274955Ssvnmir } 2162274955Ssvnmir 2163274955Ssvnmir // Storing an i1 requires special handling. 2164280031Sdim if (VTIsi1 && SrcReg != AArch64::WZR) { 2165280031Sdim unsigned ANDReg = emitAnd_ri(MVT::i32, SrcReg, /*TODO:IsKill=*/false, 1); 2166280031Sdim assert(ANDReg && "Unexpected AND instruction emission failure."); 2167274955Ssvnmir SrcReg = ANDReg; 2168274955Ssvnmir } 2169274955Ssvnmir // Create the base instruction, then add the operands. 2170280031Sdim const MCInstrDesc &II = TII.get(Opc); 2171280031Sdim SrcReg = constrainOperandRegClass(II, SrcReg, II.getNumDefs()); 2172280031Sdim MachineInstrBuilder MIB = 2173280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II).addReg(SrcReg); 2174280031Sdim addLoadStoreOperands(Addr, MIB, MachineMemOperand::MOStore, ScaleFactor, MMO); 2175280031Sdim 2176274955Ssvnmir return true; 2177274955Ssvnmir} 2178274955Ssvnmir 2179280031Sdimbool AArch64FastISel::selectStore(const Instruction *I) { 2180274955Ssvnmir MVT VT; 2181280031Sdim const Value *Op0 = I->getOperand(0); 2182274955Ssvnmir // Verify we have a legal type before going any further. Currently, we handle 2183274955Ssvnmir // simple types that will directly fit in a register (i32/f32/i64/f64) or 2184274955Ssvnmir // those that can be sign or zero-extended to a basic operation (i1/i8/i16). 2185314564Sdim if (!isTypeSupported(Op0->getType(), VT, /*IsVectorAllowed=*/true)) 2186274955Ssvnmir return false; 2187274955Ssvnmir 2188309124Sdim const Value *PtrV = I->getOperand(1); 2189309124Sdim if (TLI.supportSwiftError()) { 2190309124Sdim // Swifterror values can come from either a function parameter with 2191309124Sdim // swifterror attribute or an alloca with swifterror attribute. 2192309124Sdim if (const Argument *Arg = dyn_cast<Argument>(PtrV)) { 2193309124Sdim if (Arg->hasSwiftErrorAttr()) 2194309124Sdim return false; 2195309124Sdim } 2196309124Sdim 2197309124Sdim if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) { 2198309124Sdim if (Alloca->isSwiftError()) 2199309124Sdim return false; 2200309124Sdim } 2201309124Sdim } 2202309124Sdim 2203280031Sdim // Get the value to be stored into a register. Use the zero register directly 2204280031Sdim // when possible to avoid an unnecessary copy and a wasted register. 2205280031Sdim unsigned SrcReg = 0; 2206280031Sdim if (const auto *CI = dyn_cast<ConstantInt>(Op0)) { 2207280031Sdim if (CI->isZero()) 2208280031Sdim SrcReg = (VT == MVT::i64) ? AArch64::XZR : AArch64::WZR; 2209280031Sdim } else if (const auto *CF = dyn_cast<ConstantFP>(Op0)) { 2210280031Sdim if (CF->isZero() && !CF->isNegative()) { 2211280031Sdim VT = MVT::getIntegerVT(VT.getSizeInBits()); 2212280031Sdim SrcReg = (VT == MVT::i64) ? AArch64::XZR : AArch64::WZR; 2213280031Sdim } 2214280031Sdim } 2215280031Sdim 2216280031Sdim if (!SrcReg) 2217280031Sdim SrcReg = getRegForValue(Op0); 2218280031Sdim 2219280031Sdim if (!SrcReg) 2220274955Ssvnmir return false; 2221274955Ssvnmir 2222314564Sdim auto *SI = cast<StoreInst>(I); 2223314564Sdim 2224314564Sdim // Try to emit a STLR for seq_cst/release. 2225314564Sdim if (SI->isAtomic()) { 2226314564Sdim AtomicOrdering Ord = SI->getOrdering(); 2227314564Sdim // The non-atomic instructions are sufficient for relaxed stores. 2228314564Sdim if (isReleaseOrStronger(Ord)) { 2229314564Sdim // The STLR addressing mode only supports a base reg; pass that directly. 2230314564Sdim unsigned AddrReg = getRegForValue(PtrV); 2231314564Sdim return emitStoreRelease(VT, SrcReg, AddrReg, 2232314564Sdim createMachineMemOperandFor(I)); 2233314564Sdim } 2234314564Sdim } 2235314564Sdim 2236274955Ssvnmir // See if we can handle this address. 2237274955Ssvnmir Address Addr; 2238314564Sdim if (!computeAddress(PtrV, Addr, Op0->getType())) 2239274955Ssvnmir return false; 2240274955Ssvnmir 2241280031Sdim if (!emitStore(VT, SrcReg, Addr, createMachineMemOperandFor(I))) 2242274955Ssvnmir return false; 2243274955Ssvnmir return true; 2244274955Ssvnmir} 2245274955Ssvnmir 2246274955Ssvnmirstatic AArch64CC::CondCode getCompareCC(CmpInst::Predicate Pred) { 2247274955Ssvnmir switch (Pred) { 2248274955Ssvnmir case CmpInst::FCMP_ONE: 2249274955Ssvnmir case CmpInst::FCMP_UEQ: 2250274955Ssvnmir default: 2251274955Ssvnmir // AL is our "false" for now. The other two need more compares. 2252274955Ssvnmir return AArch64CC::AL; 2253274955Ssvnmir case CmpInst::ICMP_EQ: 2254274955Ssvnmir case CmpInst::FCMP_OEQ: 2255274955Ssvnmir return AArch64CC::EQ; 2256274955Ssvnmir case CmpInst::ICMP_SGT: 2257274955Ssvnmir case CmpInst::FCMP_OGT: 2258274955Ssvnmir return AArch64CC::GT; 2259274955Ssvnmir case CmpInst::ICMP_SGE: 2260274955Ssvnmir case CmpInst::FCMP_OGE: 2261274955Ssvnmir return AArch64CC::GE; 2262274955Ssvnmir case CmpInst::ICMP_UGT: 2263274955Ssvnmir case CmpInst::FCMP_UGT: 2264274955Ssvnmir return AArch64CC::HI; 2265274955Ssvnmir case CmpInst::FCMP_OLT: 2266274955Ssvnmir return AArch64CC::MI; 2267274955Ssvnmir case CmpInst::ICMP_ULE: 2268274955Ssvnmir case CmpInst::FCMP_OLE: 2269274955Ssvnmir return AArch64CC::LS; 2270274955Ssvnmir case CmpInst::FCMP_ORD: 2271274955Ssvnmir return AArch64CC::VC; 2272274955Ssvnmir case CmpInst::FCMP_UNO: 2273274955Ssvnmir return AArch64CC::VS; 2274274955Ssvnmir case CmpInst::FCMP_UGE: 2275274955Ssvnmir return AArch64CC::PL; 2276274955Ssvnmir case CmpInst::ICMP_SLT: 2277274955Ssvnmir case CmpInst::FCMP_ULT: 2278274955Ssvnmir return AArch64CC::LT; 2279274955Ssvnmir case CmpInst::ICMP_SLE: 2280274955Ssvnmir case CmpInst::FCMP_ULE: 2281274955Ssvnmir return AArch64CC::LE; 2282274955Ssvnmir case CmpInst::FCMP_UNE: 2283274955Ssvnmir case CmpInst::ICMP_NE: 2284274955Ssvnmir return AArch64CC::NE; 2285274955Ssvnmir case CmpInst::ICMP_UGE: 2286274955Ssvnmir return AArch64CC::HS; 2287274955Ssvnmir case CmpInst::ICMP_ULT: 2288274955Ssvnmir return AArch64CC::LO; 2289274955Ssvnmir } 2290274955Ssvnmir} 2291274955Ssvnmir 2292341825Sdim/// Try to emit a combined compare-and-branch instruction. 2293280031Sdimbool AArch64FastISel::emitCompareAndBranch(const BranchInst *BI) { 2294344779Sdim // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z instructions 2295344779Sdim // will not be produced, as they are conditional branch instructions that do 2296344779Sdim // not set flags. 2297344779Sdim if (FuncInfo.MF->getFunction().hasFnAttribute( 2298344779Sdim Attribute::SpeculativeLoadHardening)) 2299344779Sdim return false; 2300344779Sdim 2301280031Sdim assert(isa<CmpInst>(BI->getCondition()) && "Expected cmp instruction"); 2302280031Sdim const CmpInst *CI = cast<CmpInst>(BI->getCondition()); 2303280031Sdim CmpInst::Predicate Predicate = optimizeCmpPredicate(CI); 2304280031Sdim 2305280031Sdim const Value *LHS = CI->getOperand(0); 2306280031Sdim const Value *RHS = CI->getOperand(1); 2307280031Sdim 2308280031Sdim MVT VT; 2309280031Sdim if (!isTypeSupported(LHS->getType(), VT)) 2310280031Sdim return false; 2311280031Sdim 2312280031Sdim unsigned BW = VT.getSizeInBits(); 2313280031Sdim if (BW > 64) 2314280031Sdim return false; 2315280031Sdim 2316280031Sdim MachineBasicBlock *TBB = FuncInfo.MBBMap[BI->getSuccessor(0)]; 2317280031Sdim MachineBasicBlock *FBB = FuncInfo.MBBMap[BI->getSuccessor(1)]; 2318280031Sdim 2319280031Sdim // Try to take advantage of fallthrough opportunities. 2320280031Sdim if (FuncInfo.MBB->isLayoutSuccessor(TBB)) { 2321280031Sdim std::swap(TBB, FBB); 2322280031Sdim Predicate = CmpInst::getInversePredicate(Predicate); 2323280031Sdim } 2324280031Sdim 2325280031Sdim int TestBit = -1; 2326280031Sdim bool IsCmpNE; 2327280031Sdim switch (Predicate) { 2328280031Sdim default: 2329280031Sdim return false; 2330280031Sdim case CmpInst::ICMP_EQ: 2331280031Sdim case CmpInst::ICMP_NE: 2332280031Sdim if (isa<Constant>(LHS) && cast<Constant>(LHS)->isNullValue()) 2333280031Sdim std::swap(LHS, RHS); 2334280031Sdim 2335280031Sdim if (!isa<Constant>(RHS) || !cast<Constant>(RHS)->isNullValue()) 2336280031Sdim return false; 2337280031Sdim 2338280031Sdim if (const auto *AI = dyn_cast<BinaryOperator>(LHS)) 2339280031Sdim if (AI->getOpcode() == Instruction::And && isValueAvailable(AI)) { 2340280031Sdim const Value *AndLHS = AI->getOperand(0); 2341280031Sdim const Value *AndRHS = AI->getOperand(1); 2342280031Sdim 2343280031Sdim if (const auto *C = dyn_cast<ConstantInt>(AndLHS)) 2344280031Sdim if (C->getValue().isPowerOf2()) 2345280031Sdim std::swap(AndLHS, AndRHS); 2346280031Sdim 2347280031Sdim if (const auto *C = dyn_cast<ConstantInt>(AndRHS)) 2348280031Sdim if (C->getValue().isPowerOf2()) { 2349280031Sdim TestBit = C->getValue().logBase2(); 2350280031Sdim LHS = AndLHS; 2351280031Sdim } 2352280031Sdim } 2353280031Sdim 2354280031Sdim if (VT == MVT::i1) 2355280031Sdim TestBit = 0; 2356280031Sdim 2357280031Sdim IsCmpNE = Predicate == CmpInst::ICMP_NE; 2358280031Sdim break; 2359280031Sdim case CmpInst::ICMP_SLT: 2360280031Sdim case CmpInst::ICMP_SGE: 2361280031Sdim if (!isa<Constant>(RHS) || !cast<Constant>(RHS)->isNullValue()) 2362280031Sdim return false; 2363280031Sdim 2364280031Sdim TestBit = BW - 1; 2365280031Sdim IsCmpNE = Predicate == CmpInst::ICMP_SLT; 2366280031Sdim break; 2367280031Sdim case CmpInst::ICMP_SGT: 2368280031Sdim case CmpInst::ICMP_SLE: 2369280031Sdim if (!isa<ConstantInt>(RHS)) 2370280031Sdim return false; 2371280031Sdim 2372280031Sdim if (cast<ConstantInt>(RHS)->getValue() != APInt(BW, -1, true)) 2373280031Sdim return false; 2374280031Sdim 2375280031Sdim TestBit = BW - 1; 2376280031Sdim IsCmpNE = Predicate == CmpInst::ICMP_SLE; 2377280031Sdim break; 2378280031Sdim } // end switch 2379280031Sdim 2380280031Sdim static const unsigned OpcTable[2][2][2] = { 2381280031Sdim { {AArch64::CBZW, AArch64::CBZX }, 2382280031Sdim {AArch64::CBNZW, AArch64::CBNZX} }, 2383280031Sdim { {AArch64::TBZW, AArch64::TBZX }, 2384280031Sdim {AArch64::TBNZW, AArch64::TBNZX} } 2385280031Sdim }; 2386280031Sdim 2387280031Sdim bool IsBitTest = TestBit != -1; 2388280031Sdim bool Is64Bit = BW == 64; 2389280031Sdim if (TestBit < 32 && TestBit >= 0) 2390280031Sdim Is64Bit = false; 2391280031Sdim 2392280031Sdim unsigned Opc = OpcTable[IsBitTest][IsCmpNE][Is64Bit]; 2393280031Sdim const MCInstrDesc &II = TII.get(Opc); 2394280031Sdim 2395280031Sdim unsigned SrcReg = getRegForValue(LHS); 2396280031Sdim if (!SrcReg) 2397280031Sdim return false; 2398280031Sdim bool SrcIsKill = hasTrivialKill(LHS); 2399280031Sdim 2400280031Sdim if (BW == 64 && !Is64Bit) 2401280031Sdim SrcReg = fastEmitInst_extractsubreg(MVT::i32, SrcReg, SrcIsKill, 2402280031Sdim AArch64::sub_32); 2403280031Sdim 2404280031Sdim if ((BW < 32) && !IsBitTest) 2405353358Sdim SrcReg = emitIntExt(VT, SrcReg, MVT::i32, /*isZExt=*/true); 2406280031Sdim 2407280031Sdim // Emit the combined compare and branch instruction. 2408280031Sdim SrcReg = constrainOperandRegClass(II, SrcReg, II.getNumDefs()); 2409280031Sdim MachineInstrBuilder MIB = 2410280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc)) 2411280031Sdim .addReg(SrcReg, getKillRegState(SrcIsKill)); 2412280031Sdim if (IsBitTest) 2413280031Sdim MIB.addImm(TestBit); 2414280031Sdim MIB.addMBB(TBB); 2415280031Sdim 2416296417Sdim finishCondBranch(BI->getParent(), TBB, FBB); 2417280031Sdim return true; 2418280031Sdim} 2419280031Sdim 2420280031Sdimbool AArch64FastISel::selectBranch(const Instruction *I) { 2421274955Ssvnmir const BranchInst *BI = cast<BranchInst>(I); 2422280031Sdim if (BI->isUnconditional()) { 2423280031Sdim MachineBasicBlock *MSucc = FuncInfo.MBBMap[BI->getSuccessor(0)]; 2424280031Sdim fastEmitBranch(MSucc, BI->getDebugLoc()); 2425280031Sdim return true; 2426280031Sdim } 2427280031Sdim 2428274955Ssvnmir MachineBasicBlock *TBB = FuncInfo.MBBMap[BI->getSuccessor(0)]; 2429274955Ssvnmir MachineBasicBlock *FBB = FuncInfo.MBBMap[BI->getSuccessor(1)]; 2430274955Ssvnmir 2431274955Ssvnmir if (const CmpInst *CI = dyn_cast<CmpInst>(BI->getCondition())) { 2432280031Sdim if (CI->hasOneUse() && isValueAvailable(CI)) { 2433280031Sdim // Try to optimize or fold the cmp. 2434280031Sdim CmpInst::Predicate Predicate = optimizeCmpPredicate(CI); 2435280031Sdim switch (Predicate) { 2436280031Sdim default: 2437280031Sdim break; 2438280031Sdim case CmpInst::FCMP_FALSE: 2439280031Sdim fastEmitBranch(FBB, DbgLoc); 2440280031Sdim return true; 2441280031Sdim case CmpInst::FCMP_TRUE: 2442280031Sdim fastEmitBranch(TBB, DbgLoc); 2443280031Sdim return true; 2444280031Sdim } 2445274955Ssvnmir 2446280031Sdim // Try to emit a combined compare-and-branch first. 2447280031Sdim if (emitCompareAndBranch(BI)) 2448280031Sdim return true; 2449280031Sdim 2450280031Sdim // Try to take advantage of fallthrough opportunities. 2451280031Sdim if (FuncInfo.MBB->isLayoutSuccessor(TBB)) { 2452280031Sdim std::swap(TBB, FBB); 2453280031Sdim Predicate = CmpInst::getInversePredicate(Predicate); 2454280031Sdim } 2455280031Sdim 2456274955Ssvnmir // Emit the cmp. 2457280031Sdim if (!emitCmp(CI->getOperand(0), CI->getOperand(1), CI->isUnsigned())) 2458274955Ssvnmir return false; 2459274955Ssvnmir 2460280031Sdim // FCMP_UEQ and FCMP_ONE cannot be checked with a single branch 2461280031Sdim // instruction. 2462296417Sdim AArch64CC::CondCode CC = getCompareCC(Predicate); 2463280031Sdim AArch64CC::CondCode ExtraCC = AArch64CC::AL; 2464280031Sdim switch (Predicate) { 2465280031Sdim default: 2466280031Sdim break; 2467280031Sdim case CmpInst::FCMP_UEQ: 2468280031Sdim ExtraCC = AArch64CC::EQ; 2469280031Sdim CC = AArch64CC::VS; 2470280031Sdim break; 2471280031Sdim case CmpInst::FCMP_ONE: 2472280031Sdim ExtraCC = AArch64CC::MI; 2473280031Sdim CC = AArch64CC::GT; 2474280031Sdim break; 2475280031Sdim } 2476280031Sdim assert((CC != AArch64CC::AL) && "Unexpected condition code."); 2477280031Sdim 2478280031Sdim // Emit the extra branch for FCMP_UEQ and FCMP_ONE. 2479280031Sdim if (ExtraCC != AArch64CC::AL) { 2480280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::Bcc)) 2481280031Sdim .addImm(ExtraCC) 2482280031Sdim .addMBB(TBB); 2483280031Sdim } 2484280031Sdim 2485274955Ssvnmir // Emit the branch. 2486274955Ssvnmir BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::Bcc)) 2487274955Ssvnmir .addImm(CC) 2488274955Ssvnmir .addMBB(TBB); 2489274955Ssvnmir 2490296417Sdim finishCondBranch(BI->getParent(), TBB, FBB); 2491274955Ssvnmir return true; 2492274955Ssvnmir } 2493280031Sdim } else if (const auto *CI = dyn_cast<ConstantInt>(BI->getCondition())) { 2494274955Ssvnmir uint64_t Imm = CI->getZExtValue(); 2495274955Ssvnmir MachineBasicBlock *Target = (Imm == 0) ? FBB : TBB; 2496274955Ssvnmir BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::B)) 2497274955Ssvnmir .addMBB(Target); 2498280031Sdim 2499296417Sdim // Obtain the branch probability and add the target to the successor list. 2500296417Sdim if (FuncInfo.BPI) { 2501296417Sdim auto BranchProbability = FuncInfo.BPI->getEdgeProbability( 2502296417Sdim BI->getParent(), Target->getBasicBlock()); 2503296417Sdim FuncInfo.MBB->addSuccessor(Target, BranchProbability); 2504296417Sdim } else 2505296417Sdim FuncInfo.MBB->addSuccessorWithoutProb(Target); 2506274955Ssvnmir return true; 2507296417Sdim } else { 2508296417Sdim AArch64CC::CondCode CC = AArch64CC::NE; 2509296417Sdim if (foldXALUIntrinsic(CC, I, BI->getCondition())) { 2510296417Sdim // Fake request the condition, otherwise the intrinsic might be completely 2511296417Sdim // optimized away. 2512296417Sdim unsigned CondReg = getRegForValue(BI->getCondition()); 2513296417Sdim if (!CondReg) 2514296417Sdim return false; 2515280031Sdim 2516296417Sdim // Emit the branch. 2517296417Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::Bcc)) 2518296417Sdim .addImm(CC) 2519296417Sdim .addMBB(TBB); 2520280031Sdim 2521296417Sdim finishCondBranch(BI->getParent(), TBB, FBB); 2522296417Sdim return true; 2523296417Sdim } 2524274955Ssvnmir } 2525274955Ssvnmir 2526274955Ssvnmir unsigned CondReg = getRegForValue(BI->getCondition()); 2527274955Ssvnmir if (CondReg == 0) 2528274955Ssvnmir return false; 2529280031Sdim bool CondRegIsKill = hasTrivialKill(BI->getCondition()); 2530274955Ssvnmir 2531296417Sdim // i1 conditions come as i32 values, test the lowest bit with tb(n)z. 2532296417Sdim unsigned Opcode = AArch64::TBNZW; 2533274955Ssvnmir if (FuncInfo.MBB->isLayoutSuccessor(TBB)) { 2534274955Ssvnmir std::swap(TBB, FBB); 2535296417Sdim Opcode = AArch64::TBZW; 2536274955Ssvnmir } 2537274955Ssvnmir 2538296417Sdim const MCInstrDesc &II = TII.get(Opcode); 2539296417Sdim unsigned ConstrainedCondReg 2540296417Sdim = constrainOperandRegClass(II, CondReg, II.getNumDefs()); 2541296417Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II) 2542296417Sdim .addReg(ConstrainedCondReg, getKillRegState(CondRegIsKill)) 2543296417Sdim .addImm(0) 2544274955Ssvnmir .addMBB(TBB); 2545280031Sdim 2546296417Sdim finishCondBranch(BI->getParent(), TBB, FBB); 2547274955Ssvnmir return true; 2548274955Ssvnmir} 2549274955Ssvnmir 2550280031Sdimbool AArch64FastISel::selectIndirectBr(const Instruction *I) { 2551274955Ssvnmir const IndirectBrInst *BI = cast<IndirectBrInst>(I); 2552274955Ssvnmir unsigned AddrReg = getRegForValue(BI->getOperand(0)); 2553274955Ssvnmir if (AddrReg == 0) 2554274955Ssvnmir return false; 2555274955Ssvnmir 2556274955Ssvnmir // Emit the indirect branch. 2557280031Sdim const MCInstrDesc &II = TII.get(AArch64::BR); 2558280031Sdim AddrReg = constrainOperandRegClass(II, AddrReg, II.getNumDefs()); 2559280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II).addReg(AddrReg); 2560274955Ssvnmir 2561274955Ssvnmir // Make sure the CFG is up-to-date. 2562296417Sdim for (auto *Succ : BI->successors()) 2563296417Sdim FuncInfo.MBB->addSuccessor(FuncInfo.MBBMap[Succ]); 2564274955Ssvnmir 2565274955Ssvnmir return true; 2566274955Ssvnmir} 2567274955Ssvnmir 2568280031Sdimbool AArch64FastISel::selectCmp(const Instruction *I) { 2569280031Sdim const CmpInst *CI = cast<CmpInst>(I); 2570274955Ssvnmir 2571296417Sdim // Vectors of i1 are weird: bail out. 2572296417Sdim if (CI->getType()->isVectorTy()) 2573296417Sdim return false; 2574296417Sdim 2575280031Sdim // Try to optimize or fold the cmp. 2576280031Sdim CmpInst::Predicate Predicate = optimizeCmpPredicate(CI); 2577280031Sdim unsigned ResultReg = 0; 2578280031Sdim switch (Predicate) { 2579274955Ssvnmir default: 2580274955Ssvnmir break; 2581280031Sdim case CmpInst::FCMP_FALSE: 2582280031Sdim ResultReg = createResultReg(&AArch64::GPR32RegClass); 2583280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, 2584280031Sdim TII.get(TargetOpcode::COPY), ResultReg) 2585280031Sdim .addReg(AArch64::WZR, getKillRegState(true)); 2586274955Ssvnmir break; 2587280031Sdim case CmpInst::FCMP_TRUE: 2588280031Sdim ResultReg = fastEmit_i(MVT::i32, MVT::i32, ISD::Constant, 1); 2589274955Ssvnmir break; 2590274955Ssvnmir } 2591274955Ssvnmir 2592280031Sdim if (ResultReg) { 2593280031Sdim updateValueMap(I, ResultReg); 2594280031Sdim return true; 2595280031Sdim } 2596280031Sdim 2597280031Sdim // Emit the cmp. 2598280031Sdim if (!emitCmp(CI->getOperand(0), CI->getOperand(1), CI->isUnsigned())) 2599274955Ssvnmir return false; 2600274955Ssvnmir 2601280031Sdim ResultReg = createResultReg(&AArch64::GPR32RegClass); 2602274955Ssvnmir 2603280031Sdim // FCMP_UEQ and FCMP_ONE cannot be checked with a single instruction. These 2604280031Sdim // condition codes are inverted, because they are used by CSINC. 2605280031Sdim static unsigned CondCodeTable[2][2] = { 2606280031Sdim { AArch64CC::NE, AArch64CC::VC }, 2607280031Sdim { AArch64CC::PL, AArch64CC::LE } 2608280031Sdim }; 2609280031Sdim unsigned *CondCodes = nullptr; 2610280031Sdim switch (Predicate) { 2611280031Sdim default: 2612280031Sdim break; 2613280031Sdim case CmpInst::FCMP_UEQ: 2614280031Sdim CondCodes = &CondCodeTable[0][0]; 2615280031Sdim break; 2616280031Sdim case CmpInst::FCMP_ONE: 2617280031Sdim CondCodes = &CondCodeTable[1][0]; 2618280031Sdim break; 2619274955Ssvnmir } 2620274955Ssvnmir 2621280031Sdim if (CondCodes) { 2622280031Sdim unsigned TmpReg1 = createResultReg(&AArch64::GPR32RegClass); 2623280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::CSINCWr), 2624280031Sdim TmpReg1) 2625280031Sdim .addReg(AArch64::WZR, getKillRegState(true)) 2626280031Sdim .addReg(AArch64::WZR, getKillRegState(true)) 2627280031Sdim .addImm(CondCodes[0]); 2628280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::CSINCWr), 2629280031Sdim ResultReg) 2630280031Sdim .addReg(TmpReg1, getKillRegState(true)) 2631280031Sdim .addReg(AArch64::WZR, getKillRegState(true)) 2632280031Sdim .addImm(CondCodes[1]); 2633280031Sdim 2634280031Sdim updateValueMap(I, ResultReg); 2635280031Sdim return true; 2636274955Ssvnmir } 2637274955Ssvnmir 2638274955Ssvnmir // Now set a register based on the comparison. 2639280031Sdim AArch64CC::CondCode CC = getCompareCC(Predicate); 2640280031Sdim assert((CC != AArch64CC::AL) && "Unexpected condition code."); 2641274955Ssvnmir AArch64CC::CondCode invertedCC = getInvertedCondCode(CC); 2642274955Ssvnmir BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::CSINCWr), 2643274955Ssvnmir ResultReg) 2644280031Sdim .addReg(AArch64::WZR, getKillRegState(true)) 2645280031Sdim .addReg(AArch64::WZR, getKillRegState(true)) 2646274955Ssvnmir .addImm(invertedCC); 2647274955Ssvnmir 2648280031Sdim updateValueMap(I, ResultReg); 2649274955Ssvnmir return true; 2650274955Ssvnmir} 2651274955Ssvnmir 2652341825Sdim/// Optimize selects of i1 if one of the operands has a 'true' or 'false' 2653280031Sdim/// value. 2654280031Sdimbool AArch64FastISel::optimizeSelect(const SelectInst *SI) { 2655280031Sdim if (!SI->getType()->isIntegerTy(1)) 2656280031Sdim return false; 2657274955Ssvnmir 2658280031Sdim const Value *Src1Val, *Src2Val; 2659280031Sdim unsigned Opc = 0; 2660280031Sdim bool NeedExtraOp = false; 2661280031Sdim if (auto *CI = dyn_cast<ConstantInt>(SI->getTrueValue())) { 2662280031Sdim if (CI->isOne()) { 2663280031Sdim Src1Val = SI->getCondition(); 2664280031Sdim Src2Val = SI->getFalseValue(); 2665280031Sdim Opc = AArch64::ORRWrr; 2666280031Sdim } else { 2667280031Sdim assert(CI->isZero()); 2668280031Sdim Src1Val = SI->getFalseValue(); 2669280031Sdim Src2Val = SI->getCondition(); 2670280031Sdim Opc = AArch64::BICWrr; 2671280031Sdim } 2672280031Sdim } else if (auto *CI = dyn_cast<ConstantInt>(SI->getFalseValue())) { 2673280031Sdim if (CI->isOne()) { 2674280031Sdim Src1Val = SI->getCondition(); 2675280031Sdim Src2Val = SI->getTrueValue(); 2676280031Sdim Opc = AArch64::ORRWrr; 2677280031Sdim NeedExtraOp = true; 2678280031Sdim } else { 2679280031Sdim assert(CI->isZero()); 2680280031Sdim Src1Val = SI->getCondition(); 2681280031Sdim Src2Val = SI->getTrueValue(); 2682280031Sdim Opc = AArch64::ANDWrr; 2683280031Sdim } 2684280031Sdim } 2685280031Sdim 2686280031Sdim if (!Opc) 2687274955Ssvnmir return false; 2688274955Ssvnmir 2689280031Sdim unsigned Src1Reg = getRegForValue(Src1Val); 2690280031Sdim if (!Src1Reg) 2691274955Ssvnmir return false; 2692280031Sdim bool Src1IsKill = hasTrivialKill(Src1Val); 2693274955Ssvnmir 2694280031Sdim unsigned Src2Reg = getRegForValue(Src2Val); 2695280031Sdim if (!Src2Reg) 2696274955Ssvnmir return false; 2697280031Sdim bool Src2IsKill = hasTrivialKill(Src2Val); 2698274955Ssvnmir 2699280031Sdim if (NeedExtraOp) { 2700280031Sdim Src1Reg = emitLogicalOp_ri(ISD::XOR, MVT::i32, Src1Reg, Src1IsKill, 1); 2701280031Sdim Src1IsKill = true; 2702280031Sdim } 2703288943Sdim unsigned ResultReg = fastEmitInst_rr(Opc, &AArch64::GPR32RegClass, Src1Reg, 2704280031Sdim Src1IsKill, Src2Reg, Src2IsKill); 2705280031Sdim updateValueMap(SI, ResultReg); 2706280031Sdim return true; 2707280031Sdim} 2708274955Ssvnmir 2709280031Sdimbool AArch64FastISel::selectSelect(const Instruction *I) { 2710280031Sdim assert(isa<SelectInst>(I) && "Expected a select instruction."); 2711280031Sdim MVT VT; 2712280031Sdim if (!isTypeSupported(I->getType(), VT)) 2713280031Sdim return false; 2714274955Ssvnmir 2715280031Sdim unsigned Opc; 2716280031Sdim const TargetRegisterClass *RC; 2717280031Sdim switch (VT.SimpleTy) { 2718274955Ssvnmir default: 2719274955Ssvnmir return false; 2720280031Sdim case MVT::i1: 2721280031Sdim case MVT::i8: 2722280031Sdim case MVT::i16: 2723274955Ssvnmir case MVT::i32: 2724280031Sdim Opc = AArch64::CSELWr; 2725280031Sdim RC = &AArch64::GPR32RegClass; 2726274955Ssvnmir break; 2727274955Ssvnmir case MVT::i64: 2728280031Sdim Opc = AArch64::CSELXr; 2729280031Sdim RC = &AArch64::GPR64RegClass; 2730274955Ssvnmir break; 2731274955Ssvnmir case MVT::f32: 2732280031Sdim Opc = AArch64::FCSELSrrr; 2733280031Sdim RC = &AArch64::FPR32RegClass; 2734274955Ssvnmir break; 2735274955Ssvnmir case MVT::f64: 2736280031Sdim Opc = AArch64::FCSELDrrr; 2737280031Sdim RC = &AArch64::FPR64RegClass; 2738274955Ssvnmir break; 2739274955Ssvnmir } 2740274955Ssvnmir 2741280031Sdim const SelectInst *SI = cast<SelectInst>(I); 2742280031Sdim const Value *Cond = SI->getCondition(); 2743280031Sdim AArch64CC::CondCode CC = AArch64CC::NE; 2744280031Sdim AArch64CC::CondCode ExtraCC = AArch64CC::AL; 2745274955Ssvnmir 2746280031Sdim if (optimizeSelect(SI)) 2747280031Sdim return true; 2748280031Sdim 2749280031Sdim // Try to pickup the flags, so we don't have to emit another compare. 2750280031Sdim if (foldXALUIntrinsic(CC, I, Cond)) { 2751280031Sdim // Fake request the condition to force emission of the XALU intrinsic. 2752280031Sdim unsigned CondReg = getRegForValue(Cond); 2753280031Sdim if (!CondReg) 2754280031Sdim return false; 2755280031Sdim } else if (isa<CmpInst>(Cond) && cast<CmpInst>(Cond)->hasOneUse() && 2756280031Sdim isValueAvailable(Cond)) { 2757280031Sdim const auto *Cmp = cast<CmpInst>(Cond); 2758280031Sdim // Try to optimize or fold the cmp. 2759280031Sdim CmpInst::Predicate Predicate = optimizeCmpPredicate(Cmp); 2760280031Sdim const Value *FoldSelect = nullptr; 2761280031Sdim switch (Predicate) { 2762280031Sdim default: 2763280031Sdim break; 2764280031Sdim case CmpInst::FCMP_FALSE: 2765280031Sdim FoldSelect = SI->getFalseValue(); 2766280031Sdim break; 2767280031Sdim case CmpInst::FCMP_TRUE: 2768280031Sdim FoldSelect = SI->getTrueValue(); 2769280031Sdim break; 2770280031Sdim } 2771280031Sdim 2772280031Sdim if (FoldSelect) { 2773280031Sdim unsigned SrcReg = getRegForValue(FoldSelect); 2774280031Sdim if (!SrcReg) 2775280031Sdim return false; 2776280031Sdim unsigned UseReg = lookUpRegForValue(SI); 2777280031Sdim if (UseReg) 2778280031Sdim MRI.clearKillFlags(UseReg); 2779280031Sdim 2780280031Sdim updateValueMap(I, SrcReg); 2781280031Sdim return true; 2782280031Sdim } 2783280031Sdim 2784280031Sdim // Emit the cmp. 2785280031Sdim if (!emitCmp(Cmp->getOperand(0), Cmp->getOperand(1), Cmp->isUnsigned())) 2786280031Sdim return false; 2787280031Sdim 2788280031Sdim // FCMP_UEQ and FCMP_ONE cannot be checked with a single select instruction. 2789280031Sdim CC = getCompareCC(Predicate); 2790280031Sdim switch (Predicate) { 2791280031Sdim default: 2792280031Sdim break; 2793280031Sdim case CmpInst::FCMP_UEQ: 2794280031Sdim ExtraCC = AArch64CC::EQ; 2795280031Sdim CC = AArch64CC::VS; 2796280031Sdim break; 2797280031Sdim case CmpInst::FCMP_ONE: 2798280031Sdim ExtraCC = AArch64CC::MI; 2799280031Sdim CC = AArch64CC::GT; 2800280031Sdim break; 2801280031Sdim } 2802280031Sdim assert((CC != AArch64CC::AL) && "Unexpected condition code."); 2803280031Sdim } else { 2804280031Sdim unsigned CondReg = getRegForValue(Cond); 2805280031Sdim if (!CondReg) 2806280031Sdim return false; 2807280031Sdim bool CondIsKill = hasTrivialKill(Cond); 2808280031Sdim 2809288943Sdim const MCInstrDesc &II = TII.get(AArch64::ANDSWri); 2810288943Sdim CondReg = constrainOperandRegClass(II, CondReg, 1); 2811288943Sdim 2812280031Sdim // Emit a TST instruction (ANDS wzr, reg, #imm). 2813288943Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, 2814280031Sdim AArch64::WZR) 2815280031Sdim .addReg(CondReg, getKillRegState(CondIsKill)) 2816280031Sdim .addImm(AArch64_AM::encodeLogicalImmediate(1, 32)); 2817280031Sdim } 2818280031Sdim 2819280031Sdim unsigned Src1Reg = getRegForValue(SI->getTrueValue()); 2820280031Sdim bool Src1IsKill = hasTrivialKill(SI->getTrueValue()); 2821280031Sdim 2822280031Sdim unsigned Src2Reg = getRegForValue(SI->getFalseValue()); 2823280031Sdim bool Src2IsKill = hasTrivialKill(SI->getFalseValue()); 2824280031Sdim 2825280031Sdim if (!Src1Reg || !Src2Reg) 2826280031Sdim return false; 2827280031Sdim 2828280031Sdim if (ExtraCC != AArch64CC::AL) { 2829280031Sdim Src2Reg = fastEmitInst_rri(Opc, RC, Src1Reg, Src1IsKill, Src2Reg, 2830280031Sdim Src2IsKill, ExtraCC); 2831280031Sdim Src2IsKill = true; 2832280031Sdim } 2833280031Sdim unsigned ResultReg = fastEmitInst_rri(Opc, RC, Src1Reg, Src1IsKill, Src2Reg, 2834280031Sdim Src2IsKill, CC); 2835280031Sdim updateValueMap(I, ResultReg); 2836274955Ssvnmir return true; 2837274955Ssvnmir} 2838274955Ssvnmir 2839280031Sdimbool AArch64FastISel::selectFPExt(const Instruction *I) { 2840274955Ssvnmir Value *V = I->getOperand(0); 2841274955Ssvnmir if (!I->getType()->isDoubleTy() || !V->getType()->isFloatTy()) 2842274955Ssvnmir return false; 2843274955Ssvnmir 2844274955Ssvnmir unsigned Op = getRegForValue(V); 2845274955Ssvnmir if (Op == 0) 2846274955Ssvnmir return false; 2847274955Ssvnmir 2848274955Ssvnmir unsigned ResultReg = createResultReg(&AArch64::FPR64RegClass); 2849274955Ssvnmir BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::FCVTDSr), 2850274955Ssvnmir ResultReg).addReg(Op); 2851280031Sdim updateValueMap(I, ResultReg); 2852274955Ssvnmir return true; 2853274955Ssvnmir} 2854274955Ssvnmir 2855280031Sdimbool AArch64FastISel::selectFPTrunc(const Instruction *I) { 2856274955Ssvnmir Value *V = I->getOperand(0); 2857274955Ssvnmir if (!I->getType()->isFloatTy() || !V->getType()->isDoubleTy()) 2858274955Ssvnmir return false; 2859274955Ssvnmir 2860274955Ssvnmir unsigned Op = getRegForValue(V); 2861274955Ssvnmir if (Op == 0) 2862274955Ssvnmir return false; 2863274955Ssvnmir 2864274955Ssvnmir unsigned ResultReg = createResultReg(&AArch64::FPR32RegClass); 2865274955Ssvnmir BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::FCVTSDr), 2866274955Ssvnmir ResultReg).addReg(Op); 2867280031Sdim updateValueMap(I, ResultReg); 2868274955Ssvnmir return true; 2869274955Ssvnmir} 2870274955Ssvnmir 2871274955Ssvnmir// FPToUI and FPToSI 2872280031Sdimbool AArch64FastISel::selectFPToInt(const Instruction *I, bool Signed) { 2873274955Ssvnmir MVT DestVT; 2874274955Ssvnmir if (!isTypeLegal(I->getType(), DestVT) || DestVT.isVector()) 2875274955Ssvnmir return false; 2876274955Ssvnmir 2877274955Ssvnmir unsigned SrcReg = getRegForValue(I->getOperand(0)); 2878274955Ssvnmir if (SrcReg == 0) 2879274955Ssvnmir return false; 2880274955Ssvnmir 2881288943Sdim EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType(), true); 2882321369Sdim if (SrcVT == MVT::f128 || SrcVT == MVT::f16) 2883274955Ssvnmir return false; 2884274955Ssvnmir 2885274955Ssvnmir unsigned Opc; 2886274955Ssvnmir if (SrcVT == MVT::f64) { 2887274955Ssvnmir if (Signed) 2888274955Ssvnmir Opc = (DestVT == MVT::i32) ? AArch64::FCVTZSUWDr : AArch64::FCVTZSUXDr; 2889274955Ssvnmir else 2890274955Ssvnmir Opc = (DestVT == MVT::i32) ? AArch64::FCVTZUUWDr : AArch64::FCVTZUUXDr; 2891274955Ssvnmir } else { 2892274955Ssvnmir if (Signed) 2893274955Ssvnmir Opc = (DestVT == MVT::i32) ? AArch64::FCVTZSUWSr : AArch64::FCVTZSUXSr; 2894274955Ssvnmir else 2895274955Ssvnmir Opc = (DestVT == MVT::i32) ? AArch64::FCVTZUUWSr : AArch64::FCVTZUUXSr; 2896274955Ssvnmir } 2897274955Ssvnmir unsigned ResultReg = createResultReg( 2898274955Ssvnmir DestVT == MVT::i32 ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass); 2899274955Ssvnmir BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg) 2900274955Ssvnmir .addReg(SrcReg); 2901280031Sdim updateValueMap(I, ResultReg); 2902274955Ssvnmir return true; 2903274955Ssvnmir} 2904274955Ssvnmir 2905280031Sdimbool AArch64FastISel::selectIntToFP(const Instruction *I, bool Signed) { 2906274955Ssvnmir MVT DestVT; 2907274955Ssvnmir if (!isTypeLegal(I->getType(), DestVT) || DestVT.isVector()) 2908274955Ssvnmir return false; 2909321369Sdim // Let regular ISEL handle FP16 2910321369Sdim if (DestVT == MVT::f16) 2911321369Sdim return false; 2912274955Ssvnmir 2913321369Sdim assert((DestVT == MVT::f32 || DestVT == MVT::f64) && 2914321369Sdim "Unexpected value type."); 2915321369Sdim 2916274955Ssvnmir unsigned SrcReg = getRegForValue(I->getOperand(0)); 2917280031Sdim if (!SrcReg) 2918274955Ssvnmir return false; 2919280031Sdim bool SrcIsKill = hasTrivialKill(I->getOperand(0)); 2920274955Ssvnmir 2921288943Sdim EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType(), true); 2922274955Ssvnmir 2923274955Ssvnmir // Handle sign-extension. 2924274955Ssvnmir if (SrcVT == MVT::i16 || SrcVT == MVT::i8 || SrcVT == MVT::i1) { 2925274955Ssvnmir SrcReg = 2926280031Sdim emitIntExt(SrcVT.getSimpleVT(), SrcReg, MVT::i32, /*isZExt*/ !Signed); 2927280031Sdim if (!SrcReg) 2928274955Ssvnmir return false; 2929280031Sdim SrcIsKill = true; 2930274955Ssvnmir } 2931274955Ssvnmir 2932274955Ssvnmir unsigned Opc; 2933274955Ssvnmir if (SrcVT == MVT::i64) { 2934274955Ssvnmir if (Signed) 2935274955Ssvnmir Opc = (DestVT == MVT::f32) ? AArch64::SCVTFUXSri : AArch64::SCVTFUXDri; 2936274955Ssvnmir else 2937274955Ssvnmir Opc = (DestVT == MVT::f32) ? AArch64::UCVTFUXSri : AArch64::UCVTFUXDri; 2938274955Ssvnmir } else { 2939274955Ssvnmir if (Signed) 2940274955Ssvnmir Opc = (DestVT == MVT::f32) ? AArch64::SCVTFUWSri : AArch64::SCVTFUWDri; 2941274955Ssvnmir else 2942274955Ssvnmir Opc = (DestVT == MVT::f32) ? AArch64::UCVTFUWSri : AArch64::UCVTFUWDri; 2943274955Ssvnmir } 2944274955Ssvnmir 2945280031Sdim unsigned ResultReg = fastEmitInst_r(Opc, TLI.getRegClassFor(DestVT), SrcReg, 2946280031Sdim SrcIsKill); 2947280031Sdim updateValueMap(I, ResultReg); 2948274955Ssvnmir return true; 2949274955Ssvnmir} 2950274955Ssvnmir 2951280031Sdimbool AArch64FastISel::fastLowerArguments() { 2952280031Sdim if (!FuncInfo.CanLowerReturn) 2953280031Sdim return false; 2954280031Sdim 2955280031Sdim const Function *F = FuncInfo.Fn; 2956280031Sdim if (F->isVarArg()) 2957280031Sdim return false; 2958280031Sdim 2959280031Sdim CallingConv::ID CC = F->getCallingConv(); 2960314564Sdim if (CC != CallingConv::C && CC != CallingConv::Swift) 2961280031Sdim return false; 2962280031Sdim 2963344779Sdim if (Subtarget->hasCustomCallingConv()) 2964344779Sdim return false; 2965344779Sdim 2966280031Sdim // Only handle simple cases of up to 8 GPR and FPR each. 2967280031Sdim unsigned GPRCnt = 0; 2968280031Sdim unsigned FPRCnt = 0; 2969280031Sdim for (auto const &Arg : F->args()) { 2970321369Sdim if (Arg.hasAttribute(Attribute::ByVal) || 2971321369Sdim Arg.hasAttribute(Attribute::InReg) || 2972321369Sdim Arg.hasAttribute(Attribute::StructRet) || 2973321369Sdim Arg.hasAttribute(Attribute::SwiftSelf) || 2974321369Sdim Arg.hasAttribute(Attribute::SwiftError) || 2975321369Sdim Arg.hasAttribute(Attribute::Nest)) 2976280031Sdim return false; 2977280031Sdim 2978280031Sdim Type *ArgTy = Arg.getType(); 2979280031Sdim if (ArgTy->isStructTy() || ArgTy->isArrayTy()) 2980280031Sdim return false; 2981280031Sdim 2982288943Sdim EVT ArgVT = TLI.getValueType(DL, ArgTy); 2983280031Sdim if (!ArgVT.isSimple()) 2984280031Sdim return false; 2985280031Sdim 2986280031Sdim MVT VT = ArgVT.getSimpleVT().SimpleTy; 2987280031Sdim if (VT.isFloatingPoint() && !Subtarget->hasFPARMv8()) 2988280031Sdim return false; 2989280031Sdim 2990280031Sdim if (VT.isVector() && 2991280031Sdim (!Subtarget->hasNEON() || !Subtarget->isLittleEndian())) 2992280031Sdim return false; 2993280031Sdim 2994280031Sdim if (VT >= MVT::i1 && VT <= MVT::i64) 2995280031Sdim ++GPRCnt; 2996280031Sdim else if ((VT >= MVT::f16 && VT <= MVT::f64) || VT.is64BitVector() || 2997280031Sdim VT.is128BitVector()) 2998280031Sdim ++FPRCnt; 2999280031Sdim else 3000280031Sdim return false; 3001280031Sdim 3002280031Sdim if (GPRCnt > 8 || FPRCnt > 8) 3003280031Sdim return false; 3004280031Sdim } 3005280031Sdim 3006280031Sdim static const MCPhysReg Registers[6][8] = { 3007280031Sdim { AArch64::W0, AArch64::W1, AArch64::W2, AArch64::W3, AArch64::W4, 3008280031Sdim AArch64::W5, AArch64::W6, AArch64::W7 }, 3009280031Sdim { AArch64::X0, AArch64::X1, AArch64::X2, AArch64::X3, AArch64::X4, 3010280031Sdim AArch64::X5, AArch64::X6, AArch64::X7 }, 3011280031Sdim { AArch64::H0, AArch64::H1, AArch64::H2, AArch64::H3, AArch64::H4, 3012280031Sdim AArch64::H5, AArch64::H6, AArch64::H7 }, 3013280031Sdim { AArch64::S0, AArch64::S1, AArch64::S2, AArch64::S3, AArch64::S4, 3014280031Sdim AArch64::S5, AArch64::S6, AArch64::S7 }, 3015280031Sdim { AArch64::D0, AArch64::D1, AArch64::D2, AArch64::D3, AArch64::D4, 3016280031Sdim AArch64::D5, AArch64::D6, AArch64::D7 }, 3017280031Sdim { AArch64::Q0, AArch64::Q1, AArch64::Q2, AArch64::Q3, AArch64::Q4, 3018280031Sdim AArch64::Q5, AArch64::Q6, AArch64::Q7 } 3019280031Sdim }; 3020280031Sdim 3021280031Sdim unsigned GPRIdx = 0; 3022280031Sdim unsigned FPRIdx = 0; 3023280031Sdim for (auto const &Arg : F->args()) { 3024288943Sdim MVT VT = TLI.getSimpleValueType(DL, Arg.getType()); 3025280031Sdim unsigned SrcReg; 3026280031Sdim const TargetRegisterClass *RC; 3027280031Sdim if (VT >= MVT::i1 && VT <= MVT::i32) { 3028280031Sdim SrcReg = Registers[0][GPRIdx++]; 3029280031Sdim RC = &AArch64::GPR32RegClass; 3030280031Sdim VT = MVT::i32; 3031280031Sdim } else if (VT == MVT::i64) { 3032280031Sdim SrcReg = Registers[1][GPRIdx++]; 3033280031Sdim RC = &AArch64::GPR64RegClass; 3034280031Sdim } else if (VT == MVT::f16) { 3035280031Sdim SrcReg = Registers[2][FPRIdx++]; 3036280031Sdim RC = &AArch64::FPR16RegClass; 3037280031Sdim } else if (VT == MVT::f32) { 3038280031Sdim SrcReg = Registers[3][FPRIdx++]; 3039280031Sdim RC = &AArch64::FPR32RegClass; 3040280031Sdim } else if ((VT == MVT::f64) || VT.is64BitVector()) { 3041280031Sdim SrcReg = Registers[4][FPRIdx++]; 3042280031Sdim RC = &AArch64::FPR64RegClass; 3043280031Sdim } else if (VT.is128BitVector()) { 3044280031Sdim SrcReg = Registers[5][FPRIdx++]; 3045280031Sdim RC = &AArch64::FPR128RegClass; 3046280031Sdim } else 3047280031Sdim llvm_unreachable("Unexpected value type."); 3048280031Sdim 3049280031Sdim unsigned DstReg = FuncInfo.MF->addLiveIn(SrcReg, RC); 3050280031Sdim // FIXME: Unfortunately it's necessary to emit a copy from the livein copy. 3051280031Sdim // Without this, EmitLiveInCopies may eliminate the livein if its only 3052280031Sdim // use is a bitcast (which isn't turned into an instruction). 3053280031Sdim unsigned ResultReg = createResultReg(RC); 3054280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, 3055280031Sdim TII.get(TargetOpcode::COPY), ResultReg) 3056280031Sdim .addReg(DstReg, getKillRegState(true)); 3057280031Sdim updateValueMap(&Arg, ResultReg); 3058280031Sdim } 3059280031Sdim return true; 3060280031Sdim} 3061280031Sdim 3062280031Sdimbool AArch64FastISel::processCallArgs(CallLoweringInfo &CLI, 3063280031Sdim SmallVectorImpl<MVT> &OutVTs, 3064280031Sdim unsigned &NumBytes) { 3065280031Sdim CallingConv::ID CC = CLI.CallConv; 3066274955Ssvnmir SmallVector<CCValAssign, 16> ArgLocs; 3067280031Sdim CCState CCInfo(CC, false, *FuncInfo.MF, ArgLocs, *Context); 3068280031Sdim CCInfo.AnalyzeCallOperands(OutVTs, CLI.OutFlags, CCAssignFnForCall(CC)); 3069274955Ssvnmir 3070274955Ssvnmir // Get a count of how many bytes are to be pushed on the stack. 3071274955Ssvnmir NumBytes = CCInfo.getNextStackOffset(); 3072274955Ssvnmir 3073274955Ssvnmir // Issue CALLSEQ_START 3074274955Ssvnmir unsigned AdjStackDown = TII.getCallFrameSetupOpcode(); 3075274955Ssvnmir BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackDown)) 3076321369Sdim .addImm(NumBytes).addImm(0); 3077274955Ssvnmir 3078274955Ssvnmir // Process the args. 3079296417Sdim for (CCValAssign &VA : ArgLocs) { 3080280031Sdim const Value *ArgVal = CLI.OutVals[VA.getValNo()]; 3081280031Sdim MVT ArgVT = OutVTs[VA.getValNo()]; 3082274955Ssvnmir 3083280031Sdim unsigned ArgReg = getRegForValue(ArgVal); 3084280031Sdim if (!ArgReg) 3085280031Sdim return false; 3086280031Sdim 3087274955Ssvnmir // Handle arg promotion: SExt, ZExt, AExt. 3088274955Ssvnmir switch (VA.getLocInfo()) { 3089274955Ssvnmir case CCValAssign::Full: 3090274955Ssvnmir break; 3091274955Ssvnmir case CCValAssign::SExt: { 3092274955Ssvnmir MVT DestVT = VA.getLocVT(); 3093274955Ssvnmir MVT SrcVT = ArgVT; 3094280031Sdim ArgReg = emitIntExt(SrcVT, ArgReg, DestVT, /*isZExt=*/false); 3095280031Sdim if (!ArgReg) 3096274955Ssvnmir return false; 3097274955Ssvnmir break; 3098274955Ssvnmir } 3099274955Ssvnmir case CCValAssign::AExt: 3100274955Ssvnmir // Intentional fall-through. 3101274955Ssvnmir case CCValAssign::ZExt: { 3102274955Ssvnmir MVT DestVT = VA.getLocVT(); 3103274955Ssvnmir MVT SrcVT = ArgVT; 3104280031Sdim ArgReg = emitIntExt(SrcVT, ArgReg, DestVT, /*isZExt=*/true); 3105280031Sdim if (!ArgReg) 3106274955Ssvnmir return false; 3107274955Ssvnmir break; 3108274955Ssvnmir } 3109274955Ssvnmir default: 3110274955Ssvnmir llvm_unreachable("Unknown arg promotion!"); 3111274955Ssvnmir } 3112274955Ssvnmir 3113274955Ssvnmir // Now copy/store arg to correct locations. 3114274955Ssvnmir if (VA.isRegLoc() && !VA.needsCustom()) { 3115274955Ssvnmir BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, 3116280031Sdim TII.get(TargetOpcode::COPY), VA.getLocReg()).addReg(ArgReg); 3117280031Sdim CLI.OutRegs.push_back(VA.getLocReg()); 3118274955Ssvnmir } else if (VA.needsCustom()) { 3119274955Ssvnmir // FIXME: Handle custom args. 3120274955Ssvnmir return false; 3121274955Ssvnmir } else { 3122274955Ssvnmir assert(VA.isMemLoc() && "Assuming store on stack."); 3123274955Ssvnmir 3124280031Sdim // Don't emit stores for undef values. 3125280031Sdim if (isa<UndefValue>(ArgVal)) 3126280031Sdim continue; 3127280031Sdim 3128274955Ssvnmir // Need to store on the stack. 3129274955Ssvnmir unsigned ArgSize = (ArgVT.getSizeInBits() + 7) / 8; 3130274955Ssvnmir 3131274955Ssvnmir unsigned BEAlign = 0; 3132274955Ssvnmir if (ArgSize < 8 && !Subtarget->isLittleEndian()) 3133274955Ssvnmir BEAlign = 8 - ArgSize; 3134274955Ssvnmir 3135274955Ssvnmir Address Addr; 3136274955Ssvnmir Addr.setKind(Address::RegBase); 3137274955Ssvnmir Addr.setReg(AArch64::SP); 3138274955Ssvnmir Addr.setOffset(VA.getLocMemOffset() + BEAlign); 3139274955Ssvnmir 3140280031Sdim unsigned Alignment = DL.getABITypeAlignment(ArgVal->getType()); 3141280031Sdim MachineMemOperand *MMO = FuncInfo.MF->getMachineMemOperand( 3142296417Sdim MachinePointerInfo::getStack(*FuncInfo.MF, Addr.getOffset()), 3143296417Sdim MachineMemOperand::MOStore, ArgVT.getStoreSize(), Alignment); 3144280031Sdim 3145280031Sdim if (!emitStore(ArgVT, ArgReg, Addr, MMO)) 3146274955Ssvnmir return false; 3147274955Ssvnmir } 3148274955Ssvnmir } 3149274955Ssvnmir return true; 3150274955Ssvnmir} 3151274955Ssvnmir 3152280031Sdimbool AArch64FastISel::finishCall(CallLoweringInfo &CLI, MVT RetVT, 3153280031Sdim unsigned NumBytes) { 3154280031Sdim CallingConv::ID CC = CLI.CallConv; 3155280031Sdim 3156274955Ssvnmir // Issue CALLSEQ_END 3157274955Ssvnmir unsigned AdjStackUp = TII.getCallFrameDestroyOpcode(); 3158274955Ssvnmir BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackUp)) 3159280031Sdim .addImm(NumBytes).addImm(0); 3160274955Ssvnmir 3161274955Ssvnmir // Now the return value. 3162274955Ssvnmir if (RetVT != MVT::isVoid) { 3163274955Ssvnmir SmallVector<CCValAssign, 16> RVLocs; 3164280031Sdim CCState CCInfo(CC, false, *FuncInfo.MF, RVLocs, *Context); 3165274955Ssvnmir CCInfo.AnalyzeCallResult(RetVT, CCAssignFnForCall(CC)); 3166274955Ssvnmir 3167274955Ssvnmir // Only handle a single return value. 3168274955Ssvnmir if (RVLocs.size() != 1) 3169274955Ssvnmir return false; 3170274955Ssvnmir 3171274955Ssvnmir // Copy all of the result registers out of their specified physreg. 3172274955Ssvnmir MVT CopyVT = RVLocs[0].getValVT(); 3173288943Sdim 3174288943Sdim // TODO: Handle big-endian results 3175288943Sdim if (CopyVT.isVector() && !Subtarget->isLittleEndian()) 3176288943Sdim return false; 3177288943Sdim 3178274955Ssvnmir unsigned ResultReg = createResultReg(TLI.getRegClassFor(CopyVT)); 3179274955Ssvnmir BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, 3180280031Sdim TII.get(TargetOpcode::COPY), ResultReg) 3181280031Sdim .addReg(RVLocs[0].getLocReg()); 3182280031Sdim CLI.InRegs.push_back(RVLocs[0].getLocReg()); 3183274955Ssvnmir 3184280031Sdim CLI.ResultReg = ResultReg; 3185280031Sdim CLI.NumResultRegs = 1; 3186274955Ssvnmir } 3187274955Ssvnmir 3188274955Ssvnmir return true; 3189274955Ssvnmir} 3190274955Ssvnmir 3191280031Sdimbool AArch64FastISel::fastLowerCall(CallLoweringInfo &CLI) { 3192280031Sdim CallingConv::ID CC = CLI.CallConv; 3193280031Sdim bool IsTailCall = CLI.IsTailCall; 3194280031Sdim bool IsVarArg = CLI.IsVarArg; 3195280031Sdim const Value *Callee = CLI.Callee; 3196288943Sdim MCSymbol *Symbol = CLI.Symbol; 3197274955Ssvnmir 3198288943Sdim if (!Callee && !Symbol) 3199274955Ssvnmir return false; 3200274955Ssvnmir 3201280031Sdim // Allow SelectionDAG isel to handle tail calls. 3202280031Sdim if (IsTailCall) 3203274955Ssvnmir return false; 3204274955Ssvnmir 3205360784Sdim // FIXME: we could and should support this, but for now correctness at -O0 is 3206360784Sdim // more important. 3207360784Sdim if (Subtarget->isTargetILP32()) 3208360784Sdim return false; 3209360784Sdim 3210280031Sdim CodeModel::Model CM = TM.getCodeModel(); 3211321369Sdim // Only support the small-addressing and large code models. 3212321369Sdim if (CM != CodeModel::Large && !Subtarget->useSmallAddressing()) 3213280031Sdim return false; 3214274955Ssvnmir 3215280031Sdim // FIXME: Add large code model support for ELF. 3216280031Sdim if (CM == CodeModel::Large && !Subtarget->isTargetMachO()) 3217280031Sdim return false; 3218280031Sdim 3219274955Ssvnmir // Let SDISel handle vararg functions. 3220280031Sdim if (IsVarArg) 3221274955Ssvnmir return false; 3222274955Ssvnmir 3223280031Sdim // FIXME: Only handle *simple* calls for now. 3224274955Ssvnmir MVT RetVT; 3225280031Sdim if (CLI.RetTy->isVoidTy()) 3226274955Ssvnmir RetVT = MVT::isVoid; 3227280031Sdim else if (!isTypeLegal(CLI.RetTy, RetVT)) 3228274955Ssvnmir return false; 3229274955Ssvnmir 3230280031Sdim for (auto Flag : CLI.OutFlags) 3231309124Sdim if (Flag.isInReg() || Flag.isSRet() || Flag.isNest() || Flag.isByVal() || 3232309124Sdim Flag.isSwiftSelf() || Flag.isSwiftError()) 3233274955Ssvnmir return false; 3234274955Ssvnmir 3235280031Sdim // Set up the argument vectors. 3236280031Sdim SmallVector<MVT, 16> OutVTs; 3237280031Sdim OutVTs.reserve(CLI.OutVals.size()); 3238274955Ssvnmir 3239280031Sdim for (auto *Val : CLI.OutVals) { 3240280031Sdim MVT VT; 3241280031Sdim if (!isTypeLegal(Val->getType(), VT) && 3242280031Sdim !(VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)) 3243274955Ssvnmir return false; 3244274955Ssvnmir 3245274955Ssvnmir // We don't handle vector parameters yet. 3246280031Sdim if (VT.isVector() || VT.getSizeInBits() > 64) 3247274955Ssvnmir return false; 3248274955Ssvnmir 3249280031Sdim OutVTs.push_back(VT); 3250274955Ssvnmir } 3251274955Ssvnmir 3252280031Sdim Address Addr; 3253280031Sdim if (Callee && !computeCallAddress(Callee, Addr)) 3254280031Sdim return false; 3255280031Sdim 3256360784Sdim // The weak function target may be zero; in that case we must use indirect 3257360784Sdim // addressing via a stub on windows as it may be out of range for a 3258360784Sdim // PC-relative jump. 3259360784Sdim if (Subtarget->isTargetWindows() && Addr.getGlobalValue() && 3260360784Sdim Addr.getGlobalValue()->hasExternalWeakLinkage()) 3261360784Sdim return false; 3262360784Sdim 3263274955Ssvnmir // Handle the arguments now that we've gotten them. 3264274955Ssvnmir unsigned NumBytes; 3265280031Sdim if (!processCallArgs(CLI, OutVTs, NumBytes)) 3266274955Ssvnmir return false; 3267274955Ssvnmir 3268344779Sdim const AArch64RegisterInfo *RegInfo = Subtarget->getRegisterInfo(); 3269344779Sdim if (RegInfo->isAnyArgRegReserved(*MF)) 3270344779Sdim RegInfo->emitReservedArgRegCallError(*MF); 3271344779Sdim 3272274955Ssvnmir // Issue the call. 3273274955Ssvnmir MachineInstrBuilder MIB; 3274321369Sdim if (Subtarget->useSmallAddressing()) { 3275280031Sdim const MCInstrDesc &II = TII.get(Addr.getReg() ? AArch64::BLR : AArch64::BL); 3276280031Sdim MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II); 3277288943Sdim if (Symbol) 3278288943Sdim MIB.addSym(Symbol, 0); 3279280031Sdim else if (Addr.getGlobalValue()) 3280280031Sdim MIB.addGlobalAddress(Addr.getGlobalValue(), 0, 0); 3281280031Sdim else if (Addr.getReg()) { 3282280031Sdim unsigned Reg = constrainOperandRegClass(II, Addr.getReg(), 0); 3283280031Sdim MIB.addReg(Reg); 3284280031Sdim } else 3285280031Sdim return false; 3286280031Sdim } else { 3287280031Sdim unsigned CallReg = 0; 3288288943Sdim if (Symbol) { 3289280031Sdim unsigned ADRPReg = createResultReg(&AArch64::GPR64commonRegClass); 3290280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::ADRP), 3291280031Sdim ADRPReg) 3292288943Sdim .addSym(Symbol, AArch64II::MO_GOT | AArch64II::MO_PAGE); 3293274955Ssvnmir 3294280031Sdim CallReg = createResultReg(&AArch64::GPR64RegClass); 3295288943Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, 3296288943Sdim TII.get(AArch64::LDRXui), CallReg) 3297288943Sdim .addReg(ADRPReg) 3298288943Sdim .addSym(Symbol, 3299288943Sdim AArch64II::MO_GOT | AArch64II::MO_PAGEOFF | AArch64II::MO_NC); 3300280031Sdim } else if (Addr.getGlobalValue()) 3301280031Sdim CallReg = materializeGV(Addr.getGlobalValue()); 3302280031Sdim else if (Addr.getReg()) 3303280031Sdim CallReg = Addr.getReg(); 3304280031Sdim 3305280031Sdim if (!CallReg) 3306280031Sdim return false; 3307280031Sdim 3308280031Sdim const MCInstrDesc &II = TII.get(AArch64::BLR); 3309280031Sdim CallReg = constrainOperandRegClass(II, CallReg, 0); 3310280031Sdim MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II).addReg(CallReg); 3311280031Sdim } 3312280031Sdim 3313274955Ssvnmir // Add implicit physical register uses to the call. 3314280031Sdim for (auto Reg : CLI.OutRegs) 3315280031Sdim MIB.addReg(Reg, RegState::Implicit); 3316274955Ssvnmir 3317274955Ssvnmir // Add a register mask with the call-preserved registers. 3318274955Ssvnmir // Proper defs for return values will be added by setPhysRegsDeadExcept(). 3319288943Sdim MIB.addRegMask(TRI.getCallPreservedMask(*FuncInfo.MF, CC)); 3320274955Ssvnmir 3321280031Sdim CLI.Call = MIB; 3322280031Sdim 3323274955Ssvnmir // Finish off the call including any return values. 3324280031Sdim return finishCall(CLI, RetVT, NumBytes); 3325274955Ssvnmir} 3326274955Ssvnmir 3327280031Sdimbool AArch64FastISel::isMemCpySmall(uint64_t Len, unsigned Alignment) { 3328274955Ssvnmir if (Alignment) 3329274955Ssvnmir return Len / Alignment <= 4; 3330274955Ssvnmir else 3331274955Ssvnmir return Len < 32; 3332274955Ssvnmir} 3333274955Ssvnmir 3334280031Sdimbool AArch64FastISel::tryEmitSmallMemCpy(Address Dest, Address Src, 3335274955Ssvnmir uint64_t Len, unsigned Alignment) { 3336274955Ssvnmir // Make sure we don't bloat code by inlining very large memcpy's. 3337280031Sdim if (!isMemCpySmall(Len, Alignment)) 3338274955Ssvnmir return false; 3339274955Ssvnmir 3340274955Ssvnmir int64_t UnscaledOffset = 0; 3341274955Ssvnmir Address OrigDest = Dest; 3342274955Ssvnmir Address OrigSrc = Src; 3343274955Ssvnmir 3344274955Ssvnmir while (Len) { 3345274955Ssvnmir MVT VT; 3346274955Ssvnmir if (!Alignment || Alignment >= 8) { 3347274955Ssvnmir if (Len >= 8) 3348274955Ssvnmir VT = MVT::i64; 3349274955Ssvnmir else if (Len >= 4) 3350274955Ssvnmir VT = MVT::i32; 3351274955Ssvnmir else if (Len >= 2) 3352274955Ssvnmir VT = MVT::i16; 3353274955Ssvnmir else { 3354274955Ssvnmir VT = MVT::i8; 3355274955Ssvnmir } 3356274955Ssvnmir } else { 3357274955Ssvnmir // Bound based on alignment. 3358274955Ssvnmir if (Len >= 4 && Alignment == 4) 3359274955Ssvnmir VT = MVT::i32; 3360274955Ssvnmir else if (Len >= 2 && Alignment == 2) 3361274955Ssvnmir VT = MVT::i16; 3362274955Ssvnmir else { 3363274955Ssvnmir VT = MVT::i8; 3364274955Ssvnmir } 3365274955Ssvnmir } 3366274955Ssvnmir 3367280031Sdim unsigned ResultReg = emitLoad(VT, VT, Src); 3368280031Sdim if (!ResultReg) 3369274955Ssvnmir return false; 3370274955Ssvnmir 3371280031Sdim if (!emitStore(VT, ResultReg, Dest)) 3372274955Ssvnmir return false; 3373274955Ssvnmir 3374274955Ssvnmir int64_t Size = VT.getSizeInBits() / 8; 3375274955Ssvnmir Len -= Size; 3376274955Ssvnmir UnscaledOffset += Size; 3377274955Ssvnmir 3378274955Ssvnmir // We need to recompute the unscaled offset for each iteration. 3379274955Ssvnmir Dest.setOffset(OrigDest.getOffset() + UnscaledOffset); 3380274955Ssvnmir Src.setOffset(OrigSrc.getOffset() + UnscaledOffset); 3381274955Ssvnmir } 3382274955Ssvnmir 3383274955Ssvnmir return true; 3384274955Ssvnmir} 3385274955Ssvnmir 3386341825Sdim/// Check if it is possible to fold the condition from the XALU intrinsic 3387280031Sdim/// into the user. The condition code will only be updated on success. 3388280031Sdimbool AArch64FastISel::foldXALUIntrinsic(AArch64CC::CondCode &CC, 3389280031Sdim const Instruction *I, 3390280031Sdim const Value *Cond) { 3391280031Sdim if (!isa<ExtractValueInst>(Cond)) 3392280031Sdim return false; 3393280031Sdim 3394280031Sdim const auto *EV = cast<ExtractValueInst>(Cond); 3395280031Sdim if (!isa<IntrinsicInst>(EV->getAggregateOperand())) 3396280031Sdim return false; 3397280031Sdim 3398280031Sdim const auto *II = cast<IntrinsicInst>(EV->getAggregateOperand()); 3399280031Sdim MVT RetVT; 3400280031Sdim const Function *Callee = II->getCalledFunction(); 3401280031Sdim Type *RetTy = 3402280031Sdim cast<StructType>(Callee->getReturnType())->getTypeAtIndex(0U); 3403280031Sdim if (!isTypeLegal(RetTy, RetVT)) 3404280031Sdim return false; 3405280031Sdim 3406280031Sdim if (RetVT != MVT::i32 && RetVT != MVT::i64) 3407280031Sdim return false; 3408280031Sdim 3409280031Sdim const Value *LHS = II->getArgOperand(0); 3410280031Sdim const Value *RHS = II->getArgOperand(1); 3411280031Sdim 3412280031Sdim // Canonicalize immediate to the RHS. 3413280031Sdim if (isa<ConstantInt>(LHS) && !isa<ConstantInt>(RHS) && 3414280031Sdim isCommutativeIntrinsic(II)) 3415280031Sdim std::swap(LHS, RHS); 3416280031Sdim 3417280031Sdim // Simplify multiplies. 3418288943Sdim Intrinsic::ID IID = II->getIntrinsicID(); 3419280031Sdim switch (IID) { 3420274955Ssvnmir default: 3421280031Sdim break; 3422280031Sdim case Intrinsic::smul_with_overflow: 3423280031Sdim if (const auto *C = dyn_cast<ConstantInt>(RHS)) 3424280031Sdim if (C->getValue() == 2) 3425280031Sdim IID = Intrinsic::sadd_with_overflow; 3426280031Sdim break; 3427280031Sdim case Intrinsic::umul_with_overflow: 3428280031Sdim if (const auto *C = dyn_cast<ConstantInt>(RHS)) 3429280031Sdim if (C->getValue() == 2) 3430280031Sdim IID = Intrinsic::uadd_with_overflow; 3431280031Sdim break; 3432280031Sdim } 3433280031Sdim 3434280031Sdim AArch64CC::CondCode TmpCC; 3435280031Sdim switch (IID) { 3436280031Sdim default: 3437274955Ssvnmir return false; 3438280031Sdim case Intrinsic::sadd_with_overflow: 3439280031Sdim case Intrinsic::ssub_with_overflow: 3440280031Sdim TmpCC = AArch64CC::VS; 3441280031Sdim break; 3442280031Sdim case Intrinsic::uadd_with_overflow: 3443280031Sdim TmpCC = AArch64CC::HS; 3444280031Sdim break; 3445280031Sdim case Intrinsic::usub_with_overflow: 3446280031Sdim TmpCC = AArch64CC::LO; 3447280031Sdim break; 3448280031Sdim case Intrinsic::smul_with_overflow: 3449280031Sdim case Intrinsic::umul_with_overflow: 3450280031Sdim TmpCC = AArch64CC::NE; 3451280031Sdim break; 3452280031Sdim } 3453280031Sdim 3454280031Sdim // Check if both instructions are in the same basic block. 3455280031Sdim if (!isValueAvailable(II)) 3456280031Sdim return false; 3457280031Sdim 3458280031Sdim // Make sure nothing is in the way 3459296417Sdim BasicBlock::const_iterator Start(I); 3460296417Sdim BasicBlock::const_iterator End(II); 3461280031Sdim for (auto Itr = std::prev(Start); Itr != End; --Itr) { 3462280031Sdim // We only expect extractvalue instructions between the intrinsic and the 3463280031Sdim // instruction to be selected. 3464280031Sdim if (!isa<ExtractValueInst>(Itr)) 3465280031Sdim return false; 3466280031Sdim 3467280031Sdim // Check that the extractvalue operand comes from the intrinsic. 3468280031Sdim const auto *EVI = cast<ExtractValueInst>(Itr); 3469280031Sdim if (EVI->getAggregateOperand() != II) 3470280031Sdim return false; 3471280031Sdim } 3472280031Sdim 3473280031Sdim CC = TmpCC; 3474280031Sdim return true; 3475280031Sdim} 3476280031Sdim 3477280031Sdimbool AArch64FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) { 3478280031Sdim // FIXME: Handle more intrinsics. 3479280031Sdim switch (II->getIntrinsicID()) { 3480280031Sdim default: return false; 3481280031Sdim case Intrinsic::frameaddress: { 3482314564Sdim MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo(); 3483314564Sdim MFI.setFrameAddressIsTaken(true); 3484280031Sdim 3485321369Sdim const AArch64RegisterInfo *RegInfo = Subtarget->getRegisterInfo(); 3486360784Sdim Register FramePtr = RegInfo->getFrameRegister(*(FuncInfo.MF)); 3487360784Sdim Register SrcReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass); 3488280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, 3489280031Sdim TII.get(TargetOpcode::COPY), SrcReg).addReg(FramePtr); 3490280031Sdim // Recursively load frame address 3491280031Sdim // ldr x0, [fp] 3492280031Sdim // ldr x0, [x0] 3493280031Sdim // ldr x0, [x0] 3494280031Sdim // ... 3495280031Sdim unsigned DestReg; 3496280031Sdim unsigned Depth = cast<ConstantInt>(II->getOperand(0))->getZExtValue(); 3497280031Sdim while (Depth--) { 3498280031Sdim DestReg = fastEmitInst_ri(AArch64::LDRXui, &AArch64::GPR64RegClass, 3499280031Sdim SrcReg, /*IsKill=*/true, 0); 3500280031Sdim assert(DestReg && "Unexpected LDR instruction emission failure."); 3501280031Sdim SrcReg = DestReg; 3502280031Sdim } 3503280031Sdim 3504280031Sdim updateValueMap(II, SrcReg); 3505280031Sdim return true; 3506280031Sdim } 3507344779Sdim case Intrinsic::sponentry: { 3508344779Sdim MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo(); 3509344779Sdim 3510344779Sdim // SP = FP + Fixed Object + 16 3511344779Sdim int FI = MFI.CreateFixedObject(4, 0, false); 3512344779Sdim unsigned ResultReg = createResultReg(&AArch64::GPR64spRegClass); 3513344779Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, 3514344779Sdim TII.get(AArch64::ADDXri), ResultReg) 3515344779Sdim .addFrameIndex(FI) 3516344779Sdim .addImm(0) 3517344779Sdim .addImm(0); 3518344779Sdim 3519344779Sdim updateValueMap(II, ResultReg); 3520344779Sdim return true; 3521344779Sdim } 3522274955Ssvnmir case Intrinsic::memcpy: 3523274955Ssvnmir case Intrinsic::memmove: { 3524280031Sdim const auto *MTI = cast<MemTransferInst>(II); 3525274955Ssvnmir // Don't handle volatile. 3526280031Sdim if (MTI->isVolatile()) 3527274955Ssvnmir return false; 3528274955Ssvnmir 3529274955Ssvnmir // Disable inlining for memmove before calls to ComputeAddress. Otherwise, 3530274955Ssvnmir // we would emit dead code because we don't currently handle memmoves. 3531280031Sdim bool IsMemCpy = (II->getIntrinsicID() == Intrinsic::memcpy); 3532280031Sdim if (isa<ConstantInt>(MTI->getLength()) && IsMemCpy) { 3533274955Ssvnmir // Small memcpy's are common enough that we want to do them without a call 3534274955Ssvnmir // if possible. 3535280031Sdim uint64_t Len = cast<ConstantInt>(MTI->getLength())->getZExtValue(); 3536341825Sdim unsigned Alignment = MinAlign(MTI->getDestAlignment(), 3537341825Sdim MTI->getSourceAlignment()); 3538280031Sdim if (isMemCpySmall(Len, Alignment)) { 3539274955Ssvnmir Address Dest, Src; 3540280031Sdim if (!computeAddress(MTI->getRawDest(), Dest) || 3541280031Sdim !computeAddress(MTI->getRawSource(), Src)) 3542274955Ssvnmir return false; 3543280031Sdim if (tryEmitSmallMemCpy(Dest, Src, Len, Alignment)) 3544274955Ssvnmir return true; 3545274955Ssvnmir } 3546274955Ssvnmir } 3547274955Ssvnmir 3548280031Sdim if (!MTI->getLength()->getType()->isIntegerTy(64)) 3549274955Ssvnmir return false; 3550274955Ssvnmir 3551280031Sdim if (MTI->getSourceAddressSpace() > 255 || MTI->getDestAddressSpace() > 255) 3552274955Ssvnmir // Fast instruction selection doesn't support the special 3553274955Ssvnmir // address spaces. 3554274955Ssvnmir return false; 3555274955Ssvnmir 3556280031Sdim const char *IntrMemName = isa<MemCpyInst>(II) ? "memcpy" : "memmove"; 3557341825Sdim return lowerCallTo(II, IntrMemName, II->getNumArgOperands() - 1); 3558274955Ssvnmir } 3559274955Ssvnmir case Intrinsic::memset: { 3560280031Sdim const MemSetInst *MSI = cast<MemSetInst>(II); 3561274955Ssvnmir // Don't handle volatile. 3562280031Sdim if (MSI->isVolatile()) 3563274955Ssvnmir return false; 3564274955Ssvnmir 3565280031Sdim if (!MSI->getLength()->getType()->isIntegerTy(64)) 3566274955Ssvnmir return false; 3567274955Ssvnmir 3568280031Sdim if (MSI->getDestAddressSpace() > 255) 3569274955Ssvnmir // Fast instruction selection doesn't support the special 3570274955Ssvnmir // address spaces. 3571274955Ssvnmir return false; 3572274955Ssvnmir 3573341825Sdim return lowerCallTo(II, "memset", II->getNumArgOperands() - 1); 3574274955Ssvnmir } 3575280031Sdim case Intrinsic::sin: 3576280031Sdim case Intrinsic::cos: 3577280031Sdim case Intrinsic::pow: { 3578280031Sdim MVT RetVT; 3579280031Sdim if (!isTypeLegal(II->getType(), RetVT)) 3580280031Sdim return false; 3581280031Sdim 3582280031Sdim if (RetVT != MVT::f32 && RetVT != MVT::f64) 3583280031Sdim return false; 3584280031Sdim 3585280031Sdim static const RTLIB::Libcall LibCallTable[3][2] = { 3586280031Sdim { RTLIB::SIN_F32, RTLIB::SIN_F64 }, 3587280031Sdim { RTLIB::COS_F32, RTLIB::COS_F64 }, 3588280031Sdim { RTLIB::POW_F32, RTLIB::POW_F64 } 3589280031Sdim }; 3590280031Sdim RTLIB::Libcall LC; 3591280031Sdim bool Is64Bit = RetVT == MVT::f64; 3592280031Sdim switch (II->getIntrinsicID()) { 3593280031Sdim default: 3594280031Sdim llvm_unreachable("Unexpected intrinsic."); 3595280031Sdim case Intrinsic::sin: 3596280031Sdim LC = LibCallTable[0][Is64Bit]; 3597280031Sdim break; 3598280031Sdim case Intrinsic::cos: 3599280031Sdim LC = LibCallTable[1][Is64Bit]; 3600280031Sdim break; 3601280031Sdim case Intrinsic::pow: 3602280031Sdim LC = LibCallTable[2][Is64Bit]; 3603280031Sdim break; 3604280031Sdim } 3605280031Sdim 3606280031Sdim ArgListTy Args; 3607280031Sdim Args.reserve(II->getNumArgOperands()); 3608280031Sdim 3609280031Sdim // Populate the argument list. 3610280031Sdim for (auto &Arg : II->arg_operands()) { 3611280031Sdim ArgListEntry Entry; 3612280031Sdim Entry.Val = Arg; 3613280031Sdim Entry.Ty = Arg->getType(); 3614280031Sdim Args.push_back(Entry); 3615280031Sdim } 3616280031Sdim 3617280031Sdim CallLoweringInfo CLI; 3618288943Sdim MCContext &Ctx = MF->getContext(); 3619288943Sdim CLI.setCallee(DL, Ctx, TLI.getLibcallCallingConv(LC), II->getType(), 3620280031Sdim TLI.getLibcallName(LC), std::move(Args)); 3621280031Sdim if (!lowerCallTo(CLI)) 3622280031Sdim return false; 3623280031Sdim updateValueMap(II, CLI.ResultReg); 3624280031Sdim return true; 3625280031Sdim } 3626280031Sdim case Intrinsic::fabs: { 3627280031Sdim MVT VT; 3628280031Sdim if (!isTypeLegal(II->getType(), VT)) 3629280031Sdim return false; 3630280031Sdim 3631280031Sdim unsigned Opc; 3632280031Sdim switch (VT.SimpleTy) { 3633280031Sdim default: 3634280031Sdim return false; 3635280031Sdim case MVT::f32: 3636280031Sdim Opc = AArch64::FABSSr; 3637280031Sdim break; 3638280031Sdim case MVT::f64: 3639280031Sdim Opc = AArch64::FABSDr; 3640280031Sdim break; 3641280031Sdim } 3642280031Sdim unsigned SrcReg = getRegForValue(II->getOperand(0)); 3643280031Sdim if (!SrcReg) 3644280031Sdim return false; 3645280031Sdim bool SrcRegIsKill = hasTrivialKill(II->getOperand(0)); 3646280031Sdim unsigned ResultReg = createResultReg(TLI.getRegClassFor(VT)); 3647280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg) 3648280031Sdim .addReg(SrcReg, getKillRegState(SrcRegIsKill)); 3649280031Sdim updateValueMap(II, ResultReg); 3650280031Sdim return true; 3651280031Sdim } 3652321369Sdim case Intrinsic::trap: 3653274955Ssvnmir BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::BRK)) 3654274955Ssvnmir .addImm(1); 3655274955Ssvnmir return true; 3656353358Sdim case Intrinsic::debugtrap: { 3657353358Sdim if (Subtarget->isTargetWindows()) { 3658353358Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::BRK)) 3659353358Sdim .addImm(0xF000); 3660353358Sdim return true; 3661353358Sdim } 3662353358Sdim break; 3663353358Sdim } 3664321369Sdim 3665280031Sdim case Intrinsic::sqrt: { 3666280031Sdim Type *RetTy = II->getCalledFunction()->getReturnType(); 3667280031Sdim 3668280031Sdim MVT VT; 3669280031Sdim if (!isTypeLegal(RetTy, VT)) 3670280031Sdim return false; 3671280031Sdim 3672280031Sdim unsigned Op0Reg = getRegForValue(II->getOperand(0)); 3673280031Sdim if (!Op0Reg) 3674280031Sdim return false; 3675280031Sdim bool Op0IsKill = hasTrivialKill(II->getOperand(0)); 3676280031Sdim 3677280031Sdim unsigned ResultReg = fastEmit_r(VT, VT, ISD::FSQRT, Op0Reg, Op0IsKill); 3678280031Sdim if (!ResultReg) 3679280031Sdim return false; 3680280031Sdim 3681280031Sdim updateValueMap(II, ResultReg); 3682280031Sdim return true; 3683274955Ssvnmir } 3684280031Sdim case Intrinsic::sadd_with_overflow: 3685280031Sdim case Intrinsic::uadd_with_overflow: 3686280031Sdim case Intrinsic::ssub_with_overflow: 3687280031Sdim case Intrinsic::usub_with_overflow: 3688280031Sdim case Intrinsic::smul_with_overflow: 3689280031Sdim case Intrinsic::umul_with_overflow: { 3690280031Sdim // This implements the basic lowering of the xalu with overflow intrinsics. 3691280031Sdim const Function *Callee = II->getCalledFunction(); 3692280031Sdim auto *Ty = cast<StructType>(Callee->getReturnType()); 3693280031Sdim Type *RetTy = Ty->getTypeAtIndex(0U); 3694280031Sdim 3695280031Sdim MVT VT; 3696280031Sdim if (!isTypeLegal(RetTy, VT)) 3697280031Sdim return false; 3698280031Sdim 3699280031Sdim if (VT != MVT::i32 && VT != MVT::i64) 3700280031Sdim return false; 3701280031Sdim 3702280031Sdim const Value *LHS = II->getArgOperand(0); 3703280031Sdim const Value *RHS = II->getArgOperand(1); 3704280031Sdim // Canonicalize immediate to the RHS. 3705280031Sdim if (isa<ConstantInt>(LHS) && !isa<ConstantInt>(RHS) && 3706280031Sdim isCommutativeIntrinsic(II)) 3707280031Sdim std::swap(LHS, RHS); 3708280031Sdim 3709280031Sdim // Simplify multiplies. 3710288943Sdim Intrinsic::ID IID = II->getIntrinsicID(); 3711280031Sdim switch (IID) { 3712280031Sdim default: 3713280031Sdim break; 3714280031Sdim case Intrinsic::smul_with_overflow: 3715280031Sdim if (const auto *C = dyn_cast<ConstantInt>(RHS)) 3716280031Sdim if (C->getValue() == 2) { 3717280031Sdim IID = Intrinsic::sadd_with_overflow; 3718280031Sdim RHS = LHS; 3719280031Sdim } 3720280031Sdim break; 3721280031Sdim case Intrinsic::umul_with_overflow: 3722280031Sdim if (const auto *C = dyn_cast<ConstantInt>(RHS)) 3723280031Sdim if (C->getValue() == 2) { 3724280031Sdim IID = Intrinsic::uadd_with_overflow; 3725280031Sdim RHS = LHS; 3726280031Sdim } 3727280031Sdim break; 3728280031Sdim } 3729280031Sdim 3730280031Sdim unsigned ResultReg1 = 0, ResultReg2 = 0, MulReg = 0; 3731280031Sdim AArch64CC::CondCode CC = AArch64CC::Invalid; 3732280031Sdim switch (IID) { 3733280031Sdim default: llvm_unreachable("Unexpected intrinsic!"); 3734280031Sdim case Intrinsic::sadd_with_overflow: 3735280031Sdim ResultReg1 = emitAdd(VT, LHS, RHS, /*SetFlags=*/true); 3736280031Sdim CC = AArch64CC::VS; 3737280031Sdim break; 3738280031Sdim case Intrinsic::uadd_with_overflow: 3739280031Sdim ResultReg1 = emitAdd(VT, LHS, RHS, /*SetFlags=*/true); 3740280031Sdim CC = AArch64CC::HS; 3741280031Sdim break; 3742280031Sdim case Intrinsic::ssub_with_overflow: 3743280031Sdim ResultReg1 = emitSub(VT, LHS, RHS, /*SetFlags=*/true); 3744280031Sdim CC = AArch64CC::VS; 3745280031Sdim break; 3746280031Sdim case Intrinsic::usub_with_overflow: 3747280031Sdim ResultReg1 = emitSub(VT, LHS, RHS, /*SetFlags=*/true); 3748280031Sdim CC = AArch64CC::LO; 3749280031Sdim break; 3750280031Sdim case Intrinsic::smul_with_overflow: { 3751280031Sdim CC = AArch64CC::NE; 3752280031Sdim unsigned LHSReg = getRegForValue(LHS); 3753280031Sdim if (!LHSReg) 3754280031Sdim return false; 3755280031Sdim bool LHSIsKill = hasTrivialKill(LHS); 3756280031Sdim 3757280031Sdim unsigned RHSReg = getRegForValue(RHS); 3758280031Sdim if (!RHSReg) 3759280031Sdim return false; 3760280031Sdim bool RHSIsKill = hasTrivialKill(RHS); 3761280031Sdim 3762280031Sdim if (VT == MVT::i32) { 3763280031Sdim MulReg = emitSMULL_rr(MVT::i64, LHSReg, LHSIsKill, RHSReg, RHSIsKill); 3764280031Sdim unsigned ShiftReg = emitLSR_ri(MVT::i64, MVT::i64, MulReg, 3765280031Sdim /*IsKill=*/false, 32); 3766280031Sdim MulReg = fastEmitInst_extractsubreg(VT, MulReg, /*IsKill=*/true, 3767280031Sdim AArch64::sub_32); 3768280031Sdim ShiftReg = fastEmitInst_extractsubreg(VT, ShiftReg, /*IsKill=*/true, 3769280031Sdim AArch64::sub_32); 3770280031Sdim emitSubs_rs(VT, ShiftReg, /*IsKill=*/true, MulReg, /*IsKill=*/false, 3771280031Sdim AArch64_AM::ASR, 31, /*WantResult=*/false); 3772280031Sdim } else { 3773280031Sdim assert(VT == MVT::i64 && "Unexpected value type."); 3774288943Sdim // LHSReg and RHSReg cannot be killed by this Mul, since they are 3775288943Sdim // reused in the next instruction. 3776288943Sdim MulReg = emitMul_rr(VT, LHSReg, /*IsKill=*/false, RHSReg, 3777288943Sdim /*IsKill=*/false); 3778280031Sdim unsigned SMULHReg = fastEmit_rr(VT, VT, ISD::MULHS, LHSReg, LHSIsKill, 3779280031Sdim RHSReg, RHSIsKill); 3780280031Sdim emitSubs_rs(VT, SMULHReg, /*IsKill=*/true, MulReg, /*IsKill=*/false, 3781280031Sdim AArch64_AM::ASR, 63, /*WantResult=*/false); 3782280031Sdim } 3783280031Sdim break; 3784280031Sdim } 3785280031Sdim case Intrinsic::umul_with_overflow: { 3786280031Sdim CC = AArch64CC::NE; 3787280031Sdim unsigned LHSReg = getRegForValue(LHS); 3788280031Sdim if (!LHSReg) 3789280031Sdim return false; 3790280031Sdim bool LHSIsKill = hasTrivialKill(LHS); 3791280031Sdim 3792280031Sdim unsigned RHSReg = getRegForValue(RHS); 3793280031Sdim if (!RHSReg) 3794280031Sdim return false; 3795280031Sdim bool RHSIsKill = hasTrivialKill(RHS); 3796280031Sdim 3797280031Sdim if (VT == MVT::i32) { 3798280031Sdim MulReg = emitUMULL_rr(MVT::i64, LHSReg, LHSIsKill, RHSReg, RHSIsKill); 3799280031Sdim emitSubs_rs(MVT::i64, AArch64::XZR, /*IsKill=*/true, MulReg, 3800280031Sdim /*IsKill=*/false, AArch64_AM::LSR, 32, 3801280031Sdim /*WantResult=*/false); 3802280031Sdim MulReg = fastEmitInst_extractsubreg(VT, MulReg, /*IsKill=*/true, 3803280031Sdim AArch64::sub_32); 3804280031Sdim } else { 3805280031Sdim assert(VT == MVT::i64 && "Unexpected value type."); 3806288943Sdim // LHSReg and RHSReg cannot be killed by this Mul, since they are 3807288943Sdim // reused in the next instruction. 3808288943Sdim MulReg = emitMul_rr(VT, LHSReg, /*IsKill=*/false, RHSReg, 3809288943Sdim /*IsKill=*/false); 3810280031Sdim unsigned UMULHReg = fastEmit_rr(VT, VT, ISD::MULHU, LHSReg, LHSIsKill, 3811280031Sdim RHSReg, RHSIsKill); 3812280031Sdim emitSubs_rr(VT, AArch64::XZR, /*IsKill=*/true, UMULHReg, 3813280031Sdim /*IsKill=*/false, /*WantResult=*/false); 3814280031Sdim } 3815280031Sdim break; 3816280031Sdim } 3817280031Sdim } 3818280031Sdim 3819280031Sdim if (MulReg) { 3820280031Sdim ResultReg1 = createResultReg(TLI.getRegClassFor(VT)); 3821280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, 3822280031Sdim TII.get(TargetOpcode::COPY), ResultReg1).addReg(MulReg); 3823280031Sdim } 3824280031Sdim 3825344779Sdim if (!ResultReg1) 3826344779Sdim return false; 3827344779Sdim 3828280031Sdim ResultReg2 = fastEmitInst_rri(AArch64::CSINCWr, &AArch64::GPR32RegClass, 3829280031Sdim AArch64::WZR, /*IsKill=*/true, AArch64::WZR, 3830280031Sdim /*IsKill=*/true, getInvertedCondCode(CC)); 3831280031Sdim (void)ResultReg2; 3832280031Sdim assert((ResultReg1 + 1) == ResultReg2 && 3833280031Sdim "Nonconsecutive result registers."); 3834280031Sdim updateValueMap(II, ResultReg1, 2); 3835280031Sdim return true; 3836280031Sdim } 3837280031Sdim } 3838274955Ssvnmir return false; 3839274955Ssvnmir} 3840274955Ssvnmir 3841280031Sdimbool AArch64FastISel::selectRet(const Instruction *I) { 3842274955Ssvnmir const ReturnInst *Ret = cast<ReturnInst>(I); 3843274955Ssvnmir const Function &F = *I->getParent()->getParent(); 3844274955Ssvnmir 3845274955Ssvnmir if (!FuncInfo.CanLowerReturn) 3846274955Ssvnmir return false; 3847274955Ssvnmir 3848274955Ssvnmir if (F.isVarArg()) 3849274955Ssvnmir return false; 3850274955Ssvnmir 3851309124Sdim if (TLI.supportSwiftError() && 3852309124Sdim F.getAttributes().hasAttrSomewhere(Attribute::SwiftError)) 3853309124Sdim return false; 3854309124Sdim 3855296417Sdim if (TLI.supportSplitCSR(FuncInfo.MF)) 3856296417Sdim return false; 3857296417Sdim 3858274955Ssvnmir // Build a list of return value registers. 3859274955Ssvnmir SmallVector<unsigned, 4> RetRegs; 3860274955Ssvnmir 3861274955Ssvnmir if (Ret->getNumOperands() > 0) { 3862274955Ssvnmir CallingConv::ID CC = F.getCallingConv(); 3863274955Ssvnmir SmallVector<ISD::OutputArg, 4> Outs; 3864341825Sdim GetReturnInfo(CC, F.getReturnType(), F.getAttributes(), Outs, TLI, DL); 3865274955Ssvnmir 3866274955Ssvnmir // Analyze operands of the call, assigning locations to each operand. 3867274955Ssvnmir SmallVector<CCValAssign, 16> ValLocs; 3868280031Sdim CCState CCInfo(CC, F.isVarArg(), *FuncInfo.MF, ValLocs, I->getContext()); 3869274955Ssvnmir CCAssignFn *RetCC = CC == CallingConv::WebKit_JS ? RetCC_AArch64_WebKit_JS 3870274955Ssvnmir : RetCC_AArch64_AAPCS; 3871274955Ssvnmir CCInfo.AnalyzeReturn(Outs, RetCC); 3872274955Ssvnmir 3873274955Ssvnmir // Only handle a single return value for now. 3874274955Ssvnmir if (ValLocs.size() != 1) 3875274955Ssvnmir return false; 3876274955Ssvnmir 3877274955Ssvnmir CCValAssign &VA = ValLocs[0]; 3878274955Ssvnmir const Value *RV = Ret->getOperand(0); 3879274955Ssvnmir 3880274955Ssvnmir // Don't bother handling odd stuff for now. 3881280031Sdim if ((VA.getLocInfo() != CCValAssign::Full) && 3882280031Sdim (VA.getLocInfo() != CCValAssign::BCvt)) 3883274955Ssvnmir return false; 3884280031Sdim 3885274955Ssvnmir // Only handle register returns for now. 3886274955Ssvnmir if (!VA.isRegLoc()) 3887274955Ssvnmir return false; 3888280031Sdim 3889274955Ssvnmir unsigned Reg = getRegForValue(RV); 3890274955Ssvnmir if (Reg == 0) 3891274955Ssvnmir return false; 3892274955Ssvnmir 3893274955Ssvnmir unsigned SrcReg = Reg + VA.getValNo(); 3894360784Sdim Register DestReg = VA.getLocReg(); 3895274955Ssvnmir // Avoid a cross-class copy. This is very unlikely. 3896274955Ssvnmir if (!MRI.getRegClass(SrcReg)->contains(DestReg)) 3897274955Ssvnmir return false; 3898274955Ssvnmir 3899288943Sdim EVT RVEVT = TLI.getValueType(DL, RV->getType()); 3900274955Ssvnmir if (!RVEVT.isSimple()) 3901274955Ssvnmir return false; 3902274955Ssvnmir 3903274955Ssvnmir // Vectors (of > 1 lane) in big endian need tricky handling. 3904280031Sdim if (RVEVT.isVector() && RVEVT.getVectorNumElements() > 1 && 3905280031Sdim !Subtarget->isLittleEndian()) 3906274955Ssvnmir return false; 3907274955Ssvnmir 3908274955Ssvnmir MVT RVVT = RVEVT.getSimpleVT(); 3909274955Ssvnmir if (RVVT == MVT::f128) 3910274955Ssvnmir return false; 3911280031Sdim 3912274955Ssvnmir MVT DestVT = VA.getValVT(); 3913274955Ssvnmir // Special handling for extended integers. 3914274955Ssvnmir if (RVVT != DestVT) { 3915274955Ssvnmir if (RVVT != MVT::i1 && RVVT != MVT::i8 && RVVT != MVT::i16) 3916274955Ssvnmir return false; 3917274955Ssvnmir 3918274955Ssvnmir if (!Outs[0].Flags.isZExt() && !Outs[0].Flags.isSExt()) 3919274955Ssvnmir return false; 3920274955Ssvnmir 3921280031Sdim bool IsZExt = Outs[0].Flags.isZExt(); 3922280031Sdim SrcReg = emitIntExt(RVVT, SrcReg, DestVT, IsZExt); 3923274955Ssvnmir if (SrcReg == 0) 3924274955Ssvnmir return false; 3925274955Ssvnmir } 3926274955Ssvnmir 3927360784Sdim // "Callee" (i.e. value producer) zero extends pointers at function 3928360784Sdim // boundary. 3929360784Sdim if (Subtarget->isTargetILP32() && RV->getType()->isPointerTy()) 3930360784Sdim SrcReg = emitAnd_ri(MVT::i64, SrcReg, false, 0xffffffff); 3931360784Sdim 3932274955Ssvnmir // Make the copy. 3933274955Ssvnmir BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, 3934274955Ssvnmir TII.get(TargetOpcode::COPY), DestReg).addReg(SrcReg); 3935274955Ssvnmir 3936274955Ssvnmir // Add register to return instruction. 3937274955Ssvnmir RetRegs.push_back(VA.getLocReg()); 3938274955Ssvnmir } 3939274955Ssvnmir 3940274955Ssvnmir MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, 3941274955Ssvnmir TII.get(AArch64::RET_ReallyLR)); 3942296417Sdim for (unsigned RetReg : RetRegs) 3943296417Sdim MIB.addReg(RetReg, RegState::Implicit); 3944274955Ssvnmir return true; 3945274955Ssvnmir} 3946274955Ssvnmir 3947280031Sdimbool AArch64FastISel::selectTrunc(const Instruction *I) { 3948274955Ssvnmir Type *DestTy = I->getType(); 3949274955Ssvnmir Value *Op = I->getOperand(0); 3950274955Ssvnmir Type *SrcTy = Op->getType(); 3951274955Ssvnmir 3952288943Sdim EVT SrcEVT = TLI.getValueType(DL, SrcTy, true); 3953288943Sdim EVT DestEVT = TLI.getValueType(DL, DestTy, true); 3954274955Ssvnmir if (!SrcEVT.isSimple()) 3955274955Ssvnmir return false; 3956274955Ssvnmir if (!DestEVT.isSimple()) 3957274955Ssvnmir return false; 3958274955Ssvnmir 3959274955Ssvnmir MVT SrcVT = SrcEVT.getSimpleVT(); 3960274955Ssvnmir MVT DestVT = DestEVT.getSimpleVT(); 3961274955Ssvnmir 3962274955Ssvnmir if (SrcVT != MVT::i64 && SrcVT != MVT::i32 && SrcVT != MVT::i16 && 3963274955Ssvnmir SrcVT != MVT::i8) 3964274955Ssvnmir return false; 3965274955Ssvnmir if (DestVT != MVT::i32 && DestVT != MVT::i16 && DestVT != MVT::i8 && 3966274955Ssvnmir DestVT != MVT::i1) 3967274955Ssvnmir return false; 3968274955Ssvnmir 3969274955Ssvnmir unsigned SrcReg = getRegForValue(Op); 3970274955Ssvnmir if (!SrcReg) 3971274955Ssvnmir return false; 3972280031Sdim bool SrcIsKill = hasTrivialKill(Op); 3973274955Ssvnmir 3974274955Ssvnmir // If we're truncating from i64 to a smaller non-legal type then generate an 3975280031Sdim // AND. Otherwise, we know the high bits are undefined and a truncate only 3976280031Sdim // generate a COPY. We cannot mark the source register also as result 3977280031Sdim // register, because this can incorrectly transfer the kill flag onto the 3978280031Sdim // source register. 3979280031Sdim unsigned ResultReg; 3980274955Ssvnmir if (SrcVT == MVT::i64) { 3981274955Ssvnmir uint64_t Mask = 0; 3982274955Ssvnmir switch (DestVT.SimpleTy) { 3983274955Ssvnmir default: 3984274955Ssvnmir // Trunc i64 to i32 is handled by the target-independent fast-isel. 3985274955Ssvnmir return false; 3986274955Ssvnmir case MVT::i1: 3987274955Ssvnmir Mask = 0x1; 3988274955Ssvnmir break; 3989274955Ssvnmir case MVT::i8: 3990274955Ssvnmir Mask = 0xff; 3991274955Ssvnmir break; 3992274955Ssvnmir case MVT::i16: 3993274955Ssvnmir Mask = 0xffff; 3994274955Ssvnmir break; 3995274955Ssvnmir } 3996274955Ssvnmir // Issue an extract_subreg to get the lower 32-bits. 3997280031Sdim unsigned Reg32 = fastEmitInst_extractsubreg(MVT::i32, SrcReg, SrcIsKill, 3998274955Ssvnmir AArch64::sub_32); 3999274955Ssvnmir // Create the AND instruction which performs the actual truncation. 4000280031Sdim ResultReg = emitAnd_ri(MVT::i32, Reg32, /*IsKill=*/true, Mask); 4001280031Sdim assert(ResultReg && "Unexpected AND instruction emission failure."); 4002280031Sdim } else { 4003280031Sdim ResultReg = createResultReg(&AArch64::GPR32RegClass); 4004280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, 4005280031Sdim TII.get(TargetOpcode::COPY), ResultReg) 4006280031Sdim .addReg(SrcReg, getKillRegState(SrcIsKill)); 4007274955Ssvnmir } 4008274955Ssvnmir 4009280031Sdim updateValueMap(I, ResultReg); 4010274955Ssvnmir return true; 4011274955Ssvnmir} 4012274955Ssvnmir 4013280031Sdimunsigned AArch64FastISel::emiti1Ext(unsigned SrcReg, MVT DestVT, bool IsZExt) { 4014274955Ssvnmir assert((DestVT == MVT::i8 || DestVT == MVT::i16 || DestVT == MVT::i32 || 4015274955Ssvnmir DestVT == MVT::i64) && 4016274955Ssvnmir "Unexpected value type."); 4017274955Ssvnmir // Handle i8 and i16 as i32. 4018274955Ssvnmir if (DestVT == MVT::i8 || DestVT == MVT::i16) 4019274955Ssvnmir DestVT = MVT::i32; 4020274955Ssvnmir 4021280031Sdim if (IsZExt) { 4022280031Sdim unsigned ResultReg = emitAnd_ri(MVT::i32, SrcReg, /*TODO:IsKill=*/false, 1); 4023280031Sdim assert(ResultReg && "Unexpected AND instruction emission failure."); 4024274955Ssvnmir if (DestVT == MVT::i64) { 4025274955Ssvnmir // We're ZExt i1 to i64. The ANDWri Wd, Ws, #1 implicitly clears the 4026274955Ssvnmir // upper 32 bits. Emit a SUBREG_TO_REG to extend from Wd to Xd. 4027360784Sdim Register Reg64 = MRI.createVirtualRegister(&AArch64::GPR64RegClass); 4028274955Ssvnmir BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, 4029274955Ssvnmir TII.get(AArch64::SUBREG_TO_REG), Reg64) 4030274955Ssvnmir .addImm(0) 4031274955Ssvnmir .addReg(ResultReg) 4032274955Ssvnmir .addImm(AArch64::sub_32); 4033274955Ssvnmir ResultReg = Reg64; 4034274955Ssvnmir } 4035274955Ssvnmir return ResultReg; 4036274955Ssvnmir } else { 4037274955Ssvnmir if (DestVT == MVT::i64) { 4038274955Ssvnmir // FIXME: We're SExt i1 to i64. 4039274955Ssvnmir return 0; 4040274955Ssvnmir } 4041280031Sdim return fastEmitInst_rii(AArch64::SBFMWri, &AArch64::GPR32RegClass, SrcReg, 4042280031Sdim /*TODO:IsKill=*/false, 0, 0); 4043280031Sdim } 4044280031Sdim} 4045280031Sdim 4046280031Sdimunsigned AArch64FastISel::emitMul_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, 4047280031Sdim unsigned Op1, bool Op1IsKill) { 4048280031Sdim unsigned Opc, ZReg; 4049280031Sdim switch (RetVT.SimpleTy) { 4050280031Sdim default: return 0; 4051280031Sdim case MVT::i8: 4052280031Sdim case MVT::i16: 4053280031Sdim case MVT::i32: 4054280031Sdim RetVT = MVT::i32; 4055280031Sdim Opc = AArch64::MADDWrrr; ZReg = AArch64::WZR; break; 4056280031Sdim case MVT::i64: 4057280031Sdim Opc = AArch64::MADDXrrr; ZReg = AArch64::XZR; break; 4058280031Sdim } 4059280031Sdim 4060280031Sdim const TargetRegisterClass *RC = 4061280031Sdim (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass; 4062280031Sdim return fastEmitInst_rrr(Opc, RC, Op0, Op0IsKill, Op1, Op1IsKill, 4063280031Sdim /*IsKill=*/ZReg, true); 4064280031Sdim} 4065280031Sdim 4066280031Sdimunsigned AArch64FastISel::emitSMULL_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, 4067280031Sdim unsigned Op1, bool Op1IsKill) { 4068280031Sdim if (RetVT != MVT::i64) 4069280031Sdim return 0; 4070280031Sdim 4071280031Sdim return fastEmitInst_rrr(AArch64::SMADDLrrr, &AArch64::GPR64RegClass, 4072280031Sdim Op0, Op0IsKill, Op1, Op1IsKill, 4073280031Sdim AArch64::XZR, /*IsKill=*/true); 4074280031Sdim} 4075280031Sdim 4076280031Sdimunsigned AArch64FastISel::emitUMULL_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, 4077280031Sdim unsigned Op1, bool Op1IsKill) { 4078280031Sdim if (RetVT != MVT::i64) 4079280031Sdim return 0; 4080280031Sdim 4081280031Sdim return fastEmitInst_rrr(AArch64::UMADDLrrr, &AArch64::GPR64RegClass, 4082280031Sdim Op0, Op0IsKill, Op1, Op1IsKill, 4083280031Sdim AArch64::XZR, /*IsKill=*/true); 4084280031Sdim} 4085280031Sdim 4086280031Sdimunsigned AArch64FastISel::emitLSL_rr(MVT RetVT, unsigned Op0Reg, bool Op0IsKill, 4087280031Sdim unsigned Op1Reg, bool Op1IsKill) { 4088280031Sdim unsigned Opc = 0; 4089280031Sdim bool NeedTrunc = false; 4090280031Sdim uint64_t Mask = 0; 4091280031Sdim switch (RetVT.SimpleTy) { 4092280031Sdim default: return 0; 4093280031Sdim case MVT::i8: Opc = AArch64::LSLVWr; NeedTrunc = true; Mask = 0xff; break; 4094280031Sdim case MVT::i16: Opc = AArch64::LSLVWr; NeedTrunc = true; Mask = 0xffff; break; 4095280031Sdim case MVT::i32: Opc = AArch64::LSLVWr; break; 4096280031Sdim case MVT::i64: Opc = AArch64::LSLVXr; break; 4097280031Sdim } 4098280031Sdim 4099280031Sdim const TargetRegisterClass *RC = 4100280031Sdim (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass; 4101280031Sdim if (NeedTrunc) { 4102280031Sdim Op1Reg = emitAnd_ri(MVT::i32, Op1Reg, Op1IsKill, Mask); 4103280031Sdim Op1IsKill = true; 4104280031Sdim } 4105280031Sdim unsigned ResultReg = fastEmitInst_rr(Opc, RC, Op0Reg, Op0IsKill, Op1Reg, 4106280031Sdim Op1IsKill); 4107280031Sdim if (NeedTrunc) 4108280031Sdim ResultReg = emitAnd_ri(MVT::i32, ResultReg, /*IsKill=*/true, Mask); 4109280031Sdim return ResultReg; 4110280031Sdim} 4111280031Sdim 4112280031Sdimunsigned AArch64FastISel::emitLSL_ri(MVT RetVT, MVT SrcVT, unsigned Op0, 4113280031Sdim bool Op0IsKill, uint64_t Shift, 4114280031Sdim bool IsZExt) { 4115280031Sdim assert(RetVT.SimpleTy >= SrcVT.SimpleTy && 4116280031Sdim "Unexpected source/return type pair."); 4117280031Sdim assert((SrcVT == MVT::i1 || SrcVT == MVT::i8 || SrcVT == MVT::i16 || 4118280031Sdim SrcVT == MVT::i32 || SrcVT == MVT::i64) && 4119280031Sdim "Unexpected source value type."); 4120280031Sdim assert((RetVT == MVT::i8 || RetVT == MVT::i16 || RetVT == MVT::i32 || 4121280031Sdim RetVT == MVT::i64) && "Unexpected return value type."); 4122280031Sdim 4123280031Sdim bool Is64Bit = (RetVT == MVT::i64); 4124280031Sdim unsigned RegSize = Is64Bit ? 64 : 32; 4125280031Sdim unsigned DstBits = RetVT.getSizeInBits(); 4126280031Sdim unsigned SrcBits = SrcVT.getSizeInBits(); 4127280031Sdim const TargetRegisterClass *RC = 4128280031Sdim Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass; 4129280031Sdim 4130280031Sdim // Just emit a copy for "zero" shifts. 4131280031Sdim if (Shift == 0) { 4132280031Sdim if (RetVT == SrcVT) { 4133280031Sdim unsigned ResultReg = createResultReg(RC); 4134280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, 4135280031Sdim TII.get(TargetOpcode::COPY), ResultReg) 4136280031Sdim .addReg(Op0, getKillRegState(Op0IsKill)); 4137280031Sdim return ResultReg; 4138280031Sdim } else 4139280031Sdim return emitIntExt(SrcVT, Op0, RetVT, IsZExt); 4140280031Sdim } 4141280031Sdim 4142280031Sdim // Don't deal with undefined shifts. 4143280031Sdim if (Shift >= DstBits) 4144280031Sdim return 0; 4145280031Sdim 4146280031Sdim // For immediate shifts we can fold the zero-/sign-extension into the shift. 4147280031Sdim // {S|U}BFM Wd, Wn, #r, #s 4148280031Sdim // Wd<32+s-r,32-r> = Wn<s:0> when r > s 4149280031Sdim 4150280031Sdim // %1 = {s|z}ext i8 {0b1010_1010|0b0101_0101} to i16 4151280031Sdim // %2 = shl i16 %1, 4 4152280031Sdim // Wd<32+7-28,32-28> = Wn<7:0> <- clamp s to 7 4153280031Sdim // 0b1111_1111_1111_1111__1111_1010_1010_0000 sext 4154280031Sdim // 0b0000_0000_0000_0000__0000_0101_0101_0000 sext | zext 4155280031Sdim // 0b0000_0000_0000_0000__0000_1010_1010_0000 zext 4156280031Sdim 4157280031Sdim // %1 = {s|z}ext i8 {0b1010_1010|0b0101_0101} to i16 4158280031Sdim // %2 = shl i16 %1, 8 4159280031Sdim // Wd<32+7-24,32-24> = Wn<7:0> 4160280031Sdim // 0b1111_1111_1111_1111__1010_1010_0000_0000 sext 4161280031Sdim // 0b0000_0000_0000_0000__0101_0101_0000_0000 sext | zext 4162280031Sdim // 0b0000_0000_0000_0000__1010_1010_0000_0000 zext 4163280031Sdim 4164280031Sdim // %1 = {s|z}ext i8 {0b1010_1010|0b0101_0101} to i16 4165280031Sdim // %2 = shl i16 %1, 12 4166280031Sdim // Wd<32+3-20,32-20> = Wn<3:0> 4167280031Sdim // 0b1111_1111_1111_1111__1010_0000_0000_0000 sext 4168280031Sdim // 0b0000_0000_0000_0000__0101_0000_0000_0000 sext | zext 4169280031Sdim // 0b0000_0000_0000_0000__1010_0000_0000_0000 zext 4170280031Sdim 4171280031Sdim unsigned ImmR = RegSize - Shift; 4172280031Sdim // Limit the width to the length of the source type. 4173280031Sdim unsigned ImmS = std::min<unsigned>(SrcBits - 1, DstBits - 1 - Shift); 4174280031Sdim static const unsigned OpcTable[2][2] = { 4175280031Sdim {AArch64::SBFMWri, AArch64::SBFMXri}, 4176280031Sdim {AArch64::UBFMWri, AArch64::UBFMXri} 4177280031Sdim }; 4178280031Sdim unsigned Opc = OpcTable[IsZExt][Is64Bit]; 4179280031Sdim if (SrcVT.SimpleTy <= MVT::i32 && RetVT == MVT::i64) { 4180360784Sdim Register TmpReg = MRI.createVirtualRegister(RC); 4181280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, 4182280031Sdim TII.get(AArch64::SUBREG_TO_REG), TmpReg) 4183274955Ssvnmir .addImm(0) 4184280031Sdim .addReg(Op0, getKillRegState(Op0IsKill)) 4185280031Sdim .addImm(AArch64::sub_32); 4186280031Sdim Op0 = TmpReg; 4187280031Sdim Op0IsKill = true; 4188274955Ssvnmir } 4189280031Sdim return fastEmitInst_rii(Opc, RC, Op0, Op0IsKill, ImmR, ImmS); 4190274955Ssvnmir} 4191274955Ssvnmir 4192280031Sdimunsigned AArch64FastISel::emitLSR_rr(MVT RetVT, unsigned Op0Reg, bool Op0IsKill, 4193280031Sdim unsigned Op1Reg, bool Op1IsKill) { 4194280031Sdim unsigned Opc = 0; 4195280031Sdim bool NeedTrunc = false; 4196280031Sdim uint64_t Mask = 0; 4197280031Sdim switch (RetVT.SimpleTy) { 4198280031Sdim default: return 0; 4199280031Sdim case MVT::i8: Opc = AArch64::LSRVWr; NeedTrunc = true; Mask = 0xff; break; 4200280031Sdim case MVT::i16: Opc = AArch64::LSRVWr; NeedTrunc = true; Mask = 0xffff; break; 4201280031Sdim case MVT::i32: Opc = AArch64::LSRVWr; break; 4202280031Sdim case MVT::i64: Opc = AArch64::LSRVXr; break; 4203280031Sdim } 4204280031Sdim 4205280031Sdim const TargetRegisterClass *RC = 4206280031Sdim (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass; 4207280031Sdim if (NeedTrunc) { 4208280031Sdim Op0Reg = emitAnd_ri(MVT::i32, Op0Reg, Op0IsKill, Mask); 4209280031Sdim Op1Reg = emitAnd_ri(MVT::i32, Op1Reg, Op1IsKill, Mask); 4210280031Sdim Op0IsKill = Op1IsKill = true; 4211280031Sdim } 4212280031Sdim unsigned ResultReg = fastEmitInst_rr(Opc, RC, Op0Reg, Op0IsKill, Op1Reg, 4213280031Sdim Op1IsKill); 4214280031Sdim if (NeedTrunc) 4215280031Sdim ResultReg = emitAnd_ri(MVT::i32, ResultReg, /*IsKill=*/true, Mask); 4216280031Sdim return ResultReg; 4217280031Sdim} 4218280031Sdim 4219280031Sdimunsigned AArch64FastISel::emitLSR_ri(MVT RetVT, MVT SrcVT, unsigned Op0, 4220280031Sdim bool Op0IsKill, uint64_t Shift, 4221280031Sdim bool IsZExt) { 4222280031Sdim assert(RetVT.SimpleTy >= SrcVT.SimpleTy && 4223280031Sdim "Unexpected source/return type pair."); 4224280031Sdim assert((SrcVT == MVT::i1 || SrcVT == MVT::i8 || SrcVT == MVT::i16 || 4225280031Sdim SrcVT == MVT::i32 || SrcVT == MVT::i64) && 4226280031Sdim "Unexpected source value type."); 4227280031Sdim assert((RetVT == MVT::i8 || RetVT == MVT::i16 || RetVT == MVT::i32 || 4228280031Sdim RetVT == MVT::i64) && "Unexpected return value type."); 4229280031Sdim 4230280031Sdim bool Is64Bit = (RetVT == MVT::i64); 4231280031Sdim unsigned RegSize = Is64Bit ? 64 : 32; 4232280031Sdim unsigned DstBits = RetVT.getSizeInBits(); 4233280031Sdim unsigned SrcBits = SrcVT.getSizeInBits(); 4234280031Sdim const TargetRegisterClass *RC = 4235280031Sdim Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass; 4236280031Sdim 4237280031Sdim // Just emit a copy for "zero" shifts. 4238280031Sdim if (Shift == 0) { 4239280031Sdim if (RetVT == SrcVT) { 4240280031Sdim unsigned ResultReg = createResultReg(RC); 4241280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, 4242280031Sdim TII.get(TargetOpcode::COPY), ResultReg) 4243280031Sdim .addReg(Op0, getKillRegState(Op0IsKill)); 4244280031Sdim return ResultReg; 4245280031Sdim } else 4246280031Sdim return emitIntExt(SrcVT, Op0, RetVT, IsZExt); 4247280031Sdim } 4248280031Sdim 4249280031Sdim // Don't deal with undefined shifts. 4250280031Sdim if (Shift >= DstBits) 4251280031Sdim return 0; 4252280031Sdim 4253280031Sdim // For immediate shifts we can fold the zero-/sign-extension into the shift. 4254280031Sdim // {S|U}BFM Wd, Wn, #r, #s 4255280031Sdim // Wd<s-r:0> = Wn<s:r> when r <= s 4256280031Sdim 4257280031Sdim // %1 = {s|z}ext i8 {0b1010_1010|0b0101_0101} to i16 4258280031Sdim // %2 = lshr i16 %1, 4 4259280031Sdim // Wd<7-4:0> = Wn<7:4> 4260280031Sdim // 0b0000_0000_0000_0000__0000_1111_1111_1010 sext 4261280031Sdim // 0b0000_0000_0000_0000__0000_0000_0000_0101 sext | zext 4262280031Sdim // 0b0000_0000_0000_0000__0000_0000_0000_1010 zext 4263280031Sdim 4264280031Sdim // %1 = {s|z}ext i8 {0b1010_1010|0b0101_0101} to i16 4265280031Sdim // %2 = lshr i16 %1, 8 4266280031Sdim // Wd<7-7,0> = Wn<7:7> 4267280031Sdim // 0b0000_0000_0000_0000__0000_0000_1111_1111 sext 4268280031Sdim // 0b0000_0000_0000_0000__0000_0000_0000_0000 sext 4269280031Sdim // 0b0000_0000_0000_0000__0000_0000_0000_0000 zext 4270280031Sdim 4271280031Sdim // %1 = {s|z}ext i8 {0b1010_1010|0b0101_0101} to i16 4272280031Sdim // %2 = lshr i16 %1, 12 4273280031Sdim // Wd<7-7,0> = Wn<7:7> <- clamp r to 7 4274280031Sdim // 0b0000_0000_0000_0000__0000_0000_0000_1111 sext 4275280031Sdim // 0b0000_0000_0000_0000__0000_0000_0000_0000 sext 4276280031Sdim // 0b0000_0000_0000_0000__0000_0000_0000_0000 zext 4277280031Sdim 4278280031Sdim if (Shift >= SrcBits && IsZExt) 4279280031Sdim return materializeInt(ConstantInt::get(*Context, APInt(RegSize, 0)), RetVT); 4280280031Sdim 4281280031Sdim // It is not possible to fold a sign-extend into the LShr instruction. In this 4282280031Sdim // case emit a sign-extend. 4283280031Sdim if (!IsZExt) { 4284280031Sdim Op0 = emitIntExt(SrcVT, Op0, RetVT, IsZExt); 4285280031Sdim if (!Op0) 4286280031Sdim return 0; 4287280031Sdim Op0IsKill = true; 4288280031Sdim SrcVT = RetVT; 4289280031Sdim SrcBits = SrcVT.getSizeInBits(); 4290280031Sdim IsZExt = true; 4291280031Sdim } 4292280031Sdim 4293280031Sdim unsigned ImmR = std::min<unsigned>(SrcBits - 1, Shift); 4294280031Sdim unsigned ImmS = SrcBits - 1; 4295280031Sdim static const unsigned OpcTable[2][2] = { 4296280031Sdim {AArch64::SBFMWri, AArch64::SBFMXri}, 4297280031Sdim {AArch64::UBFMWri, AArch64::UBFMXri} 4298280031Sdim }; 4299280031Sdim unsigned Opc = OpcTable[IsZExt][Is64Bit]; 4300280031Sdim if (SrcVT.SimpleTy <= MVT::i32 && RetVT == MVT::i64) { 4301360784Sdim Register TmpReg = MRI.createVirtualRegister(RC); 4302280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, 4303280031Sdim TII.get(AArch64::SUBREG_TO_REG), TmpReg) 4304280031Sdim .addImm(0) 4305280031Sdim .addReg(Op0, getKillRegState(Op0IsKill)) 4306280031Sdim .addImm(AArch64::sub_32); 4307280031Sdim Op0 = TmpReg; 4308280031Sdim Op0IsKill = true; 4309280031Sdim } 4310280031Sdim return fastEmitInst_rii(Opc, RC, Op0, Op0IsKill, ImmR, ImmS); 4311280031Sdim} 4312280031Sdim 4313280031Sdimunsigned AArch64FastISel::emitASR_rr(MVT RetVT, unsigned Op0Reg, bool Op0IsKill, 4314280031Sdim unsigned Op1Reg, bool Op1IsKill) { 4315280031Sdim unsigned Opc = 0; 4316280031Sdim bool NeedTrunc = false; 4317280031Sdim uint64_t Mask = 0; 4318280031Sdim switch (RetVT.SimpleTy) { 4319280031Sdim default: return 0; 4320280031Sdim case MVT::i8: Opc = AArch64::ASRVWr; NeedTrunc = true; Mask = 0xff; break; 4321280031Sdim case MVT::i16: Opc = AArch64::ASRVWr; NeedTrunc = true; Mask = 0xffff; break; 4322280031Sdim case MVT::i32: Opc = AArch64::ASRVWr; break; 4323280031Sdim case MVT::i64: Opc = AArch64::ASRVXr; break; 4324280031Sdim } 4325280031Sdim 4326280031Sdim const TargetRegisterClass *RC = 4327280031Sdim (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass; 4328280031Sdim if (NeedTrunc) { 4329353358Sdim Op0Reg = emitIntExt(RetVT, Op0Reg, MVT::i32, /*isZExt=*/false); 4330280031Sdim Op1Reg = emitAnd_ri(MVT::i32, Op1Reg, Op1IsKill, Mask); 4331280031Sdim Op0IsKill = Op1IsKill = true; 4332280031Sdim } 4333280031Sdim unsigned ResultReg = fastEmitInst_rr(Opc, RC, Op0Reg, Op0IsKill, Op1Reg, 4334280031Sdim Op1IsKill); 4335280031Sdim if (NeedTrunc) 4336280031Sdim ResultReg = emitAnd_ri(MVT::i32, ResultReg, /*IsKill=*/true, Mask); 4337280031Sdim return ResultReg; 4338280031Sdim} 4339280031Sdim 4340280031Sdimunsigned AArch64FastISel::emitASR_ri(MVT RetVT, MVT SrcVT, unsigned Op0, 4341280031Sdim bool Op0IsKill, uint64_t Shift, 4342280031Sdim bool IsZExt) { 4343280031Sdim assert(RetVT.SimpleTy >= SrcVT.SimpleTy && 4344280031Sdim "Unexpected source/return type pair."); 4345280031Sdim assert((SrcVT == MVT::i1 || SrcVT == MVT::i8 || SrcVT == MVT::i16 || 4346280031Sdim SrcVT == MVT::i32 || SrcVT == MVT::i64) && 4347280031Sdim "Unexpected source value type."); 4348280031Sdim assert((RetVT == MVT::i8 || RetVT == MVT::i16 || RetVT == MVT::i32 || 4349280031Sdim RetVT == MVT::i64) && "Unexpected return value type."); 4350280031Sdim 4351280031Sdim bool Is64Bit = (RetVT == MVT::i64); 4352280031Sdim unsigned RegSize = Is64Bit ? 64 : 32; 4353280031Sdim unsigned DstBits = RetVT.getSizeInBits(); 4354280031Sdim unsigned SrcBits = SrcVT.getSizeInBits(); 4355280031Sdim const TargetRegisterClass *RC = 4356280031Sdim Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass; 4357280031Sdim 4358280031Sdim // Just emit a copy for "zero" shifts. 4359280031Sdim if (Shift == 0) { 4360280031Sdim if (RetVT == SrcVT) { 4361280031Sdim unsigned ResultReg = createResultReg(RC); 4362280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, 4363280031Sdim TII.get(TargetOpcode::COPY), ResultReg) 4364280031Sdim .addReg(Op0, getKillRegState(Op0IsKill)); 4365280031Sdim return ResultReg; 4366280031Sdim } else 4367280031Sdim return emitIntExt(SrcVT, Op0, RetVT, IsZExt); 4368280031Sdim } 4369280031Sdim 4370280031Sdim // Don't deal with undefined shifts. 4371280031Sdim if (Shift >= DstBits) 4372280031Sdim return 0; 4373280031Sdim 4374280031Sdim // For immediate shifts we can fold the zero-/sign-extension into the shift. 4375280031Sdim // {S|U}BFM Wd, Wn, #r, #s 4376280031Sdim // Wd<s-r:0> = Wn<s:r> when r <= s 4377280031Sdim 4378280031Sdim // %1 = {s|z}ext i8 {0b1010_1010|0b0101_0101} to i16 4379280031Sdim // %2 = ashr i16 %1, 4 4380280031Sdim // Wd<7-4:0> = Wn<7:4> 4381280031Sdim // 0b1111_1111_1111_1111__1111_1111_1111_1010 sext 4382280031Sdim // 0b0000_0000_0000_0000__0000_0000_0000_0101 sext | zext 4383280031Sdim // 0b0000_0000_0000_0000__0000_0000_0000_1010 zext 4384280031Sdim 4385280031Sdim // %1 = {s|z}ext i8 {0b1010_1010|0b0101_0101} to i16 4386280031Sdim // %2 = ashr i16 %1, 8 4387280031Sdim // Wd<7-7,0> = Wn<7:7> 4388280031Sdim // 0b1111_1111_1111_1111__1111_1111_1111_1111 sext 4389280031Sdim // 0b0000_0000_0000_0000__0000_0000_0000_0000 sext 4390280031Sdim // 0b0000_0000_0000_0000__0000_0000_0000_0000 zext 4391280031Sdim 4392280031Sdim // %1 = {s|z}ext i8 {0b1010_1010|0b0101_0101} to i16 4393280031Sdim // %2 = ashr i16 %1, 12 4394280031Sdim // Wd<7-7,0> = Wn<7:7> <- clamp r to 7 4395280031Sdim // 0b1111_1111_1111_1111__1111_1111_1111_1111 sext 4396280031Sdim // 0b0000_0000_0000_0000__0000_0000_0000_0000 sext 4397280031Sdim // 0b0000_0000_0000_0000__0000_0000_0000_0000 zext 4398280031Sdim 4399280031Sdim if (Shift >= SrcBits && IsZExt) 4400280031Sdim return materializeInt(ConstantInt::get(*Context, APInt(RegSize, 0)), RetVT); 4401280031Sdim 4402280031Sdim unsigned ImmR = std::min<unsigned>(SrcBits - 1, Shift); 4403280031Sdim unsigned ImmS = SrcBits - 1; 4404280031Sdim static const unsigned OpcTable[2][2] = { 4405280031Sdim {AArch64::SBFMWri, AArch64::SBFMXri}, 4406280031Sdim {AArch64::UBFMWri, AArch64::UBFMXri} 4407280031Sdim }; 4408280031Sdim unsigned Opc = OpcTable[IsZExt][Is64Bit]; 4409280031Sdim if (SrcVT.SimpleTy <= MVT::i32 && RetVT == MVT::i64) { 4410360784Sdim Register TmpReg = MRI.createVirtualRegister(RC); 4411280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, 4412280031Sdim TII.get(AArch64::SUBREG_TO_REG), TmpReg) 4413280031Sdim .addImm(0) 4414280031Sdim .addReg(Op0, getKillRegState(Op0IsKill)) 4415280031Sdim .addImm(AArch64::sub_32); 4416280031Sdim Op0 = TmpReg; 4417280031Sdim Op0IsKill = true; 4418280031Sdim } 4419280031Sdim return fastEmitInst_rii(Opc, RC, Op0, Op0IsKill, ImmR, ImmS); 4420280031Sdim} 4421280031Sdim 4422280031Sdimunsigned AArch64FastISel::emitIntExt(MVT SrcVT, unsigned SrcReg, MVT DestVT, 4423280031Sdim bool IsZExt) { 4424274955Ssvnmir assert(DestVT != MVT::i1 && "ZeroExt/SignExt an i1?"); 4425274955Ssvnmir 4426274955Ssvnmir // FastISel does not have plumbing to deal with extensions where the SrcVT or 4427274955Ssvnmir // DestVT are odd things, so test to make sure that they are both types we can 4428274955Ssvnmir // handle (i1/i8/i16/i32 for SrcVT and i8/i16/i32/i64 for DestVT), otherwise 4429274955Ssvnmir // bail out to SelectionDAG. 4430274955Ssvnmir if (((DestVT != MVT::i8) && (DestVT != MVT::i16) && 4431274955Ssvnmir (DestVT != MVT::i32) && (DestVT != MVT::i64)) || 4432274955Ssvnmir ((SrcVT != MVT::i1) && (SrcVT != MVT::i8) && 4433274955Ssvnmir (SrcVT != MVT::i16) && (SrcVT != MVT::i32))) 4434274955Ssvnmir return 0; 4435274955Ssvnmir 4436274955Ssvnmir unsigned Opc; 4437274955Ssvnmir unsigned Imm = 0; 4438274955Ssvnmir 4439274955Ssvnmir switch (SrcVT.SimpleTy) { 4440274955Ssvnmir default: 4441274955Ssvnmir return 0; 4442274955Ssvnmir case MVT::i1: 4443280031Sdim return emiti1Ext(SrcReg, DestVT, IsZExt); 4444274955Ssvnmir case MVT::i8: 4445274955Ssvnmir if (DestVT == MVT::i64) 4446280031Sdim Opc = IsZExt ? AArch64::UBFMXri : AArch64::SBFMXri; 4447274955Ssvnmir else 4448280031Sdim Opc = IsZExt ? AArch64::UBFMWri : AArch64::SBFMWri; 4449274955Ssvnmir Imm = 7; 4450274955Ssvnmir break; 4451274955Ssvnmir case MVT::i16: 4452274955Ssvnmir if (DestVT == MVT::i64) 4453280031Sdim Opc = IsZExt ? AArch64::UBFMXri : AArch64::SBFMXri; 4454274955Ssvnmir else 4455280031Sdim Opc = IsZExt ? AArch64::UBFMWri : AArch64::SBFMWri; 4456274955Ssvnmir Imm = 15; 4457274955Ssvnmir break; 4458274955Ssvnmir case MVT::i32: 4459274955Ssvnmir assert(DestVT == MVT::i64 && "IntExt i32 to i32?!?"); 4460280031Sdim Opc = IsZExt ? AArch64::UBFMXri : AArch64::SBFMXri; 4461274955Ssvnmir Imm = 31; 4462274955Ssvnmir break; 4463274955Ssvnmir } 4464274955Ssvnmir 4465274955Ssvnmir // Handle i8 and i16 as i32. 4466274955Ssvnmir if (DestVT == MVT::i8 || DestVT == MVT::i16) 4467274955Ssvnmir DestVT = MVT::i32; 4468274955Ssvnmir else if (DestVT == MVT::i64) { 4469360784Sdim Register Src64 = MRI.createVirtualRegister(&AArch64::GPR64RegClass); 4470274955Ssvnmir BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, 4471274955Ssvnmir TII.get(AArch64::SUBREG_TO_REG), Src64) 4472274955Ssvnmir .addImm(0) 4473274955Ssvnmir .addReg(SrcReg) 4474274955Ssvnmir .addImm(AArch64::sub_32); 4475274955Ssvnmir SrcReg = Src64; 4476274955Ssvnmir } 4477274955Ssvnmir 4478280031Sdim const TargetRegisterClass *RC = 4479280031Sdim (DestVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass; 4480280031Sdim return fastEmitInst_rii(Opc, RC, SrcReg, /*TODO:IsKill=*/false, 0, Imm); 4481280031Sdim} 4482274955Ssvnmir 4483280031Sdimstatic bool isZExtLoad(const MachineInstr *LI) { 4484280031Sdim switch (LI->getOpcode()) { 4485280031Sdim default: 4486280031Sdim return false; 4487280031Sdim case AArch64::LDURBBi: 4488280031Sdim case AArch64::LDURHHi: 4489280031Sdim case AArch64::LDURWi: 4490280031Sdim case AArch64::LDRBBui: 4491280031Sdim case AArch64::LDRHHui: 4492280031Sdim case AArch64::LDRWui: 4493280031Sdim case AArch64::LDRBBroX: 4494280031Sdim case AArch64::LDRHHroX: 4495280031Sdim case AArch64::LDRWroX: 4496280031Sdim case AArch64::LDRBBroW: 4497280031Sdim case AArch64::LDRHHroW: 4498280031Sdim case AArch64::LDRWroW: 4499280031Sdim return true; 4500280031Sdim } 4501274955Ssvnmir} 4502274955Ssvnmir 4503280031Sdimstatic bool isSExtLoad(const MachineInstr *LI) { 4504280031Sdim switch (LI->getOpcode()) { 4505280031Sdim default: 4506280031Sdim return false; 4507280031Sdim case AArch64::LDURSBWi: 4508280031Sdim case AArch64::LDURSHWi: 4509280031Sdim case AArch64::LDURSBXi: 4510280031Sdim case AArch64::LDURSHXi: 4511280031Sdim case AArch64::LDURSWi: 4512280031Sdim case AArch64::LDRSBWui: 4513280031Sdim case AArch64::LDRSHWui: 4514280031Sdim case AArch64::LDRSBXui: 4515280031Sdim case AArch64::LDRSHXui: 4516280031Sdim case AArch64::LDRSWui: 4517280031Sdim case AArch64::LDRSBWroX: 4518280031Sdim case AArch64::LDRSHWroX: 4519280031Sdim case AArch64::LDRSBXroX: 4520280031Sdim case AArch64::LDRSHXroX: 4521280031Sdim case AArch64::LDRSWroX: 4522280031Sdim case AArch64::LDRSBWroW: 4523280031Sdim case AArch64::LDRSHWroW: 4524280031Sdim case AArch64::LDRSBXroW: 4525280031Sdim case AArch64::LDRSHXroW: 4526280031Sdim case AArch64::LDRSWroW: 4527280031Sdim return true; 4528280031Sdim } 4529280031Sdim} 4530274955Ssvnmir 4531280031Sdimbool AArch64FastISel::optimizeIntExtLoad(const Instruction *I, MVT RetVT, 4532280031Sdim MVT SrcVT) { 4533280031Sdim const auto *LI = dyn_cast<LoadInst>(I->getOperand(0)); 4534280031Sdim if (!LI || !LI->hasOneUse()) 4535274955Ssvnmir return false; 4536274955Ssvnmir 4537280031Sdim // Check if the load instruction has already been selected. 4538280031Sdim unsigned Reg = lookUpRegForValue(LI); 4539280031Sdim if (!Reg) 4540274955Ssvnmir return false; 4541280031Sdim 4542280031Sdim MachineInstr *MI = MRI.getUniqueVRegDef(Reg); 4543280031Sdim if (!MI) 4544274955Ssvnmir return false; 4545274955Ssvnmir 4546280031Sdim // Check if the correct load instruction has been emitted - SelectionDAG might 4547280031Sdim // have emitted a zero-extending load, but we need a sign-extending load. 4548280031Sdim bool IsZExt = isa<ZExtInst>(I); 4549280031Sdim const auto *LoadMI = MI; 4550280031Sdim if (LoadMI->getOpcode() == TargetOpcode::COPY && 4551280031Sdim LoadMI->getOperand(1).getSubReg() == AArch64::sub_32) { 4552360784Sdim Register LoadReg = MI->getOperand(1).getReg(); 4553280031Sdim LoadMI = MRI.getUniqueVRegDef(LoadReg); 4554280031Sdim assert(LoadMI && "Expected valid instruction"); 4555280031Sdim } 4556280031Sdim if (!(IsZExt && isZExtLoad(LoadMI)) && !(!IsZExt && isSExtLoad(LoadMI))) 4557274955Ssvnmir return false; 4558280031Sdim 4559280031Sdim // Nothing to be done. 4560280031Sdim if (RetVT != MVT::i64 || SrcVT > MVT::i32) { 4561280031Sdim updateValueMap(I, Reg); 4562280031Sdim return true; 4563280031Sdim } 4564280031Sdim 4565280031Sdim if (IsZExt) { 4566280031Sdim unsigned Reg64 = createResultReg(&AArch64::GPR64RegClass); 4567280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, 4568280031Sdim TII.get(AArch64::SUBREG_TO_REG), Reg64) 4569280031Sdim .addImm(0) 4570280031Sdim .addReg(Reg, getKillRegState(true)) 4571280031Sdim .addImm(AArch64::sub_32); 4572280031Sdim Reg = Reg64; 4573280031Sdim } else { 4574280031Sdim assert((MI->getOpcode() == TargetOpcode::COPY && 4575280031Sdim MI->getOperand(1).getSubReg() == AArch64::sub_32) && 4576280031Sdim "Expected copy instruction"); 4577280031Sdim Reg = MI->getOperand(1).getReg(); 4578344779Sdim MachineBasicBlock::iterator I(MI); 4579344779Sdim removeDeadCode(I, std::next(I)); 4580280031Sdim } 4581280031Sdim updateValueMap(I, Reg); 4582274955Ssvnmir return true; 4583274955Ssvnmir} 4584274955Ssvnmir 4585280031Sdimbool AArch64FastISel::selectIntExt(const Instruction *I) { 4586280031Sdim assert((isa<ZExtInst>(I) || isa<SExtInst>(I)) && 4587280031Sdim "Unexpected integer extend instruction."); 4588280031Sdim MVT RetVT; 4589280031Sdim MVT SrcVT; 4590280031Sdim if (!isTypeSupported(I->getType(), RetVT)) 4591280031Sdim return false; 4592280031Sdim 4593280031Sdim if (!isTypeSupported(I->getOperand(0)->getType(), SrcVT)) 4594280031Sdim return false; 4595280031Sdim 4596280031Sdim // Try to optimize already sign-/zero-extended values from load instructions. 4597280031Sdim if (optimizeIntExtLoad(I, RetVT, SrcVT)) 4598280031Sdim return true; 4599280031Sdim 4600280031Sdim unsigned SrcReg = getRegForValue(I->getOperand(0)); 4601280031Sdim if (!SrcReg) 4602280031Sdim return false; 4603280031Sdim bool SrcIsKill = hasTrivialKill(I->getOperand(0)); 4604280031Sdim 4605280031Sdim // Try to optimize already sign-/zero-extended values from function arguments. 4606280031Sdim bool IsZExt = isa<ZExtInst>(I); 4607280031Sdim if (const auto *Arg = dyn_cast<Argument>(I->getOperand(0))) { 4608280031Sdim if ((IsZExt && Arg->hasZExtAttr()) || (!IsZExt && Arg->hasSExtAttr())) { 4609280031Sdim if (RetVT == MVT::i64 && SrcVT != MVT::i64) { 4610280031Sdim unsigned ResultReg = createResultReg(&AArch64::GPR64RegClass); 4611280031Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, 4612280031Sdim TII.get(AArch64::SUBREG_TO_REG), ResultReg) 4613280031Sdim .addImm(0) 4614280031Sdim .addReg(SrcReg, getKillRegState(SrcIsKill)) 4615280031Sdim .addImm(AArch64::sub_32); 4616280031Sdim SrcReg = ResultReg; 4617280031Sdim } 4618280031Sdim // Conservatively clear all kill flags from all uses, because we are 4619280031Sdim // replacing a sign-/zero-extend instruction at IR level with a nop at MI 4620280031Sdim // level. The result of the instruction at IR level might have been 4621280031Sdim // trivially dead, which is now not longer true. 4622280031Sdim unsigned UseReg = lookUpRegForValue(I); 4623280031Sdim if (UseReg) 4624280031Sdim MRI.clearKillFlags(UseReg); 4625280031Sdim 4626280031Sdim updateValueMap(I, SrcReg); 4627280031Sdim return true; 4628280031Sdim } 4629280031Sdim } 4630280031Sdim 4631280031Sdim unsigned ResultReg = emitIntExt(SrcVT, SrcReg, RetVT, IsZExt); 4632280031Sdim if (!ResultReg) 4633280031Sdim return false; 4634280031Sdim 4635280031Sdim updateValueMap(I, ResultReg); 4636280031Sdim return true; 4637280031Sdim} 4638280031Sdim 4639280031Sdimbool AArch64FastISel::selectRem(const Instruction *I, unsigned ISDOpcode) { 4640288943Sdim EVT DestEVT = TLI.getValueType(DL, I->getType(), true); 4641274955Ssvnmir if (!DestEVT.isSimple()) 4642274955Ssvnmir return false; 4643274955Ssvnmir 4644274955Ssvnmir MVT DestVT = DestEVT.getSimpleVT(); 4645274955Ssvnmir if (DestVT != MVT::i64 && DestVT != MVT::i32) 4646274955Ssvnmir return false; 4647274955Ssvnmir 4648274955Ssvnmir unsigned DivOpc; 4649280031Sdim bool Is64bit = (DestVT == MVT::i64); 4650274955Ssvnmir switch (ISDOpcode) { 4651274955Ssvnmir default: 4652274955Ssvnmir return false; 4653274955Ssvnmir case ISD::SREM: 4654280031Sdim DivOpc = Is64bit ? AArch64::SDIVXr : AArch64::SDIVWr; 4655274955Ssvnmir break; 4656274955Ssvnmir case ISD::UREM: 4657280031Sdim DivOpc = Is64bit ? AArch64::UDIVXr : AArch64::UDIVWr; 4658274955Ssvnmir break; 4659274955Ssvnmir } 4660280031Sdim unsigned MSubOpc = Is64bit ? AArch64::MSUBXrrr : AArch64::MSUBWrrr; 4661274955Ssvnmir unsigned Src0Reg = getRegForValue(I->getOperand(0)); 4662274955Ssvnmir if (!Src0Reg) 4663274955Ssvnmir return false; 4664280031Sdim bool Src0IsKill = hasTrivialKill(I->getOperand(0)); 4665274955Ssvnmir 4666274955Ssvnmir unsigned Src1Reg = getRegForValue(I->getOperand(1)); 4667274955Ssvnmir if (!Src1Reg) 4668274955Ssvnmir return false; 4669280031Sdim bool Src1IsKill = hasTrivialKill(I->getOperand(1)); 4670274955Ssvnmir 4671280031Sdim const TargetRegisterClass *RC = 4672280031Sdim (DestVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass; 4673280031Sdim unsigned QuotReg = fastEmitInst_rr(DivOpc, RC, Src0Reg, /*IsKill=*/false, 4674280031Sdim Src1Reg, /*IsKill=*/false); 4675280031Sdim assert(QuotReg && "Unexpected DIV instruction emission failure."); 4676274955Ssvnmir // The remainder is computed as numerator - (quotient * denominator) using the 4677274955Ssvnmir // MSUB instruction. 4678280031Sdim unsigned ResultReg = fastEmitInst_rrr(MSubOpc, RC, QuotReg, /*IsKill=*/true, 4679280031Sdim Src1Reg, Src1IsKill, Src0Reg, 4680280031Sdim Src0IsKill); 4681280031Sdim updateValueMap(I, ResultReg); 4682274955Ssvnmir return true; 4683274955Ssvnmir} 4684274955Ssvnmir 4685280031Sdimbool AArch64FastISel::selectMul(const Instruction *I) { 4686280031Sdim MVT VT; 4687280031Sdim if (!isTypeSupported(I->getType(), VT, /*IsVectorAllowed=*/true)) 4688274955Ssvnmir return false; 4689274955Ssvnmir 4690280031Sdim if (VT.isVector()) 4691280031Sdim return selectBinaryOp(I, ISD::MUL); 4692280031Sdim 4693280031Sdim const Value *Src0 = I->getOperand(0); 4694280031Sdim const Value *Src1 = I->getOperand(1); 4695280031Sdim if (const auto *C = dyn_cast<ConstantInt>(Src0)) 4696280031Sdim if (C->getValue().isPowerOf2()) 4697280031Sdim std::swap(Src0, Src1); 4698280031Sdim 4699280031Sdim // Try to simplify to a shift instruction. 4700280031Sdim if (const auto *C = dyn_cast<ConstantInt>(Src1)) 4701280031Sdim if (C->getValue().isPowerOf2()) { 4702280031Sdim uint64_t ShiftVal = C->getValue().logBase2(); 4703280031Sdim MVT SrcVT = VT; 4704280031Sdim bool IsZExt = true; 4705280031Sdim if (const auto *ZExt = dyn_cast<ZExtInst>(Src0)) { 4706280031Sdim if (!isIntExtFree(ZExt)) { 4707280031Sdim MVT VT; 4708280031Sdim if (isValueAvailable(ZExt) && isTypeSupported(ZExt->getSrcTy(), VT)) { 4709280031Sdim SrcVT = VT; 4710280031Sdim IsZExt = true; 4711280031Sdim Src0 = ZExt->getOperand(0); 4712280031Sdim } 4713280031Sdim } 4714280031Sdim } else if (const auto *SExt = dyn_cast<SExtInst>(Src0)) { 4715280031Sdim if (!isIntExtFree(SExt)) { 4716280031Sdim MVT VT; 4717280031Sdim if (isValueAvailable(SExt) && isTypeSupported(SExt->getSrcTy(), VT)) { 4718280031Sdim SrcVT = VT; 4719280031Sdim IsZExt = false; 4720280031Sdim Src0 = SExt->getOperand(0); 4721280031Sdim } 4722280031Sdim } 4723280031Sdim } 4724280031Sdim 4725280031Sdim unsigned Src0Reg = getRegForValue(Src0); 4726280031Sdim if (!Src0Reg) 4727280031Sdim return false; 4728280031Sdim bool Src0IsKill = hasTrivialKill(Src0); 4729280031Sdim 4730280031Sdim unsigned ResultReg = 4731280031Sdim emitLSL_ri(VT, SrcVT, Src0Reg, Src0IsKill, ShiftVal, IsZExt); 4732280031Sdim 4733280031Sdim if (ResultReg) { 4734280031Sdim updateValueMap(I, ResultReg); 4735280031Sdim return true; 4736280031Sdim } 4737280031Sdim } 4738280031Sdim 4739280031Sdim unsigned Src0Reg = getRegForValue(I->getOperand(0)); 4740280031Sdim if (!Src0Reg) 4741274955Ssvnmir return false; 4742280031Sdim bool Src0IsKill = hasTrivialKill(I->getOperand(0)); 4743274955Ssvnmir 4744280031Sdim unsigned Src1Reg = getRegForValue(I->getOperand(1)); 4745280031Sdim if (!Src1Reg) 4746280031Sdim return false; 4747280031Sdim bool Src1IsKill = hasTrivialKill(I->getOperand(1)); 4748280031Sdim 4749280031Sdim unsigned ResultReg = emitMul_rr(VT, Src0Reg, Src0IsKill, Src1Reg, Src1IsKill); 4750280031Sdim 4751280031Sdim if (!ResultReg) 4752280031Sdim return false; 4753280031Sdim 4754280031Sdim updateValueMap(I, ResultReg); 4755280031Sdim return true; 4756280031Sdim} 4757280031Sdim 4758280031Sdimbool AArch64FastISel::selectShift(const Instruction *I) { 4759280031Sdim MVT RetVT; 4760280031Sdim if (!isTypeSupported(I->getType(), RetVT, /*IsVectorAllowed=*/true)) 4761280031Sdim return false; 4762280031Sdim 4763280031Sdim if (RetVT.isVector()) 4764280031Sdim return selectOperator(I, I->getOpcode()); 4765280031Sdim 4766280031Sdim if (const auto *C = dyn_cast<ConstantInt>(I->getOperand(1))) { 4767280031Sdim unsigned ResultReg = 0; 4768280031Sdim uint64_t ShiftVal = C->getZExtValue(); 4769280031Sdim MVT SrcVT = RetVT; 4770288943Sdim bool IsZExt = I->getOpcode() != Instruction::AShr; 4771280031Sdim const Value *Op0 = I->getOperand(0); 4772280031Sdim if (const auto *ZExt = dyn_cast<ZExtInst>(Op0)) { 4773280031Sdim if (!isIntExtFree(ZExt)) { 4774280031Sdim MVT TmpVT; 4775280031Sdim if (isValueAvailable(ZExt) && isTypeSupported(ZExt->getSrcTy(), TmpVT)) { 4776280031Sdim SrcVT = TmpVT; 4777280031Sdim IsZExt = true; 4778280031Sdim Op0 = ZExt->getOperand(0); 4779280031Sdim } 4780280031Sdim } 4781280031Sdim } else if (const auto *SExt = dyn_cast<SExtInst>(Op0)) { 4782280031Sdim if (!isIntExtFree(SExt)) { 4783280031Sdim MVT TmpVT; 4784280031Sdim if (isValueAvailable(SExt) && isTypeSupported(SExt->getSrcTy(), TmpVT)) { 4785280031Sdim SrcVT = TmpVT; 4786280031Sdim IsZExt = false; 4787280031Sdim Op0 = SExt->getOperand(0); 4788280031Sdim } 4789280031Sdim } 4790280031Sdim } 4791280031Sdim 4792280031Sdim unsigned Op0Reg = getRegForValue(Op0); 4793280031Sdim if (!Op0Reg) 4794280031Sdim return false; 4795280031Sdim bool Op0IsKill = hasTrivialKill(Op0); 4796280031Sdim 4797280031Sdim switch (I->getOpcode()) { 4798280031Sdim default: llvm_unreachable("Unexpected instruction."); 4799280031Sdim case Instruction::Shl: 4800280031Sdim ResultReg = emitLSL_ri(RetVT, SrcVT, Op0Reg, Op0IsKill, ShiftVal, IsZExt); 4801280031Sdim break; 4802280031Sdim case Instruction::AShr: 4803280031Sdim ResultReg = emitASR_ri(RetVT, SrcVT, Op0Reg, Op0IsKill, ShiftVal, IsZExt); 4804280031Sdim break; 4805280031Sdim case Instruction::LShr: 4806280031Sdim ResultReg = emitLSR_ri(RetVT, SrcVT, Op0Reg, Op0IsKill, ShiftVal, IsZExt); 4807280031Sdim break; 4808280031Sdim } 4809280031Sdim if (!ResultReg) 4810280031Sdim return false; 4811280031Sdim 4812280031Sdim updateValueMap(I, ResultReg); 4813280031Sdim return true; 4814280031Sdim } 4815280031Sdim 4816280031Sdim unsigned Op0Reg = getRegForValue(I->getOperand(0)); 4817280031Sdim if (!Op0Reg) 4818280031Sdim return false; 4819280031Sdim bool Op0IsKill = hasTrivialKill(I->getOperand(0)); 4820280031Sdim 4821280031Sdim unsigned Op1Reg = getRegForValue(I->getOperand(1)); 4822280031Sdim if (!Op1Reg) 4823280031Sdim return false; 4824280031Sdim bool Op1IsKill = hasTrivialKill(I->getOperand(1)); 4825280031Sdim 4826280031Sdim unsigned ResultReg = 0; 4827280031Sdim switch (I->getOpcode()) { 4828280031Sdim default: llvm_unreachable("Unexpected instruction."); 4829280031Sdim case Instruction::Shl: 4830280031Sdim ResultReg = emitLSL_rr(RetVT, Op0Reg, Op0IsKill, Op1Reg, Op1IsKill); 4831280031Sdim break; 4832280031Sdim case Instruction::AShr: 4833280031Sdim ResultReg = emitASR_rr(RetVT, Op0Reg, Op0IsKill, Op1Reg, Op1IsKill); 4834280031Sdim break; 4835280031Sdim case Instruction::LShr: 4836280031Sdim ResultReg = emitLSR_rr(RetVT, Op0Reg, Op0IsKill, Op1Reg, Op1IsKill); 4837280031Sdim break; 4838280031Sdim } 4839280031Sdim 4840280031Sdim if (!ResultReg) 4841280031Sdim return false; 4842280031Sdim 4843280031Sdim updateValueMap(I, ResultReg); 4844280031Sdim return true; 4845280031Sdim} 4846280031Sdim 4847280031Sdimbool AArch64FastISel::selectBitCast(const Instruction *I) { 4848280031Sdim MVT RetVT, SrcVT; 4849280031Sdim 4850280031Sdim if (!isTypeLegal(I->getOperand(0)->getType(), SrcVT)) 4851280031Sdim return false; 4852280031Sdim if (!isTypeLegal(I->getType(), RetVT)) 4853280031Sdim return false; 4854280031Sdim 4855274955Ssvnmir unsigned Opc; 4856280031Sdim if (RetVT == MVT::f32 && SrcVT == MVT::i32) 4857280031Sdim Opc = AArch64::FMOVWSr; 4858280031Sdim else if (RetVT == MVT::f64 && SrcVT == MVT::i64) 4859280031Sdim Opc = AArch64::FMOVXDr; 4860280031Sdim else if (RetVT == MVT::i32 && SrcVT == MVT::f32) 4861280031Sdim Opc = AArch64::FMOVSWr; 4862280031Sdim else if (RetVT == MVT::i64 && SrcVT == MVT::f64) 4863280031Sdim Opc = AArch64::FMOVDXr; 4864280031Sdim else 4865280031Sdim return false; 4866280031Sdim 4867280031Sdim const TargetRegisterClass *RC = nullptr; 4868280031Sdim switch (RetVT.SimpleTy) { 4869280031Sdim default: llvm_unreachable("Unexpected value type."); 4870280031Sdim case MVT::i32: RC = &AArch64::GPR32RegClass; break; 4871280031Sdim case MVT::i64: RC = &AArch64::GPR64RegClass; break; 4872280031Sdim case MVT::f32: RC = &AArch64::FPR32RegClass; break; 4873280031Sdim case MVT::f64: RC = &AArch64::FPR64RegClass; break; 4874280031Sdim } 4875280031Sdim unsigned Op0Reg = getRegForValue(I->getOperand(0)); 4876280031Sdim if (!Op0Reg) 4877280031Sdim return false; 4878280031Sdim bool Op0IsKill = hasTrivialKill(I->getOperand(0)); 4879280031Sdim unsigned ResultReg = fastEmitInst_r(Opc, RC, Op0Reg, Op0IsKill); 4880280031Sdim 4881280031Sdim if (!ResultReg) 4882280031Sdim return false; 4883280031Sdim 4884280031Sdim updateValueMap(I, ResultReg); 4885280031Sdim return true; 4886280031Sdim} 4887280031Sdim 4888280031Sdimbool AArch64FastISel::selectFRem(const Instruction *I) { 4889280031Sdim MVT RetVT; 4890280031Sdim if (!isTypeLegal(I->getType(), RetVT)) 4891280031Sdim return false; 4892280031Sdim 4893280031Sdim RTLIB::Libcall LC; 4894280031Sdim switch (RetVT.SimpleTy) { 4895274955Ssvnmir default: 4896274955Ssvnmir return false; 4897280031Sdim case MVT::f32: 4898280031Sdim LC = RTLIB::REM_F32; 4899274955Ssvnmir break; 4900280031Sdim case MVT::f64: 4901280031Sdim LC = RTLIB::REM_F64; 4902274955Ssvnmir break; 4903274955Ssvnmir } 4904274955Ssvnmir 4905280031Sdim ArgListTy Args; 4906280031Sdim Args.reserve(I->getNumOperands()); 4907280031Sdim 4908280031Sdim // Populate the argument list. 4909280031Sdim for (auto &Arg : I->operands()) { 4910280031Sdim ArgListEntry Entry; 4911280031Sdim Entry.Val = Arg; 4912280031Sdim Entry.Ty = Arg->getType(); 4913280031Sdim Args.push_back(Entry); 4914280031Sdim } 4915280031Sdim 4916280031Sdim CallLoweringInfo CLI; 4917288943Sdim MCContext &Ctx = MF->getContext(); 4918288943Sdim CLI.setCallee(DL, Ctx, TLI.getLibcallCallingConv(LC), I->getType(), 4919280031Sdim TLI.getLibcallName(LC), std::move(Args)); 4920280031Sdim if (!lowerCallTo(CLI)) 4921280031Sdim return false; 4922280031Sdim updateValueMap(I, CLI.ResultReg); 4923280031Sdim return true; 4924280031Sdim} 4925280031Sdim 4926280031Sdimbool AArch64FastISel::selectSDiv(const Instruction *I) { 4927280031Sdim MVT VT; 4928280031Sdim if (!isTypeLegal(I->getType(), VT)) 4929280031Sdim return false; 4930280031Sdim 4931280031Sdim if (!isa<ConstantInt>(I->getOperand(1))) 4932280031Sdim return selectBinaryOp(I, ISD::SDIV); 4933280031Sdim 4934280031Sdim const APInt &C = cast<ConstantInt>(I->getOperand(1))->getValue(); 4935280031Sdim if ((VT != MVT::i32 && VT != MVT::i64) || !C || 4936280031Sdim !(C.isPowerOf2() || (-C).isPowerOf2())) 4937280031Sdim return selectBinaryOp(I, ISD::SDIV); 4938280031Sdim 4939280031Sdim unsigned Lg2 = C.countTrailingZeros(); 4940274955Ssvnmir unsigned Src0Reg = getRegForValue(I->getOperand(0)); 4941274955Ssvnmir if (!Src0Reg) 4942274955Ssvnmir return false; 4943280031Sdim bool Src0IsKill = hasTrivialKill(I->getOperand(0)); 4944274955Ssvnmir 4945280031Sdim if (cast<BinaryOperator>(I)->isExact()) { 4946280031Sdim unsigned ResultReg = emitASR_ri(VT, VT, Src0Reg, Src0IsKill, Lg2); 4947280031Sdim if (!ResultReg) 4948280031Sdim return false; 4949280031Sdim updateValueMap(I, ResultReg); 4950280031Sdim return true; 4951280031Sdim } 4952280031Sdim 4953280031Sdim int64_t Pow2MinusOne = (1ULL << Lg2) - 1; 4954280031Sdim unsigned AddReg = emitAdd_ri_(VT, Src0Reg, /*IsKill=*/false, Pow2MinusOne); 4955280031Sdim if (!AddReg) 4956274955Ssvnmir return false; 4957274955Ssvnmir 4958280031Sdim // (Src0 < 0) ? Pow2 - 1 : 0; 4959280031Sdim if (!emitICmp_ri(VT, Src0Reg, /*IsKill=*/false, 0)) 4960280031Sdim return false; 4961280031Sdim 4962280031Sdim unsigned SelectOpc; 4963280031Sdim const TargetRegisterClass *RC; 4964280031Sdim if (VT == MVT::i64) { 4965280031Sdim SelectOpc = AArch64::CSELXr; 4966280031Sdim RC = &AArch64::GPR64RegClass; 4967280031Sdim } else { 4968280031Sdim SelectOpc = AArch64::CSELWr; 4969280031Sdim RC = &AArch64::GPR32RegClass; 4970280031Sdim } 4971280031Sdim unsigned SelectReg = 4972280031Sdim fastEmitInst_rri(SelectOpc, RC, AddReg, /*IsKill=*/true, Src0Reg, 4973280031Sdim Src0IsKill, AArch64CC::LT); 4974280031Sdim if (!SelectReg) 4975280031Sdim return false; 4976280031Sdim 4977280031Sdim // Divide by Pow2 --> ashr. If we're dividing by a negative value we must also 4978280031Sdim // negate the result. 4979280031Sdim unsigned ZeroReg = (VT == MVT::i64) ? AArch64::XZR : AArch64::WZR; 4980280031Sdim unsigned ResultReg; 4981280031Sdim if (C.isNegative()) 4982280031Sdim ResultReg = emitAddSub_rs(/*UseAdd=*/false, VT, ZeroReg, /*IsKill=*/true, 4983280031Sdim SelectReg, /*IsKill=*/true, AArch64_AM::ASR, Lg2); 4984280031Sdim else 4985280031Sdim ResultReg = emitASR_ri(VT, VT, SelectReg, /*IsKill=*/true, Lg2); 4986280031Sdim 4987280031Sdim if (!ResultReg) 4988280031Sdim return false; 4989280031Sdim 4990280031Sdim updateValueMap(I, ResultReg); 4991274955Ssvnmir return true; 4992274955Ssvnmir} 4993274955Ssvnmir 4994280031Sdim/// This is mostly a copy of the existing FastISel getRegForGEPIndex code. We 4995280031Sdim/// have to duplicate it for AArch64, because otherwise we would fail during the 4996280031Sdim/// sign-extend emission. 4997280031Sdimstd::pair<unsigned, bool> AArch64FastISel::getRegForGEPIndex(const Value *Idx) { 4998280031Sdim unsigned IdxN = getRegForValue(Idx); 4999280031Sdim if (IdxN == 0) 5000280031Sdim // Unhandled operand. Halt "fast" selection and bail. 5001280031Sdim return std::pair<unsigned, bool>(0, false); 5002280031Sdim 5003280031Sdim bool IdxNIsKill = hasTrivialKill(Idx); 5004280031Sdim 5005280031Sdim // If the index is smaller or larger than intptr_t, truncate or extend it. 5006288943Sdim MVT PtrVT = TLI.getPointerTy(DL); 5007280031Sdim EVT IdxVT = EVT::getEVT(Idx->getType(), /*HandleUnknown=*/false); 5008280031Sdim if (IdxVT.bitsLT(PtrVT)) { 5009353358Sdim IdxN = emitIntExt(IdxVT.getSimpleVT(), IdxN, PtrVT, /*isZExt=*/false); 5010280031Sdim IdxNIsKill = true; 5011280031Sdim } else if (IdxVT.bitsGT(PtrVT)) 5012280031Sdim llvm_unreachable("AArch64 FastISel doesn't support types larger than i64"); 5013280031Sdim return std::pair<unsigned, bool>(IdxN, IdxNIsKill); 5014280031Sdim} 5015280031Sdim 5016280031Sdim/// This is mostly a copy of the existing FastISel GEP code, but we have to 5017280031Sdim/// duplicate it for AArch64, because otherwise we would bail out even for 5018280031Sdim/// simple cases. This is because the standard fastEmit functions don't cover 5019280031Sdim/// MUL at all and ADD is lowered very inefficientily. 5020280031Sdimbool AArch64FastISel::selectGetElementPtr(const Instruction *I) { 5021360784Sdim if (Subtarget->isTargetILP32()) 5022360784Sdim return false; 5023360784Sdim 5024280031Sdim unsigned N = getRegForValue(I->getOperand(0)); 5025280031Sdim if (!N) 5026280031Sdim return false; 5027280031Sdim bool NIsKill = hasTrivialKill(I->getOperand(0)); 5028280031Sdim 5029280031Sdim // Keep a running tab of the total offset to coalesce multiple N = N + Offset 5030280031Sdim // into a single N = N + TotalOffset. 5031280031Sdim uint64_t TotalOffs = 0; 5032288943Sdim MVT VT = TLI.getPointerTy(DL); 5033309124Sdim for (gep_type_iterator GTI = gep_type_begin(I), E = gep_type_end(I); 5034309124Sdim GTI != E; ++GTI) { 5035309124Sdim const Value *Idx = GTI.getOperand(); 5036314564Sdim if (auto *StTy = GTI.getStructTypeOrNull()) { 5037280031Sdim unsigned Field = cast<ConstantInt>(Idx)->getZExtValue(); 5038280031Sdim // N = N + Offset 5039280031Sdim if (Field) 5040280031Sdim TotalOffs += DL.getStructLayout(StTy)->getElementOffset(Field); 5041280031Sdim } else { 5042309124Sdim Type *Ty = GTI.getIndexedType(); 5043309124Sdim 5044280031Sdim // If this is a constant subscript, handle it quickly. 5045280031Sdim if (const auto *CI = dyn_cast<ConstantInt>(Idx)) { 5046280031Sdim if (CI->isZero()) 5047280031Sdim continue; 5048280031Sdim // N = N + Offset 5049280031Sdim TotalOffs += 5050280031Sdim DL.getTypeAllocSize(Ty) * cast<ConstantInt>(CI)->getSExtValue(); 5051280031Sdim continue; 5052280031Sdim } 5053280031Sdim if (TotalOffs) { 5054280031Sdim N = emitAdd_ri_(VT, N, NIsKill, TotalOffs); 5055280031Sdim if (!N) 5056280031Sdim return false; 5057280031Sdim NIsKill = true; 5058280031Sdim TotalOffs = 0; 5059280031Sdim } 5060280031Sdim 5061280031Sdim // N = N + Idx * ElementSize; 5062280031Sdim uint64_t ElementSize = DL.getTypeAllocSize(Ty); 5063280031Sdim std::pair<unsigned, bool> Pair = getRegForGEPIndex(Idx); 5064280031Sdim unsigned IdxN = Pair.first; 5065280031Sdim bool IdxNIsKill = Pair.second; 5066280031Sdim if (!IdxN) 5067280031Sdim return false; 5068280031Sdim 5069280031Sdim if (ElementSize != 1) { 5070280031Sdim unsigned C = fastEmit_i(VT, VT, ISD::Constant, ElementSize); 5071280031Sdim if (!C) 5072280031Sdim return false; 5073280031Sdim IdxN = emitMul_rr(VT, IdxN, IdxNIsKill, C, true); 5074280031Sdim if (!IdxN) 5075280031Sdim return false; 5076280031Sdim IdxNIsKill = true; 5077280031Sdim } 5078280031Sdim N = fastEmit_rr(VT, VT, ISD::ADD, N, NIsKill, IdxN, IdxNIsKill); 5079280031Sdim if (!N) 5080280031Sdim return false; 5081280031Sdim } 5082280031Sdim } 5083280031Sdim if (TotalOffs) { 5084280031Sdim N = emitAdd_ri_(VT, N, NIsKill, TotalOffs); 5085280031Sdim if (!N) 5086280031Sdim return false; 5087280031Sdim } 5088280031Sdim updateValueMap(I, N); 5089280031Sdim return true; 5090280031Sdim} 5091280031Sdim 5092314564Sdimbool AArch64FastISel::selectAtomicCmpXchg(const AtomicCmpXchgInst *I) { 5093314564Sdim assert(TM.getOptLevel() == CodeGenOpt::None && 5094314564Sdim "cmpxchg survived AtomicExpand at optlevel > -O0"); 5095314564Sdim 5096314564Sdim auto *RetPairTy = cast<StructType>(I->getType()); 5097314564Sdim Type *RetTy = RetPairTy->getTypeAtIndex(0U); 5098314564Sdim assert(RetPairTy->getTypeAtIndex(1U)->isIntegerTy(1) && 5099314564Sdim "cmpxchg has a non-i1 status result"); 5100314564Sdim 5101314564Sdim MVT VT; 5102314564Sdim if (!isTypeLegal(RetTy, VT)) 5103314564Sdim return false; 5104314564Sdim 5105314564Sdim const TargetRegisterClass *ResRC; 5106314564Sdim unsigned Opc, CmpOpc; 5107314564Sdim // This only supports i32/i64, because i8/i16 aren't legal, and the generic 5108314564Sdim // extractvalue selection doesn't support that. 5109314564Sdim if (VT == MVT::i32) { 5110314564Sdim Opc = AArch64::CMP_SWAP_32; 5111314564Sdim CmpOpc = AArch64::SUBSWrs; 5112314564Sdim ResRC = &AArch64::GPR32RegClass; 5113314564Sdim } else if (VT == MVT::i64) { 5114314564Sdim Opc = AArch64::CMP_SWAP_64; 5115314564Sdim CmpOpc = AArch64::SUBSXrs; 5116314564Sdim ResRC = &AArch64::GPR64RegClass; 5117314564Sdim } else { 5118314564Sdim return false; 5119314564Sdim } 5120314564Sdim 5121314564Sdim const MCInstrDesc &II = TII.get(Opc); 5122314564Sdim 5123314564Sdim const unsigned AddrReg = constrainOperandRegClass( 5124314564Sdim II, getRegForValue(I->getPointerOperand()), II.getNumDefs()); 5125314564Sdim const unsigned DesiredReg = constrainOperandRegClass( 5126314564Sdim II, getRegForValue(I->getCompareOperand()), II.getNumDefs() + 1); 5127314564Sdim const unsigned NewReg = constrainOperandRegClass( 5128314564Sdim II, getRegForValue(I->getNewValOperand()), II.getNumDefs() + 2); 5129314564Sdim 5130314564Sdim const unsigned ResultReg1 = createResultReg(ResRC); 5131314564Sdim const unsigned ResultReg2 = createResultReg(&AArch64::GPR32RegClass); 5132314564Sdim const unsigned ScratchReg = createResultReg(&AArch64::GPR32RegClass); 5133314564Sdim 5134314564Sdim // FIXME: MachineMemOperand doesn't support cmpxchg yet. 5135314564Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II) 5136314564Sdim .addDef(ResultReg1) 5137314564Sdim .addDef(ScratchReg) 5138314564Sdim .addUse(AddrReg) 5139314564Sdim .addUse(DesiredReg) 5140314564Sdim .addUse(NewReg); 5141314564Sdim 5142314564Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(CmpOpc)) 5143314564Sdim .addDef(VT == MVT::i32 ? AArch64::WZR : AArch64::XZR) 5144314564Sdim .addUse(ResultReg1) 5145314564Sdim .addUse(DesiredReg) 5146314564Sdim .addImm(0); 5147314564Sdim 5148314564Sdim BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::CSINCWr)) 5149314564Sdim .addDef(ResultReg2) 5150314564Sdim .addUse(AArch64::WZR) 5151314564Sdim .addUse(AArch64::WZR) 5152314564Sdim .addImm(AArch64CC::NE); 5153314564Sdim 5154314564Sdim assert((ResultReg1 + 1) == ResultReg2 && "Nonconsecutive result registers."); 5155314564Sdim updateValueMap(I, ResultReg1, 2); 5156314564Sdim return true; 5157314564Sdim} 5158314564Sdim 5159280031Sdimbool AArch64FastISel::fastSelectInstruction(const Instruction *I) { 5160274955Ssvnmir switch (I->getOpcode()) { 5161274955Ssvnmir default: 5162274955Ssvnmir break; 5163280031Sdim case Instruction::Add: 5164280031Sdim case Instruction::Sub: 5165280031Sdim return selectAddSub(I); 5166280031Sdim case Instruction::Mul: 5167280031Sdim return selectMul(I); 5168280031Sdim case Instruction::SDiv: 5169280031Sdim return selectSDiv(I); 5170280031Sdim case Instruction::SRem: 5171280031Sdim if (!selectBinaryOp(I, ISD::SREM)) 5172280031Sdim return selectRem(I, ISD::SREM); 5173280031Sdim return true; 5174280031Sdim case Instruction::URem: 5175280031Sdim if (!selectBinaryOp(I, ISD::UREM)) 5176280031Sdim return selectRem(I, ISD::UREM); 5177280031Sdim return true; 5178280031Sdim case Instruction::Shl: 5179280031Sdim case Instruction::LShr: 5180280031Sdim case Instruction::AShr: 5181280031Sdim return selectShift(I); 5182280031Sdim case Instruction::And: 5183280031Sdim case Instruction::Or: 5184280031Sdim case Instruction::Xor: 5185280031Sdim return selectLogicalOp(I); 5186274955Ssvnmir case Instruction::Br: 5187280031Sdim return selectBranch(I); 5188274955Ssvnmir case Instruction::IndirectBr: 5189280031Sdim return selectIndirectBr(I); 5190280031Sdim case Instruction::BitCast: 5191280031Sdim if (!FastISel::selectBitCast(I)) 5192280031Sdim return selectBitCast(I); 5193280031Sdim return true; 5194280031Sdim case Instruction::FPToSI: 5195280031Sdim if (!selectCast(I, ISD::FP_TO_SINT)) 5196280031Sdim return selectFPToInt(I, /*Signed=*/true); 5197280031Sdim return true; 5198280031Sdim case Instruction::FPToUI: 5199280031Sdim return selectFPToInt(I, /*Signed=*/false); 5200280031Sdim case Instruction::ZExt: 5201280031Sdim case Instruction::SExt: 5202280031Sdim return selectIntExt(I); 5203280031Sdim case Instruction::Trunc: 5204280031Sdim if (!selectCast(I, ISD::TRUNCATE)) 5205280031Sdim return selectTrunc(I); 5206280031Sdim return true; 5207274955Ssvnmir case Instruction::FPExt: 5208280031Sdim return selectFPExt(I); 5209274955Ssvnmir case Instruction::FPTrunc: 5210280031Sdim return selectFPTrunc(I); 5211274955Ssvnmir case Instruction::SIToFP: 5212280031Sdim if (!selectCast(I, ISD::SINT_TO_FP)) 5213280031Sdim return selectIntToFP(I, /*Signed=*/true); 5214280031Sdim return true; 5215274955Ssvnmir case Instruction::UIToFP: 5216280031Sdim return selectIntToFP(I, /*Signed=*/false); 5217280031Sdim case Instruction::Load: 5218280031Sdim return selectLoad(I); 5219280031Sdim case Instruction::Store: 5220280031Sdim return selectStore(I); 5221280031Sdim case Instruction::FCmp: 5222280031Sdim case Instruction::ICmp: 5223280031Sdim return selectCmp(I); 5224280031Sdim case Instruction::Select: 5225280031Sdim return selectSelect(I); 5226274955Ssvnmir case Instruction::Ret: 5227280031Sdim return selectRet(I); 5228280031Sdim case Instruction::FRem: 5229280031Sdim return selectFRem(I); 5230280031Sdim case Instruction::GetElementPtr: 5231280031Sdim return selectGetElementPtr(I); 5232314564Sdim case Instruction::AtomicCmpXchg: 5233314564Sdim return selectAtomicCmpXchg(cast<AtomicCmpXchgInst>(I)); 5234274955Ssvnmir } 5235280031Sdim 5236327952Sdim // fall-back to target-independent instruction selection. 5237327952Sdim return selectOperator(I, I->getOpcode()); 5238274955Ssvnmir} 5239274955Ssvnmir 5240274955Ssvnmirnamespace llvm { 5241321369Sdim 5242321369SdimFastISel *AArch64::createFastISel(FunctionLoweringInfo &FuncInfo, 5243280031Sdim const TargetLibraryInfo *LibInfo) { 5244280031Sdim return new AArch64FastISel(FuncInfo, LibInfo); 5245274955Ssvnmir} 5246321369Sdim 5247321369Sdim} // end namespace llvm 5248