1193323Sed//===-- DAGCombiner.cpp - Implement a DAG node combiner -------------------===// 2193323Sed// 3193323Sed// The LLVM Compiler Infrastructure 4193323Sed// 5193323Sed// This file is distributed under the University of Illinois Open Source 6193323Sed// License. See LICENSE.TXT for details. 7193323Sed// 8193323Sed//===----------------------------------------------------------------------===// 9193323Sed// 10193323Sed// This pass combines dag nodes to form fewer, simpler DAG nodes. It can be run 11193323Sed// both before and after the DAG is legalized. 12193323Sed// 13193323Sed// This pass is not a substitute for the LLVM IR instcombine pass. This pass is 14193323Sed// primarily intended to handle simplification opportunities that are implicit 15193323Sed// in the LLVM IR and exposed by the various codegen lowering phases. 16193323Sed// 17193323Sed//===----------------------------------------------------------------------===// 18193323Sed 19193323Sed#define DEBUG_TYPE "dagcombine" 20193323Sed#include "llvm/CodeGen/SelectionDAG.h" 21193323Sed#include "llvm/ADT/SmallPtrSet.h" 22193323Sed#include "llvm/ADT/Statistic.h" 23249423Sdim#include "llvm/Analysis/AliasAnalysis.h" 24249423Sdim#include "llvm/CodeGen/MachineFrameInfo.h" 25249423Sdim#include "llvm/CodeGen/MachineFunction.h" 26249423Sdim#include "llvm/IR/DataLayout.h" 27249423Sdim#include "llvm/IR/DerivedTypes.h" 28249423Sdim#include "llvm/IR/Function.h" 29249423Sdim#include "llvm/IR/LLVMContext.h" 30193323Sed#include "llvm/Support/CommandLine.h" 31193323Sed#include "llvm/Support/Debug.h" 32198090Srdivacky#include "llvm/Support/ErrorHandling.h" 33193323Sed#include "llvm/Support/MathExtras.h" 34198090Srdivacky#include "llvm/Support/raw_ostream.h" 35249423Sdim#include "llvm/Target/TargetLowering.h" 36249423Sdim#include "llvm/Target/TargetMachine.h" 37249423Sdim#include "llvm/Target/TargetOptions.h" 38263508Sdim#include "llvm/Target/TargetRegisterInfo.h" 39263508Sdim#include "llvm/Target/TargetSubtargetInfo.h" 40193323Sed#include <algorithm> 41193323Sedusing namespace llvm; 42193323Sed 43193323SedSTATISTIC(NodesCombined , "Number of dag nodes combined"); 44193323SedSTATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created"); 45193323SedSTATISTIC(PostIndexedNodes, "Number of post-indexed nodes created"); 46193323SedSTATISTIC(OpsNarrowed , "Number of load/op/store narrowed"); 47218893SdimSTATISTIC(LdStFP2Int , "Number of fp load/store pairs transformed to int"); 48263508SdimSTATISTIC(SlicedLoads, "Number of load sliced"); 49193323Sed 50193323Sednamespace { 51193323Sed static cl::opt<bool> 52193323Sed CombinerAA("combiner-alias-analysis", cl::Hidden, 53193323Sed cl::desc("Turn on alias analysis during testing")); 54193323Sed 55193323Sed static cl::opt<bool> 56193323Sed CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden, 57193323Sed cl::desc("Include global information in alias analysis")); 58193323Sed 59263508Sdim /// Hidden option to stress test load slicing, i.e., when this option 60263508Sdim /// is enabled, load slicing bypasses most of its profitability guards. 61263508Sdim static cl::opt<bool> 62263508Sdim StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden, 63263508Sdim cl::desc("Bypass the profitability model of load " 64263508Sdim "slicing"), 65263508Sdim cl::init(false)); 66263508Sdim 67193323Sed//------------------------------ DAGCombiner ---------------------------------// 68193323Sed 69198892Srdivacky class DAGCombiner { 70193323Sed SelectionDAG &DAG; 71193323Sed const TargetLowering &TLI; 72193323Sed CombineLevel Level; 73193323Sed CodeGenOpt::Level OptLevel; 74193323Sed bool LegalOperations; 75193323Sed bool LegalTypes; 76263508Sdim bool ForCodeSize; 77193323Sed 78193323Sed // Worklist of all of the nodes that need to be simplified. 79234353Sdim // 80234353Sdim // This has the semantics that when adding to the worklist, 81234353Sdim // the item added must be next to be processed. It should 82234353Sdim // also only appear once. The naive approach to this takes 83234353Sdim // linear time. 84234353Sdim // 85234353Sdim // To reduce the insert/remove time to logarithmic, we use 86234353Sdim // a set and a vector to maintain our worklist. 87234353Sdim // 88234353Sdim // The set contains the items on the worklist, but does not 89234353Sdim // maintain the order they should be visited. 90234353Sdim // 91234353Sdim // The vector maintains the order nodes should be visited, but may 92234353Sdim // contain duplicate or removed nodes. When choosing a node to 93234353Sdim // visit, we pop off the order stack until we find an item that is 94234353Sdim // also in the contents set. All operations are O(log N). 95234353Sdim SmallPtrSet<SDNode*, 64> WorkListContents; 96234353Sdim SmallVector<SDNode*, 64> WorkListOrder; 97193323Sed 98193323Sed // AA - Used for DAG load/store alias analysis. 99193323Sed AliasAnalysis &AA; 100193323Sed 101193323Sed /// AddUsersToWorkList - When an instruction is simplified, add all users of 102193323Sed /// the instruction to the work lists because they might get more simplified 103193323Sed /// now. 104193323Sed /// 105193323Sed void AddUsersToWorkList(SDNode *N) { 106193323Sed for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); 107193323Sed UI != UE; ++UI) 108193323Sed AddToWorkList(*UI); 109193323Sed } 110193323Sed 111193323Sed /// visit - call the node-specific routine that knows how to fold each 112193323Sed /// particular type of node. 113193323Sed SDValue visit(SDNode *N); 114193323Sed 115193323Sed public: 116234353Sdim /// AddToWorkList - Add to the work list making sure its instance is at the 117234353Sdim /// back (next to be processed.) 118193323Sed void AddToWorkList(SDNode *N) { 119234353Sdim WorkListContents.insert(N); 120234353Sdim WorkListOrder.push_back(N); 121193323Sed } 122193323Sed 123193323Sed /// removeFromWorkList - remove all instances of N from the worklist. 124193323Sed /// 125193323Sed void removeFromWorkList(SDNode *N) { 126234353Sdim WorkListContents.erase(N); 127193323Sed } 128193323Sed 129193323Sed SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo, 130193323Sed bool AddTo = true); 131193323Sed 132193323Sed SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) { 133193323Sed return CombineTo(N, &Res, 1, AddTo); 134193323Sed } 135193323Sed 136193323Sed SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1, 137193323Sed bool AddTo = true) { 138193323Sed SDValue To[] = { Res0, Res1 }; 139193323Sed return CombineTo(N, To, 2, AddTo); 140193323Sed } 141193323Sed 142193323Sed void CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO); 143193323Sed 144193323Sed private: 145193323Sed 146193323Sed /// SimplifyDemandedBits - Check the specified integer node value to see if 147193323Sed /// it can be simplified or if things it uses can be simplified by bit 148193323Sed /// propagation. If so, return true. 149193323Sed bool SimplifyDemandedBits(SDValue Op) { 150200581Srdivacky unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 151200581Srdivacky APInt Demanded = APInt::getAllOnesValue(BitWidth); 152193323Sed return SimplifyDemandedBits(Op, Demanded); 153193323Sed } 154193323Sed 155193323Sed bool SimplifyDemandedBits(SDValue Op, const APInt &Demanded); 156193323Sed 157193323Sed bool CombineToPreIndexedLoadStore(SDNode *N); 158193323Sed bool CombineToPostIndexedLoadStore(SDNode *N); 159263508Sdim bool SliceUpLoad(SDNode *N); 160193323Sed 161207618Srdivacky void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad); 162207618Srdivacky SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace); 163207618Srdivacky SDValue SExtPromoteOperand(SDValue Op, EVT PVT); 164207618Srdivacky SDValue ZExtPromoteOperand(SDValue Op, EVT PVT); 165207618Srdivacky SDValue PromoteIntBinOp(SDValue Op); 166207618Srdivacky SDValue PromoteIntShiftOp(SDValue Op); 167207618Srdivacky SDValue PromoteExtend(SDValue Op); 168207618Srdivacky bool PromoteLoad(SDValue Op); 169193323Sed 170263508Sdim void ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs, 171263508Sdim SDValue Trunc, SDValue ExtLoad, SDLoc DL, 172224145Sdim ISD::NodeType ExtType); 173224145Sdim 174193323Sed /// combine - call the node-specific routine that knows how to fold each 175193323Sed /// particular type of node. If that doesn't do anything, try the 176193323Sed /// target-specific DAG combines. 177193323Sed SDValue combine(SDNode *N); 178193323Sed 179193323Sed // Visitation implementation - Implement dag node combining for different 180193323Sed // node types. The semantics are as follows: 181193323Sed // Return Value: 182193323Sed // SDValue.getNode() == 0 - No change was made 183193323Sed // SDValue.getNode() == N - N was replaced, is dead and has been handled. 184193323Sed // otherwise - N should be replaced by the returned Operand. 185193323Sed // 186193323Sed SDValue visitTokenFactor(SDNode *N); 187193323Sed SDValue visitMERGE_VALUES(SDNode *N); 188193323Sed SDValue visitADD(SDNode *N); 189193323Sed SDValue visitSUB(SDNode *N); 190193323Sed SDValue visitADDC(SDNode *N); 191234353Sdim SDValue visitSUBC(SDNode *N); 192193323Sed SDValue visitADDE(SDNode *N); 193234353Sdim SDValue visitSUBE(SDNode *N); 194193323Sed SDValue visitMUL(SDNode *N); 195193323Sed SDValue visitSDIV(SDNode *N); 196193323Sed SDValue visitUDIV(SDNode *N); 197193323Sed SDValue visitSREM(SDNode *N); 198193323Sed SDValue visitUREM(SDNode *N); 199193323Sed SDValue visitMULHU(SDNode *N); 200193323Sed SDValue visitMULHS(SDNode *N); 201193323Sed SDValue visitSMUL_LOHI(SDNode *N); 202193323Sed SDValue visitUMUL_LOHI(SDNode *N); 203223017Sdim SDValue visitSMULO(SDNode *N); 204223017Sdim SDValue visitUMULO(SDNode *N); 205193323Sed SDValue visitSDIVREM(SDNode *N); 206193323Sed SDValue visitUDIVREM(SDNode *N); 207193323Sed SDValue visitAND(SDNode *N); 208193323Sed SDValue visitOR(SDNode *N); 209193323Sed SDValue visitXOR(SDNode *N); 210193323Sed SDValue SimplifyVBinOp(SDNode *N); 211243830Sdim SDValue SimplifyVUnaryOp(SDNode *N); 212193323Sed SDValue visitSHL(SDNode *N); 213193323Sed SDValue visitSRA(SDNode *N); 214193323Sed SDValue visitSRL(SDNode *N); 215193323Sed SDValue visitCTLZ(SDNode *N); 216234353Sdim SDValue visitCTLZ_ZERO_UNDEF(SDNode *N); 217193323Sed SDValue visitCTTZ(SDNode *N); 218234353Sdim SDValue visitCTTZ_ZERO_UNDEF(SDNode *N); 219193323Sed SDValue visitCTPOP(SDNode *N); 220193323Sed SDValue visitSELECT(SDNode *N); 221251662Sdim SDValue visitVSELECT(SDNode *N); 222193323Sed SDValue visitSELECT_CC(SDNode *N); 223193323Sed SDValue visitSETCC(SDNode *N); 224193323Sed SDValue visitSIGN_EXTEND(SDNode *N); 225193323Sed SDValue visitZERO_EXTEND(SDNode *N); 226193323Sed SDValue visitANY_EXTEND(SDNode *N); 227193323Sed SDValue visitSIGN_EXTEND_INREG(SDNode *N); 228193323Sed SDValue visitTRUNCATE(SDNode *N); 229218893Sdim SDValue visitBITCAST(SDNode *N); 230193323Sed SDValue visitBUILD_PAIR(SDNode *N); 231193323Sed SDValue visitFADD(SDNode *N); 232193323Sed SDValue visitFSUB(SDNode *N); 233193323Sed SDValue visitFMUL(SDNode *N); 234239462Sdim SDValue visitFMA(SDNode *N); 235193323Sed SDValue visitFDIV(SDNode *N); 236193323Sed SDValue visitFREM(SDNode *N); 237193323Sed SDValue visitFCOPYSIGN(SDNode *N); 238193323Sed SDValue visitSINT_TO_FP(SDNode *N); 239193323Sed SDValue visitUINT_TO_FP(SDNode *N); 240193323Sed SDValue visitFP_TO_SINT(SDNode *N); 241193323Sed SDValue visitFP_TO_UINT(SDNode *N); 242193323Sed SDValue visitFP_ROUND(SDNode *N); 243193323Sed SDValue visitFP_ROUND_INREG(SDNode *N); 244193323Sed SDValue visitFP_EXTEND(SDNode *N); 245193323Sed SDValue visitFNEG(SDNode *N); 246193323Sed SDValue visitFABS(SDNode *N); 247239462Sdim SDValue visitFCEIL(SDNode *N); 248239462Sdim SDValue visitFTRUNC(SDNode *N); 249239462Sdim SDValue visitFFLOOR(SDNode *N); 250193323Sed SDValue visitBRCOND(SDNode *N); 251193323Sed SDValue visitBR_CC(SDNode *N); 252193323Sed SDValue visitLOAD(SDNode *N); 253193323Sed SDValue visitSTORE(SDNode *N); 254193323Sed SDValue visitINSERT_VECTOR_ELT(SDNode *N); 255193323Sed SDValue visitEXTRACT_VECTOR_ELT(SDNode *N); 256193323Sed SDValue visitBUILD_VECTOR(SDNode *N); 257193323Sed SDValue visitCONCAT_VECTORS(SDNode *N); 258226633Sdim SDValue visitEXTRACT_SUBVECTOR(SDNode *N); 259193323Sed SDValue visitVECTOR_SHUFFLE(SDNode *N); 260193323Sed 261193323Sed SDValue XformToShuffleWithZero(SDNode *N); 262263508Sdim SDValue ReassociateOps(unsigned Opc, SDLoc DL, SDValue LHS, SDValue RHS); 263193323Sed 264193323Sed SDValue visitShiftByConstant(SDNode *N, unsigned Amt); 265193323Sed 266193323Sed bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS); 267193323Sed SDValue SimplifyBinOpWithSameOpcodeHands(SDNode *N); 268263508Sdim SDValue SimplifySelect(SDLoc DL, SDValue N0, SDValue N1, SDValue N2); 269263508Sdim SDValue SimplifySelectCC(SDLoc DL, SDValue N0, SDValue N1, SDValue N2, 270193323Sed SDValue N3, ISD::CondCode CC, 271193323Sed bool NotExtCompare = false); 272198090Srdivacky SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, 273263508Sdim SDLoc DL, bool foldBooleans = true); 274193323Sed SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp, 275193323Sed unsigned HiOp); 276198090Srdivacky SDValue CombineConsecutiveLoads(SDNode *N, EVT VT); 277218893Sdim SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT); 278193323Sed SDValue BuildSDIV(SDNode *N); 279193323Sed SDValue BuildUDIV(SDNode *N); 280224145Sdim SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1, 281224145Sdim bool DemandHighBits = true); 282224145Sdim SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1); 283263508Sdim SDNode *MatchRotate(SDValue LHS, SDValue RHS, SDLoc DL); 284193323Sed SDValue ReduceLoadWidth(SDNode *N); 285193323Sed SDValue ReduceLoadOpStoreWidth(SDNode *N); 286218893Sdim SDValue TransformFPLoadStorePair(SDNode *N); 287243830Sdim SDValue reduceBuildVecExtToExtBuildVec(SDNode *N); 288243830Sdim SDValue reduceBuildVecConvertToConvertBuildVec(SDNode *N); 289193323Sed 290193323Sed SDValue GetDemandedBits(SDValue V, const APInt &Mask); 291193323Sed 292193323Sed /// GatherAllAliases - Walk up chain skipping non-aliasing memory nodes, 293193323Sed /// looking for aliasing nodes and adding them to the Aliases vector. 294193323Sed void GatherAllAliases(SDNode *N, SDValue OriginalChain, 295263508Sdim SmallVectorImpl<SDValue> &Aliases); 296193323Sed 297193323Sed /// isAlias - Return true if there is any possibility that the two addresses 298193323Sed /// overlap. 299263508Sdim bool isAlias(SDValue Ptr1, int64_t Size1, bool IsVolatile1, 300193323Sed const Value *SrcValue1, int SrcValueOffset1, 301198090Srdivacky unsigned SrcValueAlign1, 302218893Sdim const MDNode *TBAAInfo1, 303263508Sdim SDValue Ptr2, int64_t Size2, bool IsVolatile2, 304198090Srdivacky const Value *SrcValue2, int SrcValueOffset2, 305218893Sdim unsigned SrcValueAlign2, 306218893Sdim const MDNode *TBAAInfo2) const; 307193323Sed 308249423Sdim /// isAlias - Return true if there is any possibility that the two addresses 309249423Sdim /// overlap. 310249423Sdim bool isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1); 311249423Sdim 312193323Sed /// FindAliasInfo - Extracts the relevant alias information from the memory 313193323Sed /// node. Returns true if the operand was a load. 314193323Sed bool FindAliasInfo(SDNode *N, 315263508Sdim SDValue &Ptr, int64_t &Size, bool &IsVolatile, 316198090Srdivacky const Value *&SrcValue, int &SrcValueOffset, 317218893Sdim unsigned &SrcValueAlignment, 318218893Sdim const MDNode *&TBAAInfo) const; 319193323Sed 320193323Sed /// FindBetterChain - Walk up chain skipping non-aliasing memory nodes, 321193323Sed /// looking for a better chain (aliasing node.) 322193323Sed SDValue FindBetterChain(SDNode *N, SDValue Chain); 323193323Sed 324243830Sdim /// Merge consecutive store operations into a wide store. 325243830Sdim /// This optimization uses wide integers or vectors when possible. 326243830Sdim /// \return True if some memory operations were changed. 327243830Sdim bool MergeConsecutiveStores(StoreSDNode *N); 328243830Sdim 329207618Srdivacky public: 330207618Srdivacky DAGCombiner(SelectionDAG &D, AliasAnalysis &A, CodeGenOpt::Level OL) 331263508Sdim : DAG(D), TLI(D.getTargetLoweringInfo()), Level(BeforeLegalizeTypes), 332263508Sdim OptLevel(OL), LegalOperations(false), LegalTypes(false), AA(A) { 333263508Sdim AttributeSet FnAttrs = 334263508Sdim DAG.getMachineFunction().getFunction()->getAttributes(); 335263508Sdim ForCodeSize = 336263508Sdim FnAttrs.hasAttribute(AttributeSet::FunctionIndex, 337263508Sdim Attribute::OptimizeForSize) || 338263508Sdim FnAttrs.hasAttribute(AttributeSet::FunctionIndex, Attribute::MinSize); 339263508Sdim } 340207618Srdivacky 341207618Srdivacky /// Run - runs the dag combiner on all nodes in the work list 342207618Srdivacky void Run(CombineLevel AtLevel); 343218893Sdim 344207618Srdivacky SelectionDAG &getDAG() const { return DAG; } 345218893Sdim 346193323Sed /// getShiftAmountTy - Returns a type large enough to hold any valid 347193323Sed /// shift amount - before type legalization these can be huge. 348219077Sdim EVT getShiftAmountTy(EVT LHSTy) { 349263508Sdim assert(LHSTy.isInteger() && "Shift amount is not an integer type!"); 350263508Sdim if (LHSTy.isVector()) 351263508Sdim return LHSTy; 352263508Sdim return LegalTypes ? TLI.getScalarShiftAmountTy(LHSTy) 353263508Sdim : TLI.getPointerTy(); 354193323Sed } 355218893Sdim 356207618Srdivacky /// isTypeLegal - This method returns true if we are running before type 357207618Srdivacky /// legalization or if the specified VT is legal. 358207618Srdivacky bool isTypeLegal(const EVT &VT) { 359207618Srdivacky if (!LegalTypes) return true; 360207618Srdivacky return TLI.isTypeLegal(VT); 361207618Srdivacky } 362263508Sdim 363263508Sdim /// getSetCCResultType - Convenience wrapper around 364263508Sdim /// TargetLowering::getSetCCResultType 365263508Sdim EVT getSetCCResultType(EVT VT) const { 366263508Sdim return TLI.getSetCCResultType(*DAG.getContext(), VT); 367263508Sdim } 368193323Sed }; 369193323Sed} 370193323Sed 371193323Sed 372193323Sednamespace { 373193323Sed/// WorkListRemover - This class is a DAGUpdateListener that removes any deleted 374193323Sed/// nodes from the worklist. 375198892Srdivackyclass WorkListRemover : public SelectionDAG::DAGUpdateListener { 376193323Sed DAGCombiner &DC; 377193323Sedpublic: 378239462Sdim explicit WorkListRemover(DAGCombiner &dc) 379239462Sdim : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {} 380193323Sed 381193323Sed virtual void NodeDeleted(SDNode *N, SDNode *E) { 382193323Sed DC.removeFromWorkList(N); 383193323Sed } 384193323Sed}; 385193323Sed} 386193323Sed 387193323Sed//===----------------------------------------------------------------------===// 388193323Sed// TargetLowering::DAGCombinerInfo implementation 389193323Sed//===----------------------------------------------------------------------===// 390193323Sed 391193323Sedvoid TargetLowering::DAGCombinerInfo::AddToWorklist(SDNode *N) { 392193323Sed ((DAGCombiner*)DC)->AddToWorkList(N); 393193323Sed} 394193323Sed 395221345Sdimvoid TargetLowering::DAGCombinerInfo::RemoveFromWorklist(SDNode *N) { 396221345Sdim ((DAGCombiner*)DC)->removeFromWorkList(N); 397221345Sdim} 398221345Sdim 399193323SedSDValue TargetLowering::DAGCombinerInfo:: 400193323SedCombineTo(SDNode *N, const std::vector<SDValue> &To, bool AddTo) { 401193323Sed return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size(), AddTo); 402193323Sed} 403193323Sed 404193323SedSDValue TargetLowering::DAGCombinerInfo:: 405193323SedCombineTo(SDNode *N, SDValue Res, bool AddTo) { 406193323Sed return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo); 407193323Sed} 408193323Sed 409193323Sed 410193323SedSDValue TargetLowering::DAGCombinerInfo:: 411193323SedCombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) { 412193323Sed return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo); 413193323Sed} 414193323Sed 415193323Sedvoid TargetLowering::DAGCombinerInfo:: 416193323SedCommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) { 417193323Sed return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO); 418193323Sed} 419193323Sed 420193323Sed//===----------------------------------------------------------------------===// 421193323Sed// Helper Functions 422193323Sed//===----------------------------------------------------------------------===// 423193323Sed 424193323Sed/// isNegatibleForFree - Return 1 if we can compute the negated form of the 425193323Sed/// specified expression for the same cost as the expression itself, or 2 if we 426193323Sed/// can compute the negated form more cheaply than the expression itself. 427193323Sedstatic char isNegatibleForFree(SDValue Op, bool LegalOperations, 428234353Sdim const TargetLowering &TLI, 429234353Sdim const TargetOptions *Options, 430193323Sed unsigned Depth = 0) { 431193323Sed // fneg is removable even if it has multiple uses. 432193323Sed if (Op.getOpcode() == ISD::FNEG) return 2; 433193323Sed 434193323Sed // Don't allow anything with multiple uses. 435193323Sed if (!Op.hasOneUse()) return 0; 436193323Sed 437193323Sed // Don't recurse exponentially. 438193323Sed if (Depth > 6) return 0; 439193323Sed 440193323Sed switch (Op.getOpcode()) { 441193323Sed default: return false; 442193323Sed case ISD::ConstantFP: 443193323Sed // Don't invert constant FP values after legalize. The negated constant 444193323Sed // isn't necessarily legal. 445193323Sed return LegalOperations ? 0 : 1; 446193323Sed case ISD::FADD: 447193323Sed // FIXME: determine better conditions for this xform. 448234353Sdim if (!Options->UnsafeFPMath) return 0; 449193323Sed 450234353Sdim // After operation legalization, it might not be legal to create new FSUBs. 451234353Sdim if (LegalOperations && 452234353Sdim !TLI.isOperationLegalOrCustom(ISD::FSUB, Op.getValueType())) 453234353Sdim return 0; 454234353Sdim 455243830Sdim // fold (fneg (fadd A, B)) -> (fsub (fneg A), B) 456234353Sdim if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI, 457234353Sdim Options, Depth + 1)) 458193323Sed return V; 459193323Sed // fold (fneg (fadd A, B)) -> (fsub (fneg B), A) 460234353Sdim return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options, 461234353Sdim Depth + 1); 462193323Sed case ISD::FSUB: 463193323Sed // We can't turn -(A-B) into B-A when we honor signed zeros. 464234353Sdim if (!Options->UnsafeFPMath) return 0; 465193323Sed 466193323Sed // fold (fneg (fsub A, B)) -> (fsub B, A) 467193323Sed return 1; 468193323Sed 469193323Sed case ISD::FMUL: 470193323Sed case ISD::FDIV: 471234353Sdim if (Options->HonorSignDependentRoundingFPMath()) return 0; 472193323Sed 473193323Sed // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y)) 474234353Sdim if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI, 475234353Sdim Options, Depth + 1)) 476193323Sed return V; 477193323Sed 478234353Sdim return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options, 479234353Sdim Depth + 1); 480193323Sed 481193323Sed case ISD::FP_EXTEND: 482193323Sed case ISD::FP_ROUND: 483193323Sed case ISD::FSIN: 484234353Sdim return isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI, Options, 485234353Sdim Depth + 1); 486193323Sed } 487193323Sed} 488193323Sed 489193323Sed/// GetNegatedExpression - If isNegatibleForFree returns true, this function 490193323Sed/// returns the newly negated expression. 491193323Sedstatic SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG, 492193323Sed bool LegalOperations, unsigned Depth = 0) { 493193323Sed // fneg is removable even if it has multiple uses. 494193323Sed if (Op.getOpcode() == ISD::FNEG) return Op.getOperand(0); 495193323Sed 496193323Sed // Don't allow anything with multiple uses. 497193323Sed assert(Op.hasOneUse() && "Unknown reuse!"); 498193323Sed 499193323Sed assert(Depth <= 6 && "GetNegatedExpression doesn't match isNegatibleForFree"); 500193323Sed switch (Op.getOpcode()) { 501198090Srdivacky default: llvm_unreachable("Unknown code"); 502193323Sed case ISD::ConstantFP: { 503193323Sed APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF(); 504193323Sed V.changeSign(); 505193323Sed return DAG.getConstantFP(V, Op.getValueType()); 506193323Sed } 507193323Sed case ISD::FADD: 508193323Sed // FIXME: determine better conditions for this xform. 509234353Sdim assert(DAG.getTarget().Options.UnsafeFPMath); 510193323Sed 511193323Sed // fold (fneg (fadd A, B)) -> (fsub (fneg A), B) 512234353Sdim if (isNegatibleForFree(Op.getOperand(0), LegalOperations, 513234353Sdim DAG.getTargetLoweringInfo(), 514234353Sdim &DAG.getTarget().Options, Depth+1)) 515263508Sdim return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(), 516193323Sed GetNegatedExpression(Op.getOperand(0), DAG, 517193323Sed LegalOperations, Depth+1), 518193323Sed Op.getOperand(1)); 519193323Sed // fold (fneg (fadd A, B)) -> (fsub (fneg B), A) 520263508Sdim return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(), 521193323Sed GetNegatedExpression(Op.getOperand(1), DAG, 522193323Sed LegalOperations, Depth+1), 523193323Sed Op.getOperand(0)); 524193323Sed case ISD::FSUB: 525193323Sed // We can't turn -(A-B) into B-A when we honor signed zeros. 526234353Sdim assert(DAG.getTarget().Options.UnsafeFPMath); 527193323Sed 528193323Sed // fold (fneg (fsub 0, B)) -> B 529193323Sed if (ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Op.getOperand(0))) 530193323Sed if (N0CFP->getValueAPF().isZero()) 531193323Sed return Op.getOperand(1); 532193323Sed 533193323Sed // fold (fneg (fsub A, B)) -> (fsub B, A) 534263508Sdim return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(), 535193323Sed Op.getOperand(1), Op.getOperand(0)); 536193323Sed 537193323Sed case ISD::FMUL: 538193323Sed case ISD::FDIV: 539234353Sdim assert(!DAG.getTarget().Options.HonorSignDependentRoundingFPMath()); 540193323Sed 541193323Sed // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) 542234353Sdim if (isNegatibleForFree(Op.getOperand(0), LegalOperations, 543234353Sdim DAG.getTargetLoweringInfo(), 544234353Sdim &DAG.getTarget().Options, Depth+1)) 545263508Sdim return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(), 546193323Sed GetNegatedExpression(Op.getOperand(0), DAG, 547193323Sed LegalOperations, Depth+1), 548193323Sed Op.getOperand(1)); 549193323Sed 550193323Sed // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y)) 551263508Sdim return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(), 552193323Sed Op.getOperand(0), 553193323Sed GetNegatedExpression(Op.getOperand(1), DAG, 554193323Sed LegalOperations, Depth+1)); 555193323Sed 556193323Sed case ISD::FP_EXTEND: 557193323Sed case ISD::FSIN: 558263508Sdim return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(), 559193323Sed GetNegatedExpression(Op.getOperand(0), DAG, 560193323Sed LegalOperations, Depth+1)); 561193323Sed case ISD::FP_ROUND: 562263508Sdim return DAG.getNode(ISD::FP_ROUND, SDLoc(Op), Op.getValueType(), 563193323Sed GetNegatedExpression(Op.getOperand(0), DAG, 564193323Sed LegalOperations, Depth+1), 565193323Sed Op.getOperand(1)); 566193323Sed } 567193323Sed} 568193323Sed 569193323Sed 570193323Sed// isSetCCEquivalent - Return true if this node is a setcc, or is a select_cc 571193323Sed// that selects between the values 1 and 0, making it equivalent to a setcc. 572193323Sed// Also, set the incoming LHS, RHS, and CC references to the appropriate 573193323Sed// nodes based on the type of node we are checking. This simplifies life a 574193323Sed// bit for the callers. 575193323Sedstatic bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS, 576193323Sed SDValue &CC) { 577193323Sed if (N.getOpcode() == ISD::SETCC) { 578193323Sed LHS = N.getOperand(0); 579193323Sed RHS = N.getOperand(1); 580193323Sed CC = N.getOperand(2); 581193323Sed return true; 582193323Sed } 583193323Sed if (N.getOpcode() == ISD::SELECT_CC && 584193323Sed N.getOperand(2).getOpcode() == ISD::Constant && 585193323Sed N.getOperand(3).getOpcode() == ISD::Constant && 586193323Sed cast<ConstantSDNode>(N.getOperand(2))->getAPIntValue() == 1 && 587193323Sed cast<ConstantSDNode>(N.getOperand(3))->isNullValue()) { 588193323Sed LHS = N.getOperand(0); 589193323Sed RHS = N.getOperand(1); 590193323Sed CC = N.getOperand(4); 591193323Sed return true; 592193323Sed } 593193323Sed return false; 594193323Sed} 595193323Sed 596193323Sed// isOneUseSetCC - Return true if this is a SetCC-equivalent operation with only 597193323Sed// one use. If this is true, it allows the users to invert the operation for 598193323Sed// free when it is profitable to do so. 599193323Sedstatic bool isOneUseSetCC(SDValue N) { 600193323Sed SDValue N0, N1, N2; 601193323Sed if (isSetCCEquivalent(N, N0, N1, N2) && N.getNode()->hasOneUse()) 602193323Sed return true; 603193323Sed return false; 604193323Sed} 605193323Sed 606263508SdimSDValue DAGCombiner::ReassociateOps(unsigned Opc, SDLoc DL, 607193323Sed SDValue N0, SDValue N1) { 608198090Srdivacky EVT VT = N0.getValueType(); 609193323Sed if (N0.getOpcode() == Opc && isa<ConstantSDNode>(N0.getOperand(1))) { 610193323Sed if (isa<ConstantSDNode>(N1)) { 611193323Sed // reassoc. (op (op x, c1), c2) -> (op x, (op c1, c2)) 612193323Sed SDValue OpNode = 613193323Sed DAG.FoldConstantArithmetic(Opc, VT, 614193323Sed cast<ConstantSDNode>(N0.getOperand(1)), 615193323Sed cast<ConstantSDNode>(N1)); 616193323Sed return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode); 617223017Sdim } 618223017Sdim if (N0.hasOneUse()) { 619193323Sed // reassoc. (op (op x, c1), y) -> (op (op x, y), c1) iff x+c1 has one use 620263508Sdim SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, 621193323Sed N0.getOperand(0), N1); 622193323Sed AddToWorkList(OpNode.getNode()); 623193323Sed return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1)); 624193323Sed } 625193323Sed } 626193323Sed 627193323Sed if (N1.getOpcode() == Opc && isa<ConstantSDNode>(N1.getOperand(1))) { 628193323Sed if (isa<ConstantSDNode>(N0)) { 629193323Sed // reassoc. (op c2, (op x, c1)) -> (op x, (op c1, c2)) 630193323Sed SDValue OpNode = 631193323Sed DAG.FoldConstantArithmetic(Opc, VT, 632193323Sed cast<ConstantSDNode>(N1.getOperand(1)), 633193323Sed cast<ConstantSDNode>(N0)); 634193323Sed return DAG.getNode(Opc, DL, VT, N1.getOperand(0), OpNode); 635223017Sdim } 636223017Sdim if (N1.hasOneUse()) { 637193323Sed // reassoc. (op y, (op x, c1)) -> (op (op x, y), c1) iff x+c1 has one use 638263508Sdim SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, 639193323Sed N1.getOperand(0), N0); 640193323Sed AddToWorkList(OpNode.getNode()); 641193323Sed return DAG.getNode(Opc, DL, VT, OpNode, N1.getOperand(1)); 642193323Sed } 643193323Sed } 644193323Sed 645193323Sed return SDValue(); 646193323Sed} 647193323Sed 648193323SedSDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo, 649193323Sed bool AddTo) { 650193323Sed assert(N->getNumValues() == NumTo && "Broken CombineTo call!"); 651193323Sed ++NodesCombined; 652202375Srdivacky DEBUG(dbgs() << "\nReplacing.1 "; 653198090Srdivacky N->dump(&DAG); 654202375Srdivacky dbgs() << "\nWith: "; 655198090Srdivacky To[0].getNode()->dump(&DAG); 656202375Srdivacky dbgs() << " and " << NumTo-1 << " other values\n"; 657198090Srdivacky for (unsigned i = 0, e = NumTo; i != e; ++i) 658200581Srdivacky assert((!To[i].getNode() || 659200581Srdivacky N->getValueType(i) == To[i].getValueType()) && 660193323Sed "Cannot combine value to value of different type!")); 661193323Sed WorkListRemover DeadNodes(*this); 662239462Sdim DAG.ReplaceAllUsesWith(N, To); 663193323Sed if (AddTo) { 664193323Sed // Push the new nodes and any users onto the worklist 665193323Sed for (unsigned i = 0, e = NumTo; i != e; ++i) { 666193323Sed if (To[i].getNode()) { 667193323Sed AddToWorkList(To[i].getNode()); 668193323Sed AddUsersToWorkList(To[i].getNode()); 669193323Sed } 670193323Sed } 671193323Sed } 672193323Sed 673193323Sed // Finally, if the node is now dead, remove it from the graph. The node 674193323Sed // may not be dead if the replacement process recursively simplified to 675193323Sed // something else needing this node. 676193323Sed if (N->use_empty()) { 677193323Sed // Nodes can be reintroduced into the worklist. Make sure we do not 678193323Sed // process a node that has been replaced. 679193323Sed removeFromWorkList(N); 680193323Sed 681193323Sed // Finally, since the node is now dead, remove it from the graph. 682193323Sed DAG.DeleteNode(N); 683193323Sed } 684193323Sed return SDValue(N, 0); 685193323Sed} 686193323Sed 687207618Srdivackyvoid DAGCombiner:: 688207618SrdivackyCommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) { 689193323Sed // Replace all uses. If any nodes become isomorphic to other nodes and 690193323Sed // are deleted, make sure to remove them from our worklist. 691193323Sed WorkListRemover DeadNodes(*this); 692239462Sdim DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New); 693193323Sed 694193323Sed // Push the new node and any (possibly new) users onto the worklist. 695193323Sed AddToWorkList(TLO.New.getNode()); 696193323Sed AddUsersToWorkList(TLO.New.getNode()); 697193323Sed 698193323Sed // Finally, if the node is now dead, remove it from the graph. The node 699193323Sed // may not be dead if the replacement process recursively simplified to 700193323Sed // something else needing this node. 701193323Sed if (TLO.Old.getNode()->use_empty()) { 702193323Sed removeFromWorkList(TLO.Old.getNode()); 703193323Sed 704193323Sed // If the operands of this node are only used by the node, they will now 705193323Sed // be dead. Make sure to visit them first to delete dead nodes early. 706193323Sed for (unsigned i = 0, e = TLO.Old.getNode()->getNumOperands(); i != e; ++i) 707193323Sed if (TLO.Old.getNode()->getOperand(i).getNode()->hasOneUse()) 708193323Sed AddToWorkList(TLO.Old.getNode()->getOperand(i).getNode()); 709193323Sed 710193323Sed DAG.DeleteNode(TLO.Old.getNode()); 711193323Sed } 712193323Sed} 713193323Sed 714193323Sed/// SimplifyDemandedBits - Check the specified integer node value to see if 715193323Sed/// it can be simplified or if things it uses can be simplified by bit 716193323Sed/// propagation. If so, return true. 717193323Sedbool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &Demanded) { 718207618Srdivacky TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations); 719193323Sed APInt KnownZero, KnownOne; 720193323Sed if (!TLI.SimplifyDemandedBits(Op, Demanded, KnownZero, KnownOne, TLO)) 721193323Sed return false; 722193323Sed 723193323Sed // Revisit the node. 724193323Sed AddToWorkList(Op.getNode()); 725193323Sed 726193323Sed // Replace the old value with the new one. 727193323Sed ++NodesCombined; 728218893Sdim DEBUG(dbgs() << "\nReplacing.2 "; 729198090Srdivacky TLO.Old.getNode()->dump(&DAG); 730202375Srdivacky dbgs() << "\nWith: "; 731198090Srdivacky TLO.New.getNode()->dump(&DAG); 732202375Srdivacky dbgs() << '\n'); 733193323Sed 734193323Sed CommitTargetLoweringOpt(TLO); 735193323Sed return true; 736193323Sed} 737193323Sed 738207618Srdivackyvoid DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) { 739263508Sdim SDLoc dl(Load); 740207618Srdivacky EVT VT = Load->getValueType(0); 741207618Srdivacky SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, VT, SDValue(ExtLoad, 0)); 742207618Srdivacky 743207618Srdivacky DEBUG(dbgs() << "\nReplacing.9 "; 744207618Srdivacky Load->dump(&DAG); 745207618Srdivacky dbgs() << "\nWith: "; 746207618Srdivacky Trunc.getNode()->dump(&DAG); 747207618Srdivacky dbgs() << '\n'); 748207618Srdivacky WorkListRemover DeadNodes(*this); 749239462Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc); 750239462Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1)); 751207618Srdivacky removeFromWorkList(Load); 752207618Srdivacky DAG.DeleteNode(Load); 753207618Srdivacky AddToWorkList(Trunc.getNode()); 754207618Srdivacky} 755207618Srdivacky 756207618SrdivackySDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) { 757207618Srdivacky Replace = false; 758263508Sdim SDLoc dl(Op); 759207618Srdivacky if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) { 760207618Srdivacky EVT MemVT = LD->getMemoryVT(); 761207618Srdivacky ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) 762219077Sdim ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT) ? ISD::ZEXTLOAD 763218893Sdim : ISD::EXTLOAD) 764207618Srdivacky : LD->getExtensionType(); 765207618Srdivacky Replace = true; 766218893Sdim return DAG.getExtLoad(ExtType, dl, PVT, 767207618Srdivacky LD->getChain(), LD->getBasePtr(), 768263508Sdim MemVT, LD->getMemOperand()); 769207618Srdivacky } 770207618Srdivacky 771207618Srdivacky unsigned Opc = Op.getOpcode(); 772207618Srdivacky switch (Opc) { 773207618Srdivacky default: break; 774207618Srdivacky case ISD::AssertSext: 775207618Srdivacky return DAG.getNode(ISD::AssertSext, dl, PVT, 776207618Srdivacky SExtPromoteOperand(Op.getOperand(0), PVT), 777207618Srdivacky Op.getOperand(1)); 778207618Srdivacky case ISD::AssertZext: 779207618Srdivacky return DAG.getNode(ISD::AssertZext, dl, PVT, 780207618Srdivacky ZExtPromoteOperand(Op.getOperand(0), PVT), 781207618Srdivacky Op.getOperand(1)); 782207618Srdivacky case ISD::Constant: { 783207618Srdivacky unsigned ExtOpc = 784207618Srdivacky Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; 785207618Srdivacky return DAG.getNode(ExtOpc, dl, PVT, Op); 786207618Srdivacky } 787218893Sdim } 788207618Srdivacky 789207618Srdivacky if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT)) 790207618Srdivacky return SDValue(); 791207618Srdivacky return DAG.getNode(ISD::ANY_EXTEND, dl, PVT, Op); 792207618Srdivacky} 793207618Srdivacky 794207618SrdivackySDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) { 795207618Srdivacky if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT)) 796207618Srdivacky return SDValue(); 797207618Srdivacky EVT OldVT = Op.getValueType(); 798263508Sdim SDLoc dl(Op); 799207618Srdivacky bool Replace = false; 800207618Srdivacky SDValue NewOp = PromoteOperand(Op, PVT, Replace); 801207618Srdivacky if (NewOp.getNode() == 0) 802207618Srdivacky return SDValue(); 803207618Srdivacky AddToWorkList(NewOp.getNode()); 804207618Srdivacky 805207618Srdivacky if (Replace) 806207618Srdivacky ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode()); 807207618Srdivacky return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NewOp.getValueType(), NewOp, 808207618Srdivacky DAG.getValueType(OldVT)); 809207618Srdivacky} 810207618Srdivacky 811207618SrdivackySDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) { 812207618Srdivacky EVT OldVT = Op.getValueType(); 813263508Sdim SDLoc dl(Op); 814207618Srdivacky bool Replace = false; 815207618Srdivacky SDValue NewOp = PromoteOperand(Op, PVT, Replace); 816207618Srdivacky if (NewOp.getNode() == 0) 817207618Srdivacky return SDValue(); 818207618Srdivacky AddToWorkList(NewOp.getNode()); 819207618Srdivacky 820207618Srdivacky if (Replace) 821207618Srdivacky ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode()); 822207618Srdivacky return DAG.getZeroExtendInReg(NewOp, dl, OldVT); 823207618Srdivacky} 824207618Srdivacky 825207618Srdivacky/// PromoteIntBinOp - Promote the specified integer binary operation if the 826207618Srdivacky/// target indicates it is beneficial. e.g. On x86, it's usually better to 827207618Srdivacky/// promote i16 operations to i32 since i16 instructions are longer. 828207618SrdivackySDValue DAGCombiner::PromoteIntBinOp(SDValue Op) { 829207618Srdivacky if (!LegalOperations) 830207618Srdivacky return SDValue(); 831207618Srdivacky 832207618Srdivacky EVT VT = Op.getValueType(); 833207618Srdivacky if (VT.isVector() || !VT.isInteger()) 834207618Srdivacky return SDValue(); 835207618Srdivacky 836207618Srdivacky // If operation type is 'undesirable', e.g. i16 on x86, consider 837207618Srdivacky // promoting it. 838207618Srdivacky unsigned Opc = Op.getOpcode(); 839207618Srdivacky if (TLI.isTypeDesirableForOp(Opc, VT)) 840207618Srdivacky return SDValue(); 841207618Srdivacky 842207618Srdivacky EVT PVT = VT; 843207618Srdivacky // Consult target whether it is a good idea to promote this operation and 844207618Srdivacky // what's the right type to promote it to. 845207618Srdivacky if (TLI.IsDesirableToPromoteOp(Op, PVT)) { 846207618Srdivacky assert(PVT != VT && "Don't know what type to promote to!"); 847207618Srdivacky 848207618Srdivacky bool Replace0 = false; 849207618Srdivacky SDValue N0 = Op.getOperand(0); 850207618Srdivacky SDValue NN0 = PromoteOperand(N0, PVT, Replace0); 851207618Srdivacky if (NN0.getNode() == 0) 852207618Srdivacky return SDValue(); 853207618Srdivacky 854207618Srdivacky bool Replace1 = false; 855207618Srdivacky SDValue N1 = Op.getOperand(1); 856208599Srdivacky SDValue NN1; 857208599Srdivacky if (N0 == N1) 858208599Srdivacky NN1 = NN0; 859208599Srdivacky else { 860208599Srdivacky NN1 = PromoteOperand(N1, PVT, Replace1); 861208599Srdivacky if (NN1.getNode() == 0) 862208599Srdivacky return SDValue(); 863208599Srdivacky } 864207618Srdivacky 865207618Srdivacky AddToWorkList(NN0.getNode()); 866208599Srdivacky if (NN1.getNode()) 867208599Srdivacky AddToWorkList(NN1.getNode()); 868207618Srdivacky 869207618Srdivacky if (Replace0) 870207618Srdivacky ReplaceLoadWithPromotedLoad(N0.getNode(), NN0.getNode()); 871207618Srdivacky if (Replace1) 872207618Srdivacky ReplaceLoadWithPromotedLoad(N1.getNode(), NN1.getNode()); 873207618Srdivacky 874207618Srdivacky DEBUG(dbgs() << "\nPromoting "; 875207618Srdivacky Op.getNode()->dump(&DAG)); 876263508Sdim SDLoc dl(Op); 877207618Srdivacky return DAG.getNode(ISD::TRUNCATE, dl, VT, 878207618Srdivacky DAG.getNode(Opc, dl, PVT, NN0, NN1)); 879207618Srdivacky } 880207618Srdivacky return SDValue(); 881207618Srdivacky} 882207618Srdivacky 883207618Srdivacky/// PromoteIntShiftOp - Promote the specified integer shift operation if the 884207618Srdivacky/// target indicates it is beneficial. e.g. On x86, it's usually better to 885207618Srdivacky/// promote i16 operations to i32 since i16 instructions are longer. 886207618SrdivackySDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) { 887207618Srdivacky if (!LegalOperations) 888207618Srdivacky return SDValue(); 889207618Srdivacky 890207618Srdivacky EVT VT = Op.getValueType(); 891207618Srdivacky if (VT.isVector() || !VT.isInteger()) 892207618Srdivacky return SDValue(); 893207618Srdivacky 894207618Srdivacky // If operation type is 'undesirable', e.g. i16 on x86, consider 895207618Srdivacky // promoting it. 896207618Srdivacky unsigned Opc = Op.getOpcode(); 897207618Srdivacky if (TLI.isTypeDesirableForOp(Opc, VT)) 898207618Srdivacky return SDValue(); 899207618Srdivacky 900207618Srdivacky EVT PVT = VT; 901207618Srdivacky // Consult target whether it is a good idea to promote this operation and 902207618Srdivacky // what's the right type to promote it to. 903207618Srdivacky if (TLI.IsDesirableToPromoteOp(Op, PVT)) { 904207618Srdivacky assert(PVT != VT && "Don't know what type to promote to!"); 905207618Srdivacky 906207618Srdivacky bool Replace = false; 907207618Srdivacky SDValue N0 = Op.getOperand(0); 908207618Srdivacky if (Opc == ISD::SRA) 909207618Srdivacky N0 = SExtPromoteOperand(Op.getOperand(0), PVT); 910207618Srdivacky else if (Opc == ISD::SRL) 911207618Srdivacky N0 = ZExtPromoteOperand(Op.getOperand(0), PVT); 912207618Srdivacky else 913207618Srdivacky N0 = PromoteOperand(N0, PVT, Replace); 914207618Srdivacky if (N0.getNode() == 0) 915207618Srdivacky return SDValue(); 916207618Srdivacky 917207618Srdivacky AddToWorkList(N0.getNode()); 918207618Srdivacky if (Replace) 919207618Srdivacky ReplaceLoadWithPromotedLoad(Op.getOperand(0).getNode(), N0.getNode()); 920207618Srdivacky 921207618Srdivacky DEBUG(dbgs() << "\nPromoting "; 922207618Srdivacky Op.getNode()->dump(&DAG)); 923263508Sdim SDLoc dl(Op); 924207618Srdivacky return DAG.getNode(ISD::TRUNCATE, dl, VT, 925207618Srdivacky DAG.getNode(Opc, dl, PVT, N0, Op.getOperand(1))); 926207618Srdivacky } 927207618Srdivacky return SDValue(); 928207618Srdivacky} 929207618Srdivacky 930207618SrdivackySDValue DAGCombiner::PromoteExtend(SDValue Op) { 931207618Srdivacky if (!LegalOperations) 932207618Srdivacky return SDValue(); 933207618Srdivacky 934207618Srdivacky EVT VT = Op.getValueType(); 935207618Srdivacky if (VT.isVector() || !VT.isInteger()) 936207618Srdivacky return SDValue(); 937207618Srdivacky 938207618Srdivacky // If operation type is 'undesirable', e.g. i16 on x86, consider 939207618Srdivacky // promoting it. 940207618Srdivacky unsigned Opc = Op.getOpcode(); 941207618Srdivacky if (TLI.isTypeDesirableForOp(Opc, VT)) 942207618Srdivacky return SDValue(); 943207618Srdivacky 944207618Srdivacky EVT PVT = VT; 945207618Srdivacky // Consult target whether it is a good idea to promote this operation and 946207618Srdivacky // what's the right type to promote it to. 947207618Srdivacky if (TLI.IsDesirableToPromoteOp(Op, PVT)) { 948207618Srdivacky assert(PVT != VT && "Don't know what type to promote to!"); 949207618Srdivacky // fold (aext (aext x)) -> (aext x) 950207618Srdivacky // fold (aext (zext x)) -> (zext x) 951207618Srdivacky // fold (aext (sext x)) -> (sext x) 952207618Srdivacky DEBUG(dbgs() << "\nPromoting "; 953207618Srdivacky Op.getNode()->dump(&DAG)); 954263508Sdim return DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, Op.getOperand(0)); 955207618Srdivacky } 956207618Srdivacky return SDValue(); 957207618Srdivacky} 958207618Srdivacky 959207618Srdivackybool DAGCombiner::PromoteLoad(SDValue Op) { 960207618Srdivacky if (!LegalOperations) 961207618Srdivacky return false; 962207618Srdivacky 963207618Srdivacky EVT VT = Op.getValueType(); 964207618Srdivacky if (VT.isVector() || !VT.isInteger()) 965207618Srdivacky return false; 966207618Srdivacky 967207618Srdivacky // If operation type is 'undesirable', e.g. i16 on x86, consider 968207618Srdivacky // promoting it. 969207618Srdivacky unsigned Opc = Op.getOpcode(); 970207618Srdivacky if (TLI.isTypeDesirableForOp(Opc, VT)) 971207618Srdivacky return false; 972207618Srdivacky 973207618Srdivacky EVT PVT = VT; 974207618Srdivacky // Consult target whether it is a good idea to promote this operation and 975207618Srdivacky // what's the right type to promote it to. 976207618Srdivacky if (TLI.IsDesirableToPromoteOp(Op, PVT)) { 977207618Srdivacky assert(PVT != VT && "Don't know what type to promote to!"); 978207618Srdivacky 979263508Sdim SDLoc dl(Op); 980207618Srdivacky SDNode *N = Op.getNode(); 981207618Srdivacky LoadSDNode *LD = cast<LoadSDNode>(N); 982207618Srdivacky EVT MemVT = LD->getMemoryVT(); 983207618Srdivacky ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) 984219077Sdim ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT) ? ISD::ZEXTLOAD 985218893Sdim : ISD::EXTLOAD) 986207618Srdivacky : LD->getExtensionType(); 987218893Sdim SDValue NewLD = DAG.getExtLoad(ExtType, dl, PVT, 988207618Srdivacky LD->getChain(), LD->getBasePtr(), 989263508Sdim MemVT, LD->getMemOperand()); 990207618Srdivacky SDValue Result = DAG.getNode(ISD::TRUNCATE, dl, VT, NewLD); 991207618Srdivacky 992207618Srdivacky DEBUG(dbgs() << "\nPromoting "; 993207618Srdivacky N->dump(&DAG); 994207618Srdivacky dbgs() << "\nTo: "; 995207618Srdivacky Result.getNode()->dump(&DAG); 996207618Srdivacky dbgs() << '\n'); 997207618Srdivacky WorkListRemover DeadNodes(*this); 998239462Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result); 999239462Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1)); 1000207618Srdivacky removeFromWorkList(N); 1001207618Srdivacky DAG.DeleteNode(N); 1002207618Srdivacky AddToWorkList(Result.getNode()); 1003207618Srdivacky return true; 1004207618Srdivacky } 1005207618Srdivacky return false; 1006207618Srdivacky} 1007207618Srdivacky 1008207618Srdivacky 1009193323Sed//===----------------------------------------------------------------------===// 1010193323Sed// Main DAG Combiner implementation 1011193323Sed//===----------------------------------------------------------------------===// 1012193323Sed 1013193323Sedvoid DAGCombiner::Run(CombineLevel AtLevel) { 1014193323Sed // set the instance variables, so that the various visit routines may use it. 1015193323Sed Level = AtLevel; 1016234353Sdim LegalOperations = Level >= AfterLegalizeVectorOps; 1017234353Sdim LegalTypes = Level >= AfterLegalizeTypes; 1018193323Sed 1019193323Sed // Add all the dag nodes to the worklist. 1020193323Sed for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(), 1021193323Sed E = DAG.allnodes_end(); I != E; ++I) 1022234353Sdim AddToWorkList(I); 1023193323Sed 1024193323Sed // Create a dummy node (which is not added to allnodes), that adds a reference 1025193323Sed // to the root node, preventing it from being deleted, and tracking any 1026193323Sed // changes of the root. 1027193323Sed HandleSDNode Dummy(DAG.getRoot()); 1028193323Sed 1029193323Sed // The root of the dag may dangle to deleted nodes until the dag combiner is 1030193323Sed // done. Set it to null to avoid confusion. 1031193323Sed DAG.setRoot(SDValue()); 1032193323Sed 1033234353Sdim // while the worklist isn't empty, find a node and 1034193323Sed // try and combine it. 1035234353Sdim while (!WorkListContents.empty()) { 1036234353Sdim SDNode *N; 1037263508Sdim // The WorkListOrder holds the SDNodes in order, but it may contain 1038263508Sdim // duplicates. 1039234353Sdim // In order to avoid a linear scan, we use a set (O(log N)) to hold what the 1040234353Sdim // worklist *should* contain, and check the node we want to visit is should 1041234353Sdim // actually be visited. 1042234353Sdim do { 1043234353Sdim N = WorkListOrder.pop_back_val(); 1044234353Sdim } while (!WorkListContents.erase(N)); 1045193323Sed 1046193323Sed // If N has no uses, it is dead. Make sure to revisit all N's operands once 1047193323Sed // N is deleted from the DAG, since they too may now be dead or may have a 1048193323Sed // reduced number of uses, allowing other xforms. 1049193323Sed if (N->use_empty() && N != &Dummy) { 1050193323Sed for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 1051193323Sed AddToWorkList(N->getOperand(i).getNode()); 1052193323Sed 1053193323Sed DAG.DeleteNode(N); 1054193323Sed continue; 1055193323Sed } 1056193323Sed 1057193323Sed SDValue RV = combine(N); 1058193323Sed 1059193323Sed if (RV.getNode() == 0) 1060193323Sed continue; 1061193323Sed 1062193323Sed ++NodesCombined; 1063193323Sed 1064193323Sed // If we get back the same node we passed in, rather than a new node or 1065193323Sed // zero, we know that the node must have defined multiple values and 1066193323Sed // CombineTo was used. Since CombineTo takes care of the worklist 1067193323Sed // mechanics for us, we have no work to do in this case. 1068193323Sed if (RV.getNode() == N) 1069193323Sed continue; 1070193323Sed 1071193323Sed assert(N->getOpcode() != ISD::DELETED_NODE && 1072193323Sed RV.getNode()->getOpcode() != ISD::DELETED_NODE && 1073193323Sed "Node was deleted but visit returned new node!"); 1074193323Sed 1075218893Sdim DEBUG(dbgs() << "\nReplacing.3 "; 1076198090Srdivacky N->dump(&DAG); 1077202375Srdivacky dbgs() << "\nWith: "; 1078198090Srdivacky RV.getNode()->dump(&DAG); 1079202375Srdivacky dbgs() << '\n'); 1080224145Sdim 1081223017Sdim // Transfer debug value. 1082223017Sdim DAG.TransferDbgValues(SDValue(N, 0), RV); 1083193323Sed WorkListRemover DeadNodes(*this); 1084193323Sed if (N->getNumValues() == RV.getNode()->getNumValues()) 1085239462Sdim DAG.ReplaceAllUsesWith(N, RV.getNode()); 1086193323Sed else { 1087193323Sed assert(N->getValueType(0) == RV.getValueType() && 1088193323Sed N->getNumValues() == 1 && "Type mismatch"); 1089193323Sed SDValue OpV = RV; 1090239462Sdim DAG.ReplaceAllUsesWith(N, &OpV); 1091193323Sed } 1092193323Sed 1093193323Sed // Push the new node and any users onto the worklist 1094193323Sed AddToWorkList(RV.getNode()); 1095193323Sed AddUsersToWorkList(RV.getNode()); 1096193323Sed 1097193323Sed // Add any uses of the old node to the worklist in case this node is the 1098193323Sed // last one that uses them. They may become dead after this node is 1099193323Sed // deleted. 1100193323Sed for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 1101193323Sed AddToWorkList(N->getOperand(i).getNode()); 1102193323Sed 1103193323Sed // Finally, if the node is now dead, remove it from the graph. The node 1104193323Sed // may not be dead if the replacement process recursively simplified to 1105193323Sed // something else needing this node. 1106193323Sed if (N->use_empty()) { 1107193323Sed // Nodes can be reintroduced into the worklist. Make sure we do not 1108193323Sed // process a node that has been replaced. 1109193323Sed removeFromWorkList(N); 1110193323Sed 1111193323Sed // Finally, since the node is now dead, remove it from the graph. 1112193323Sed DAG.DeleteNode(N); 1113193323Sed } 1114193323Sed } 1115193323Sed 1116193323Sed // If the root changed (e.g. it was a dead load, update the root). 1117193323Sed DAG.setRoot(Dummy.getValue()); 1118234982Sdim DAG.RemoveDeadNodes(); 1119193323Sed} 1120193323Sed 1121193323SedSDValue DAGCombiner::visit(SDNode *N) { 1122207618Srdivacky switch (N->getOpcode()) { 1123193323Sed default: break; 1124193323Sed case ISD::TokenFactor: return visitTokenFactor(N); 1125193323Sed case ISD::MERGE_VALUES: return visitMERGE_VALUES(N); 1126193323Sed case ISD::ADD: return visitADD(N); 1127193323Sed case ISD::SUB: return visitSUB(N); 1128193323Sed case ISD::ADDC: return visitADDC(N); 1129234353Sdim case ISD::SUBC: return visitSUBC(N); 1130193323Sed case ISD::ADDE: return visitADDE(N); 1131234353Sdim case ISD::SUBE: return visitSUBE(N); 1132193323Sed case ISD::MUL: return visitMUL(N); 1133193323Sed case ISD::SDIV: return visitSDIV(N); 1134193323Sed case ISD::UDIV: return visitUDIV(N); 1135193323Sed case ISD::SREM: return visitSREM(N); 1136193323Sed case ISD::UREM: return visitUREM(N); 1137193323Sed case ISD::MULHU: return visitMULHU(N); 1138193323Sed case ISD::MULHS: return visitMULHS(N); 1139193323Sed case ISD::SMUL_LOHI: return visitSMUL_LOHI(N); 1140193323Sed case ISD::UMUL_LOHI: return visitUMUL_LOHI(N); 1141223017Sdim case ISD::SMULO: return visitSMULO(N); 1142223017Sdim case ISD::UMULO: return visitUMULO(N); 1143193323Sed case ISD::SDIVREM: return visitSDIVREM(N); 1144193323Sed case ISD::UDIVREM: return visitUDIVREM(N); 1145193323Sed case ISD::AND: return visitAND(N); 1146193323Sed case ISD::OR: return visitOR(N); 1147193323Sed case ISD::XOR: return visitXOR(N); 1148193323Sed case ISD::SHL: return visitSHL(N); 1149193323Sed case ISD::SRA: return visitSRA(N); 1150193323Sed case ISD::SRL: return visitSRL(N); 1151193323Sed case ISD::CTLZ: return visitCTLZ(N); 1152234353Sdim case ISD::CTLZ_ZERO_UNDEF: return visitCTLZ_ZERO_UNDEF(N); 1153193323Sed case ISD::CTTZ: return visitCTTZ(N); 1154234353Sdim case ISD::CTTZ_ZERO_UNDEF: return visitCTTZ_ZERO_UNDEF(N); 1155193323Sed case ISD::CTPOP: return visitCTPOP(N); 1156193323Sed case ISD::SELECT: return visitSELECT(N); 1157251662Sdim case ISD::VSELECT: return visitVSELECT(N); 1158193323Sed case ISD::SELECT_CC: return visitSELECT_CC(N); 1159193323Sed case ISD::SETCC: return visitSETCC(N); 1160193323Sed case ISD::SIGN_EXTEND: return visitSIGN_EXTEND(N); 1161193323Sed case ISD::ZERO_EXTEND: return visitZERO_EXTEND(N); 1162193323Sed case ISD::ANY_EXTEND: return visitANY_EXTEND(N); 1163193323Sed case ISD::SIGN_EXTEND_INREG: return visitSIGN_EXTEND_INREG(N); 1164193323Sed case ISD::TRUNCATE: return visitTRUNCATE(N); 1165218893Sdim case ISD::BITCAST: return visitBITCAST(N); 1166193323Sed case ISD::BUILD_PAIR: return visitBUILD_PAIR(N); 1167193323Sed case ISD::FADD: return visitFADD(N); 1168193323Sed case ISD::FSUB: return visitFSUB(N); 1169193323Sed case ISD::FMUL: return visitFMUL(N); 1170239462Sdim case ISD::FMA: return visitFMA(N); 1171193323Sed case ISD::FDIV: return visitFDIV(N); 1172193323Sed case ISD::FREM: return visitFREM(N); 1173193323Sed case ISD::FCOPYSIGN: return visitFCOPYSIGN(N); 1174193323Sed case ISD::SINT_TO_FP: return visitSINT_TO_FP(N); 1175193323Sed case ISD::UINT_TO_FP: return visitUINT_TO_FP(N); 1176193323Sed case ISD::FP_TO_SINT: return visitFP_TO_SINT(N); 1177193323Sed case ISD::FP_TO_UINT: return visitFP_TO_UINT(N); 1178193323Sed case ISD::FP_ROUND: return visitFP_ROUND(N); 1179193323Sed case ISD::FP_ROUND_INREG: return visitFP_ROUND_INREG(N); 1180193323Sed case ISD::FP_EXTEND: return visitFP_EXTEND(N); 1181193323Sed case ISD::FNEG: return visitFNEG(N); 1182193323Sed case ISD::FABS: return visitFABS(N); 1183239462Sdim case ISD::FFLOOR: return visitFFLOOR(N); 1184239462Sdim case ISD::FCEIL: return visitFCEIL(N); 1185239462Sdim case ISD::FTRUNC: return visitFTRUNC(N); 1186193323Sed case ISD::BRCOND: return visitBRCOND(N); 1187193323Sed case ISD::BR_CC: return visitBR_CC(N); 1188193323Sed case ISD::LOAD: return visitLOAD(N); 1189193323Sed case ISD::STORE: return visitSTORE(N); 1190193323Sed case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N); 1191193323Sed case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N); 1192193323Sed case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N); 1193193323Sed case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N); 1194226633Sdim case ISD::EXTRACT_SUBVECTOR: return visitEXTRACT_SUBVECTOR(N); 1195193323Sed case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N); 1196193323Sed } 1197193323Sed return SDValue(); 1198193323Sed} 1199193323Sed 1200193323SedSDValue DAGCombiner::combine(SDNode *N) { 1201193323Sed SDValue RV = visit(N); 1202193323Sed 1203193323Sed // If nothing happened, try a target-specific DAG combine. 1204193323Sed if (RV.getNode() == 0) { 1205193323Sed assert(N->getOpcode() != ISD::DELETED_NODE && 1206193323Sed "Node was deleted but visit returned NULL!"); 1207193323Sed 1208193323Sed if (N->getOpcode() >= ISD::BUILTIN_OP_END || 1209193323Sed TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) { 1210193323Sed 1211193323Sed // Expose the DAG combiner to the target combiner impls. 1212193323Sed TargetLowering::DAGCombinerInfo 1213249423Sdim DagCombineInfo(DAG, Level, false, this); 1214193323Sed 1215193323Sed RV = TLI.PerformDAGCombine(N, DagCombineInfo); 1216193323Sed } 1217193323Sed } 1218193323Sed 1219207618Srdivacky // If nothing happened still, try promoting the operation. 1220207618Srdivacky if (RV.getNode() == 0) { 1221207618Srdivacky switch (N->getOpcode()) { 1222207618Srdivacky default: break; 1223207618Srdivacky case ISD::ADD: 1224207618Srdivacky case ISD::SUB: 1225207618Srdivacky case ISD::MUL: 1226207618Srdivacky case ISD::AND: 1227207618Srdivacky case ISD::OR: 1228207618Srdivacky case ISD::XOR: 1229207618Srdivacky RV = PromoteIntBinOp(SDValue(N, 0)); 1230207618Srdivacky break; 1231207618Srdivacky case ISD::SHL: 1232207618Srdivacky case ISD::SRA: 1233207618Srdivacky case ISD::SRL: 1234207618Srdivacky RV = PromoteIntShiftOp(SDValue(N, 0)); 1235207618Srdivacky break; 1236207618Srdivacky case ISD::SIGN_EXTEND: 1237207618Srdivacky case ISD::ZERO_EXTEND: 1238207618Srdivacky case ISD::ANY_EXTEND: 1239207618Srdivacky RV = PromoteExtend(SDValue(N, 0)); 1240207618Srdivacky break; 1241207618Srdivacky case ISD::LOAD: 1242207618Srdivacky if (PromoteLoad(SDValue(N, 0))) 1243207618Srdivacky RV = SDValue(N, 0); 1244207618Srdivacky break; 1245207618Srdivacky } 1246207618Srdivacky } 1247207618Srdivacky 1248193323Sed // If N is a commutative binary node, try commuting it to enable more 1249193323Sed // sdisel CSE. 1250193323Sed if (RV.getNode() == 0 && 1251193323Sed SelectionDAG::isCommutativeBinOp(N->getOpcode()) && 1252193323Sed N->getNumValues() == 1) { 1253193323Sed SDValue N0 = N->getOperand(0); 1254193323Sed SDValue N1 = N->getOperand(1); 1255193323Sed 1256193323Sed // Constant operands are canonicalized to RHS. 1257193323Sed if (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1)) { 1258193323Sed SDValue Ops[] = { N1, N0 }; 1259193323Sed SDNode *CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), 1260193323Sed Ops, 2); 1261193323Sed if (CSENode) 1262193323Sed return SDValue(CSENode, 0); 1263193323Sed } 1264193323Sed } 1265193323Sed 1266193323Sed return RV; 1267193323Sed} 1268193323Sed 1269193323Sed/// getInputChainForNode - Given a node, return its input chain if it has one, 1270193323Sed/// otherwise return a null sd operand. 1271193323Sedstatic SDValue getInputChainForNode(SDNode *N) { 1272193323Sed if (unsigned NumOps = N->getNumOperands()) { 1273193323Sed if (N->getOperand(0).getValueType() == MVT::Other) 1274193323Sed return N->getOperand(0); 1275263508Sdim if (N->getOperand(NumOps-1).getValueType() == MVT::Other) 1276193323Sed return N->getOperand(NumOps-1); 1277193323Sed for (unsigned i = 1; i < NumOps-1; ++i) 1278193323Sed if (N->getOperand(i).getValueType() == MVT::Other) 1279193323Sed return N->getOperand(i); 1280193323Sed } 1281193323Sed return SDValue(); 1282193323Sed} 1283193323Sed 1284193323SedSDValue DAGCombiner::visitTokenFactor(SDNode *N) { 1285193323Sed // If N has two operands, where one has an input chain equal to the other, 1286193323Sed // the 'other' chain is redundant. 1287193323Sed if (N->getNumOperands() == 2) { 1288193323Sed if (getInputChainForNode(N->getOperand(0).getNode()) == N->getOperand(1)) 1289193323Sed return N->getOperand(0); 1290193323Sed if (getInputChainForNode(N->getOperand(1).getNode()) == N->getOperand(0)) 1291193323Sed return N->getOperand(1); 1292193323Sed } 1293193323Sed 1294193323Sed SmallVector<SDNode *, 8> TFs; // List of token factors to visit. 1295193323Sed SmallVector<SDValue, 8> Ops; // Ops for replacing token factor. 1296193323Sed SmallPtrSet<SDNode*, 16> SeenOps; 1297193323Sed bool Changed = false; // If we should replace this token factor. 1298193323Sed 1299193323Sed // Start out with this token factor. 1300193323Sed TFs.push_back(N); 1301193323Sed 1302193323Sed // Iterate through token factors. The TFs grows when new token factors are 1303193323Sed // encountered. 1304193323Sed for (unsigned i = 0; i < TFs.size(); ++i) { 1305193323Sed SDNode *TF = TFs[i]; 1306193323Sed 1307193323Sed // Check each of the operands. 1308193323Sed for (unsigned i = 0, ie = TF->getNumOperands(); i != ie; ++i) { 1309193323Sed SDValue Op = TF->getOperand(i); 1310193323Sed 1311193323Sed switch (Op.getOpcode()) { 1312193323Sed case ISD::EntryToken: 1313193323Sed // Entry tokens don't need to be added to the list. They are 1314193323Sed // rededundant. 1315193323Sed Changed = true; 1316193323Sed break; 1317193323Sed 1318193323Sed case ISD::TokenFactor: 1319198090Srdivacky if (Op.hasOneUse() && 1320193323Sed std::find(TFs.begin(), TFs.end(), Op.getNode()) == TFs.end()) { 1321193323Sed // Queue up for processing. 1322193323Sed TFs.push_back(Op.getNode()); 1323193323Sed // Clean up in case the token factor is removed. 1324193323Sed AddToWorkList(Op.getNode()); 1325193323Sed Changed = true; 1326193323Sed break; 1327193323Sed } 1328193323Sed // Fall thru 1329193323Sed 1330193323Sed default: 1331193323Sed // Only add if it isn't already in the list. 1332193323Sed if (SeenOps.insert(Op.getNode())) 1333193323Sed Ops.push_back(Op); 1334193323Sed else 1335193323Sed Changed = true; 1336193323Sed break; 1337193323Sed } 1338193323Sed } 1339193323Sed } 1340218893Sdim 1341193323Sed SDValue Result; 1342193323Sed 1343193323Sed // If we've change things around then replace token factor. 1344193323Sed if (Changed) { 1345193323Sed if (Ops.empty()) { 1346193323Sed // The entry token is the only possible outcome. 1347193323Sed Result = DAG.getEntryNode(); 1348193323Sed } else { 1349193323Sed // New and improved token factor. 1350263508Sdim Result = DAG.getNode(ISD::TokenFactor, SDLoc(N), 1351193323Sed MVT::Other, &Ops[0], Ops.size()); 1352193323Sed } 1353193323Sed 1354193323Sed // Don't add users to work list. 1355193323Sed return CombineTo(N, Result, false); 1356193323Sed } 1357193323Sed 1358193323Sed return Result; 1359193323Sed} 1360193323Sed 1361193323Sed/// MERGE_VALUES can always be eliminated. 1362193323SedSDValue DAGCombiner::visitMERGE_VALUES(SDNode *N) { 1363193323Sed WorkListRemover DeadNodes(*this); 1364198090Srdivacky // Replacing results may cause a different MERGE_VALUES to suddenly 1365198090Srdivacky // be CSE'd with N, and carry its uses with it. Iterate until no 1366198090Srdivacky // uses remain, to ensure that the node can be safely deleted. 1367239462Sdim // First add the users of this node to the work list so that they 1368239462Sdim // can be tried again once they have new operands. 1369239462Sdim AddUsersToWorkList(N); 1370198090Srdivacky do { 1371198090Srdivacky for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 1372239462Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(N, i), N->getOperand(i)); 1373198090Srdivacky } while (!N->use_empty()); 1374193323Sed removeFromWorkList(N); 1375193323Sed DAG.DeleteNode(N); 1376193323Sed return SDValue(N, 0); // Return N so it doesn't get rechecked! 1377193323Sed} 1378193323Sed 1379193323Sedstatic 1380263508SdimSDValue combineShlAddConstant(SDLoc DL, SDValue N0, SDValue N1, 1381193323Sed SelectionDAG &DAG) { 1382198090Srdivacky EVT VT = N0.getValueType(); 1383193323Sed SDValue N00 = N0.getOperand(0); 1384193323Sed SDValue N01 = N0.getOperand(1); 1385193323Sed ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N01); 1386193323Sed 1387193323Sed if (N01C && N00.getOpcode() == ISD::ADD && N00.getNode()->hasOneUse() && 1388193323Sed isa<ConstantSDNode>(N00.getOperand(1))) { 1389193323Sed // fold (add (shl (add x, c1), c2), ) -> (add (add (shl x, c2), c1<<c2), ) 1390263508Sdim N0 = DAG.getNode(ISD::ADD, SDLoc(N0), VT, 1391263508Sdim DAG.getNode(ISD::SHL, SDLoc(N00), VT, 1392193323Sed N00.getOperand(0), N01), 1393263508Sdim DAG.getNode(ISD::SHL, SDLoc(N01), VT, 1394193323Sed N00.getOperand(1), N01)); 1395193323Sed return DAG.getNode(ISD::ADD, DL, VT, N0, N1); 1396193323Sed } 1397193323Sed 1398193323Sed return SDValue(); 1399193323Sed} 1400193323Sed 1401193323SedSDValue DAGCombiner::visitADD(SDNode *N) { 1402193323Sed SDValue N0 = N->getOperand(0); 1403193323Sed SDValue N1 = N->getOperand(1); 1404193323Sed ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0); 1405193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 1406198090Srdivacky EVT VT = N0.getValueType(); 1407193323Sed 1408193323Sed // fold vector ops 1409193323Sed if (VT.isVector()) { 1410193323Sed SDValue FoldedVOp = SimplifyVBinOp(N); 1411193323Sed if (FoldedVOp.getNode()) return FoldedVOp; 1412249423Sdim 1413249423Sdim // fold (add x, 0) -> x, vector edition 1414249423Sdim if (ISD::isBuildVectorAllZeros(N1.getNode())) 1415249423Sdim return N0; 1416249423Sdim if (ISD::isBuildVectorAllZeros(N0.getNode())) 1417249423Sdim return N1; 1418193323Sed } 1419193323Sed 1420193323Sed // fold (add x, undef) -> undef 1421193323Sed if (N0.getOpcode() == ISD::UNDEF) 1422193323Sed return N0; 1423193323Sed if (N1.getOpcode() == ISD::UNDEF) 1424193323Sed return N1; 1425193323Sed // fold (add c1, c2) -> c1+c2 1426193323Sed if (N0C && N1C) 1427193323Sed return DAG.FoldConstantArithmetic(ISD::ADD, VT, N0C, N1C); 1428193323Sed // canonicalize constant to RHS 1429193323Sed if (N0C && !N1C) 1430263508Sdim return DAG.getNode(ISD::ADD, SDLoc(N), VT, N1, N0); 1431193323Sed // fold (add x, 0) -> x 1432193323Sed if (N1C && N1C->isNullValue()) 1433193323Sed return N0; 1434193323Sed // fold (add Sym, c) -> Sym+c 1435193323Sed if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0)) 1436193323Sed if (!LegalOperations && TLI.isOffsetFoldingLegal(GA) && N1C && 1437193323Sed GA->getOpcode() == ISD::GlobalAddress) 1438263508Sdim return DAG.getGlobalAddress(GA->getGlobal(), SDLoc(N1C), VT, 1439193323Sed GA->getOffset() + 1440193323Sed (uint64_t)N1C->getSExtValue()); 1441193323Sed // fold ((c1-A)+c2) -> (c1+c2)-A 1442193323Sed if (N1C && N0.getOpcode() == ISD::SUB) 1443193323Sed if (ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0.getOperand(0))) 1444263508Sdim return DAG.getNode(ISD::SUB, SDLoc(N), VT, 1445193323Sed DAG.getConstant(N1C->getAPIntValue()+ 1446193323Sed N0C->getAPIntValue(), VT), 1447193323Sed N0.getOperand(1)); 1448193323Sed // reassociate add 1449263508Sdim SDValue RADD = ReassociateOps(ISD::ADD, SDLoc(N), N0, N1); 1450193323Sed if (RADD.getNode() != 0) 1451193323Sed return RADD; 1452193323Sed // fold ((0-A) + B) -> B-A 1453193323Sed if (N0.getOpcode() == ISD::SUB && isa<ConstantSDNode>(N0.getOperand(0)) && 1454193323Sed cast<ConstantSDNode>(N0.getOperand(0))->isNullValue()) 1455263508Sdim return DAG.getNode(ISD::SUB, SDLoc(N), VT, N1, N0.getOperand(1)); 1456193323Sed // fold (A + (0-B)) -> A-B 1457193323Sed if (N1.getOpcode() == ISD::SUB && isa<ConstantSDNode>(N1.getOperand(0)) && 1458193323Sed cast<ConstantSDNode>(N1.getOperand(0))->isNullValue()) 1459263508Sdim return DAG.getNode(ISD::SUB, SDLoc(N), VT, N0, N1.getOperand(1)); 1460193323Sed // fold (A+(B-A)) -> B 1461193323Sed if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(1)) 1462193323Sed return N1.getOperand(0); 1463193323Sed // fold ((B-A)+A) -> B 1464193323Sed if (N0.getOpcode() == ISD::SUB && N1 == N0.getOperand(1)) 1465193323Sed return N0.getOperand(0); 1466193323Sed // fold (A+(B-(A+C))) to (B-C) 1467193323Sed if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD && 1468193323Sed N0 == N1.getOperand(1).getOperand(0)) 1469263508Sdim return DAG.getNode(ISD::SUB, SDLoc(N), VT, N1.getOperand(0), 1470193323Sed N1.getOperand(1).getOperand(1)); 1471193323Sed // fold (A+(B-(C+A))) to (B-C) 1472193323Sed if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD && 1473193323Sed N0 == N1.getOperand(1).getOperand(1)) 1474263508Sdim return DAG.getNode(ISD::SUB, SDLoc(N), VT, N1.getOperand(0), 1475193323Sed N1.getOperand(1).getOperand(0)); 1476193323Sed // fold (A+((B-A)+or-C)) to (B+or-C) 1477193323Sed if ((N1.getOpcode() == ISD::SUB || N1.getOpcode() == ISD::ADD) && 1478193323Sed N1.getOperand(0).getOpcode() == ISD::SUB && 1479193323Sed N0 == N1.getOperand(0).getOperand(1)) 1480263508Sdim return DAG.getNode(N1.getOpcode(), SDLoc(N), VT, 1481193323Sed N1.getOperand(0).getOperand(0), N1.getOperand(1)); 1482193323Sed 1483193323Sed // fold (A-B)+(C-D) to (A+C)-(B+D) when A or C is constant 1484193323Sed if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB) { 1485193323Sed SDValue N00 = N0.getOperand(0); 1486193323Sed SDValue N01 = N0.getOperand(1); 1487193323Sed SDValue N10 = N1.getOperand(0); 1488193323Sed SDValue N11 = N1.getOperand(1); 1489193323Sed 1490193323Sed if (isa<ConstantSDNode>(N00) || isa<ConstantSDNode>(N10)) 1491263508Sdim return DAG.getNode(ISD::SUB, SDLoc(N), VT, 1492263508Sdim DAG.getNode(ISD::ADD, SDLoc(N0), VT, N00, N10), 1493263508Sdim DAG.getNode(ISD::ADD, SDLoc(N1), VT, N01, N11)); 1494193323Sed } 1495193323Sed 1496193323Sed if (!VT.isVector() && SimplifyDemandedBits(SDValue(N, 0))) 1497193323Sed return SDValue(N, 0); 1498193323Sed 1499193323Sed // fold (a+b) -> (a|b) iff a and b share no bits. 1500193323Sed if (VT.isInteger() && !VT.isVector()) { 1501193323Sed APInt LHSZero, LHSOne; 1502193323Sed APInt RHSZero, RHSOne; 1503234353Sdim DAG.ComputeMaskedBits(N0, LHSZero, LHSOne); 1504193323Sed 1505193323Sed if (LHSZero.getBoolValue()) { 1506234353Sdim DAG.ComputeMaskedBits(N1, RHSZero, RHSOne); 1507193323Sed 1508193323Sed // If all possibly-set bits on the LHS are clear on the RHS, return an OR. 1509193323Sed // If all possibly-set bits on the RHS are clear on the LHS, return an OR. 1510234353Sdim if ((RHSZero & ~LHSZero) == ~LHSZero || (LHSZero & ~RHSZero) == ~RHSZero) 1511263508Sdim return DAG.getNode(ISD::OR, SDLoc(N), VT, N0, N1); 1512193323Sed } 1513193323Sed } 1514193323Sed 1515193323Sed // fold (add (shl (add x, c1), c2), ) -> (add (add (shl x, c2), c1<<c2), ) 1516193323Sed if (N0.getOpcode() == ISD::SHL && N0.getNode()->hasOneUse()) { 1517263508Sdim SDValue Result = combineShlAddConstant(SDLoc(N), N0, N1, DAG); 1518193323Sed if (Result.getNode()) return Result; 1519193323Sed } 1520193323Sed if (N1.getOpcode() == ISD::SHL && N1.getNode()->hasOneUse()) { 1521263508Sdim SDValue Result = combineShlAddConstant(SDLoc(N), N1, N0, DAG); 1522193323Sed if (Result.getNode()) return Result; 1523193323Sed } 1524193323Sed 1525202878Srdivacky // fold (add x, shl(0 - y, n)) -> sub(x, shl(y, n)) 1526202878Srdivacky if (N1.getOpcode() == ISD::SHL && 1527202878Srdivacky N1.getOperand(0).getOpcode() == ISD::SUB) 1528202878Srdivacky if (ConstantSDNode *C = 1529202878Srdivacky dyn_cast<ConstantSDNode>(N1.getOperand(0).getOperand(0))) 1530202878Srdivacky if (C->getAPIntValue() == 0) 1531263508Sdim return DAG.getNode(ISD::SUB, SDLoc(N), VT, N0, 1532263508Sdim DAG.getNode(ISD::SHL, SDLoc(N), VT, 1533202878Srdivacky N1.getOperand(0).getOperand(1), 1534202878Srdivacky N1.getOperand(1))); 1535202878Srdivacky if (N0.getOpcode() == ISD::SHL && 1536202878Srdivacky N0.getOperand(0).getOpcode() == ISD::SUB) 1537202878Srdivacky if (ConstantSDNode *C = 1538202878Srdivacky dyn_cast<ConstantSDNode>(N0.getOperand(0).getOperand(0))) 1539202878Srdivacky if (C->getAPIntValue() == 0) 1540263508Sdim return DAG.getNode(ISD::SUB, SDLoc(N), VT, N1, 1541263508Sdim DAG.getNode(ISD::SHL, SDLoc(N), VT, 1542202878Srdivacky N0.getOperand(0).getOperand(1), 1543202878Srdivacky N0.getOperand(1))); 1544202878Srdivacky 1545218893Sdim if (N1.getOpcode() == ISD::AND) { 1546218893Sdim SDValue AndOp0 = N1.getOperand(0); 1547218893Sdim ConstantSDNode *AndOp1 = dyn_cast<ConstantSDNode>(N1->getOperand(1)); 1548218893Sdim unsigned NumSignBits = DAG.ComputeNumSignBits(AndOp0); 1549218893Sdim unsigned DestBits = VT.getScalarType().getSizeInBits(); 1550218893Sdim 1551218893Sdim // (add z, (and (sbbl x, x), 1)) -> (sub z, (sbbl x, x)) 1552218893Sdim // and similar xforms where the inner op is either ~0 or 0. 1553218893Sdim if (NumSignBits == DestBits && AndOp1 && AndOp1->isOne()) { 1554263508Sdim SDLoc DL(N); 1555218893Sdim return DAG.getNode(ISD::SUB, DL, VT, N->getOperand(0), AndOp0); 1556218893Sdim } 1557218893Sdim } 1558218893Sdim 1559218893Sdim // add (sext i1), X -> sub X, (zext i1) 1560218893Sdim if (N0.getOpcode() == ISD::SIGN_EXTEND && 1561218893Sdim N0.getOperand(0).getValueType() == MVT::i1 && 1562218893Sdim !TLI.isOperationLegal(ISD::SIGN_EXTEND, MVT::i1)) { 1563263508Sdim SDLoc DL(N); 1564218893Sdim SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)); 1565218893Sdim return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt); 1566218893Sdim } 1567218893Sdim 1568193323Sed return SDValue(); 1569193323Sed} 1570193323Sed 1571193323SedSDValue DAGCombiner::visitADDC(SDNode *N) { 1572193323Sed SDValue N0 = N->getOperand(0); 1573193323Sed SDValue N1 = N->getOperand(1); 1574193323Sed ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0); 1575193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 1576198090Srdivacky EVT VT = N0.getValueType(); 1577193323Sed 1578193323Sed // If the flag result is dead, turn this into an ADD. 1579234353Sdim if (!N->hasAnyUseOfValue(1)) 1580263508Sdim return CombineTo(N, DAG.getNode(ISD::ADD, SDLoc(N), VT, N0, N1), 1581193323Sed DAG.getNode(ISD::CARRY_FALSE, 1582263508Sdim SDLoc(N), MVT::Glue)); 1583193323Sed 1584193323Sed // canonicalize constant to RHS. 1585193323Sed if (N0C && !N1C) 1586263508Sdim return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N1, N0); 1587193323Sed 1588193323Sed // fold (addc x, 0) -> x + no carry out 1589193323Sed if (N1C && N1C->isNullValue()) 1590193323Sed return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, 1591263508Sdim SDLoc(N), MVT::Glue)); 1592193323Sed 1593193323Sed // fold (addc a, b) -> (or a, b), CARRY_FALSE iff a and b share no bits. 1594193323Sed APInt LHSZero, LHSOne; 1595193323Sed APInt RHSZero, RHSOne; 1596234353Sdim DAG.ComputeMaskedBits(N0, LHSZero, LHSOne); 1597193323Sed 1598193323Sed if (LHSZero.getBoolValue()) { 1599234353Sdim DAG.ComputeMaskedBits(N1, RHSZero, RHSOne); 1600193323Sed 1601193323Sed // If all possibly-set bits on the LHS are clear on the RHS, return an OR. 1602193323Sed // If all possibly-set bits on the RHS are clear on the LHS, return an OR. 1603234353Sdim if ((RHSZero & ~LHSZero) == ~LHSZero || (LHSZero & ~RHSZero) == ~RHSZero) 1604263508Sdim return CombineTo(N, DAG.getNode(ISD::OR, SDLoc(N), VT, N0, N1), 1605193323Sed DAG.getNode(ISD::CARRY_FALSE, 1606263508Sdim SDLoc(N), MVT::Glue)); 1607193323Sed } 1608193323Sed 1609193323Sed return SDValue(); 1610193323Sed} 1611193323Sed 1612193323SedSDValue DAGCombiner::visitADDE(SDNode *N) { 1613193323Sed SDValue N0 = N->getOperand(0); 1614193323Sed SDValue N1 = N->getOperand(1); 1615193323Sed SDValue CarryIn = N->getOperand(2); 1616193323Sed ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0); 1617193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 1618193323Sed 1619193323Sed // canonicalize constant to RHS 1620193323Sed if (N0C && !N1C) 1621263508Sdim return DAG.getNode(ISD::ADDE, SDLoc(N), N->getVTList(), 1622193323Sed N1, N0, CarryIn); 1623193323Sed 1624193323Sed // fold (adde x, y, false) -> (addc x, y) 1625193323Sed if (CarryIn.getOpcode() == ISD::CARRY_FALSE) 1626263508Sdim return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N0, N1); 1627193323Sed 1628193323Sed return SDValue(); 1629193323Sed} 1630193323Sed 1631218893Sdim// Since it may not be valid to emit a fold to zero for vector initializers 1632218893Sdim// check if we can before folding. 1633263508Sdimstatic SDValue tryFoldToZero(SDLoc DL, const TargetLowering &TLI, EVT VT, 1634263508Sdim SelectionDAG &DAG, 1635263508Sdim bool LegalOperations, bool LegalTypes) { 1636263508Sdim if (!VT.isVector()) 1637218893Sdim return DAG.getConstant(0, VT); 1638263508Sdim if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) 1639263508Sdim return DAG.getConstant(0, VT); 1640218893Sdim return SDValue(); 1641218893Sdim} 1642218893Sdim 1643193323SedSDValue DAGCombiner::visitSUB(SDNode *N) { 1644193323Sed SDValue N0 = N->getOperand(0); 1645193323Sed SDValue N1 = N->getOperand(1); 1646193323Sed ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0.getNode()); 1647193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode()); 1648224145Sdim ConstantSDNode *N1C1 = N1.getOpcode() != ISD::ADD ? 0 : 1649224145Sdim dyn_cast<ConstantSDNode>(N1.getOperand(1).getNode()); 1650198090Srdivacky EVT VT = N0.getValueType(); 1651193323Sed 1652193323Sed // fold vector ops 1653193323Sed if (VT.isVector()) { 1654193323Sed SDValue FoldedVOp = SimplifyVBinOp(N); 1655193323Sed if (FoldedVOp.getNode()) return FoldedVOp; 1656249423Sdim 1657249423Sdim // fold (sub x, 0) -> x, vector edition 1658249423Sdim if (ISD::isBuildVectorAllZeros(N1.getNode())) 1659249423Sdim return N0; 1660193323Sed } 1661193323Sed 1662193323Sed // fold (sub x, x) -> 0 1663218893Sdim // FIXME: Refactor this and xor and other similar operations together. 1664193323Sed if (N0 == N1) 1665263508Sdim return tryFoldToZero(SDLoc(N), TLI, VT, DAG, LegalOperations, LegalTypes); 1666193323Sed // fold (sub c1, c2) -> c1-c2 1667193323Sed if (N0C && N1C) 1668193323Sed return DAG.FoldConstantArithmetic(ISD::SUB, VT, N0C, N1C); 1669193323Sed // fold (sub x, c) -> (add x, -c) 1670193323Sed if (N1C) 1671263508Sdim return DAG.getNode(ISD::ADD, SDLoc(N), VT, N0, 1672193323Sed DAG.getConstant(-N1C->getAPIntValue(), VT)); 1673202878Srdivacky // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1) 1674202878Srdivacky if (N0C && N0C->isAllOnesValue()) 1675263508Sdim return DAG.getNode(ISD::XOR, SDLoc(N), VT, N1, N0); 1676218893Sdim // fold A-(A-B) -> B 1677218893Sdim if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(0)) 1678218893Sdim return N1.getOperand(1); 1679193323Sed // fold (A+B)-A -> B 1680193323Sed if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1) 1681193323Sed return N0.getOperand(1); 1682193323Sed // fold (A+B)-B -> A 1683193323Sed if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1) 1684193323Sed return N0.getOperand(0); 1685224145Sdim // fold C2-(A+C1) -> (C2-C1)-A 1686224145Sdim if (N1.getOpcode() == ISD::ADD && N0C && N1C1) { 1687243830Sdim SDValue NewC = DAG.getConstant(N0C->getAPIntValue() - N1C1->getAPIntValue(), 1688243830Sdim VT); 1689263508Sdim return DAG.getNode(ISD::SUB, SDLoc(N), VT, NewC, 1690239462Sdim N1.getOperand(0)); 1691224145Sdim } 1692193323Sed // fold ((A+(B+or-C))-B) -> A+or-C 1693193323Sed if (N0.getOpcode() == ISD::ADD && 1694193323Sed (N0.getOperand(1).getOpcode() == ISD::SUB || 1695193323Sed N0.getOperand(1).getOpcode() == ISD::ADD) && 1696193323Sed N0.getOperand(1).getOperand(0) == N1) 1697263508Sdim return DAG.getNode(N0.getOperand(1).getOpcode(), SDLoc(N), VT, 1698193323Sed N0.getOperand(0), N0.getOperand(1).getOperand(1)); 1699193323Sed // fold ((A+(C+B))-B) -> A+C 1700193323Sed if (N0.getOpcode() == ISD::ADD && 1701193323Sed N0.getOperand(1).getOpcode() == ISD::ADD && 1702193323Sed N0.getOperand(1).getOperand(1) == N1) 1703263508Sdim return DAG.getNode(ISD::ADD, SDLoc(N), VT, 1704193323Sed N0.getOperand(0), N0.getOperand(1).getOperand(0)); 1705193323Sed // fold ((A-(B-C))-C) -> A-B 1706193323Sed if (N0.getOpcode() == ISD::SUB && 1707193323Sed N0.getOperand(1).getOpcode() == ISD::SUB && 1708193323Sed N0.getOperand(1).getOperand(1) == N1) 1709263508Sdim return DAG.getNode(ISD::SUB, SDLoc(N), VT, 1710193323Sed N0.getOperand(0), N0.getOperand(1).getOperand(0)); 1711193323Sed 1712193323Sed // If either operand of a sub is undef, the result is undef 1713193323Sed if (N0.getOpcode() == ISD::UNDEF) 1714193323Sed return N0; 1715193323Sed if (N1.getOpcode() == ISD::UNDEF) 1716193323Sed return N1; 1717193323Sed 1718193323Sed // If the relocation model supports it, consider symbol offsets. 1719193323Sed if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0)) 1720193323Sed if (!LegalOperations && TLI.isOffsetFoldingLegal(GA)) { 1721193323Sed // fold (sub Sym, c) -> Sym-c 1722193323Sed if (N1C && GA->getOpcode() == ISD::GlobalAddress) 1723263508Sdim return DAG.getGlobalAddress(GA->getGlobal(), SDLoc(N1C), VT, 1724193323Sed GA->getOffset() - 1725193323Sed (uint64_t)N1C->getSExtValue()); 1726193323Sed // fold (sub Sym+c1, Sym+c2) -> c1-c2 1727193323Sed if (GlobalAddressSDNode *GB = dyn_cast<GlobalAddressSDNode>(N1)) 1728193323Sed if (GA->getGlobal() == GB->getGlobal()) 1729193323Sed return DAG.getConstant((uint64_t)GA->getOffset() - GB->getOffset(), 1730193323Sed VT); 1731193323Sed } 1732193323Sed 1733193323Sed return SDValue(); 1734193323Sed} 1735193323Sed 1736234353SdimSDValue DAGCombiner::visitSUBC(SDNode *N) { 1737234353Sdim SDValue N0 = N->getOperand(0); 1738234353Sdim SDValue N1 = N->getOperand(1); 1739234353Sdim ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0); 1740234353Sdim ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 1741234353Sdim EVT VT = N0.getValueType(); 1742234353Sdim 1743234353Sdim // If the flag result is dead, turn this into an SUB. 1744234353Sdim if (!N->hasAnyUseOfValue(1)) 1745263508Sdim return CombineTo(N, DAG.getNode(ISD::SUB, SDLoc(N), VT, N0, N1), 1746263508Sdim DAG.getNode(ISD::CARRY_FALSE, SDLoc(N), 1747234353Sdim MVT::Glue)); 1748234353Sdim 1749234353Sdim // fold (subc x, x) -> 0 + no borrow 1750234353Sdim if (N0 == N1) 1751234353Sdim return CombineTo(N, DAG.getConstant(0, VT), 1752263508Sdim DAG.getNode(ISD::CARRY_FALSE, SDLoc(N), 1753234353Sdim MVT::Glue)); 1754234353Sdim 1755234353Sdim // fold (subc x, 0) -> x + no borrow 1756234353Sdim if (N1C && N1C->isNullValue()) 1757263508Sdim return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, SDLoc(N), 1758234353Sdim MVT::Glue)); 1759234353Sdim 1760234353Sdim // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1) + no borrow 1761234353Sdim if (N0C && N0C->isAllOnesValue()) 1762263508Sdim return CombineTo(N, DAG.getNode(ISD::XOR, SDLoc(N), VT, N1, N0), 1763263508Sdim DAG.getNode(ISD::CARRY_FALSE, SDLoc(N), 1764234353Sdim MVT::Glue)); 1765234353Sdim 1766234353Sdim return SDValue(); 1767234353Sdim} 1768234353Sdim 1769234353SdimSDValue DAGCombiner::visitSUBE(SDNode *N) { 1770234353Sdim SDValue N0 = N->getOperand(0); 1771234353Sdim SDValue N1 = N->getOperand(1); 1772234353Sdim SDValue CarryIn = N->getOperand(2); 1773234353Sdim 1774234353Sdim // fold (sube x, y, false) -> (subc x, y) 1775234353Sdim if (CarryIn.getOpcode() == ISD::CARRY_FALSE) 1776263508Sdim return DAG.getNode(ISD::SUBC, SDLoc(N), N->getVTList(), N0, N1); 1777234353Sdim 1778234353Sdim return SDValue(); 1779234353Sdim} 1780234353Sdim 1781263508Sdim/// isConstantSplatVector - Returns true if N is a BUILD_VECTOR node whose 1782263508Sdim/// elements are all the same constant or undefined. 1783263508Sdimstatic bool isConstantSplatVector(SDNode *N, APInt& SplatValue) { 1784263508Sdim BuildVectorSDNode *C = dyn_cast<BuildVectorSDNode>(N); 1785263508Sdim if (!C) 1786263508Sdim return false; 1787263508Sdim 1788263508Sdim APInt SplatUndef; 1789263508Sdim unsigned SplatBitSize; 1790263508Sdim bool HasAnyUndefs; 1791263508Sdim EVT EltVT = N->getValueType(0).getVectorElementType(); 1792263508Sdim return (C->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, 1793263508Sdim HasAnyUndefs) && 1794263508Sdim EltVT.getSizeInBits() >= SplatBitSize); 1795263508Sdim} 1796263508Sdim 1797193323SedSDValue DAGCombiner::visitMUL(SDNode *N) { 1798193323Sed SDValue N0 = N->getOperand(0); 1799193323Sed SDValue N1 = N->getOperand(1); 1800198090Srdivacky EVT VT = N0.getValueType(); 1801193323Sed 1802263508Sdim // fold (mul x, undef) -> 0 1803263508Sdim if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) 1804263508Sdim return DAG.getConstant(0, VT); 1805263508Sdim 1806263508Sdim bool N0IsConst = false; 1807263508Sdim bool N1IsConst = false; 1808263508Sdim APInt ConstValue0, ConstValue1; 1809193323Sed // fold vector ops 1810193323Sed if (VT.isVector()) { 1811193323Sed SDValue FoldedVOp = SimplifyVBinOp(N); 1812193323Sed if (FoldedVOp.getNode()) return FoldedVOp; 1813263508Sdim 1814263508Sdim N0IsConst = isConstantSplatVector(N0.getNode(), ConstValue0); 1815263508Sdim N1IsConst = isConstantSplatVector(N1.getNode(), ConstValue1); 1816263508Sdim } else { 1817263508Sdim N0IsConst = dyn_cast<ConstantSDNode>(N0) != 0; 1818263508Sdim ConstValue0 = N0IsConst ? (dyn_cast<ConstantSDNode>(N0))->getAPIntValue() 1819263508Sdim : APInt(); 1820263508Sdim N1IsConst = dyn_cast<ConstantSDNode>(N1) != 0; 1821263508Sdim ConstValue1 = N1IsConst ? (dyn_cast<ConstantSDNode>(N1))->getAPIntValue() 1822263508Sdim : APInt(); 1823193323Sed } 1824193323Sed 1825193323Sed // fold (mul c1, c2) -> c1*c2 1826263508Sdim if (N0IsConst && N1IsConst) 1827263508Sdim return DAG.FoldConstantArithmetic(ISD::MUL, VT, N0.getNode(), N1.getNode()); 1828263508Sdim 1829193323Sed // canonicalize constant to RHS 1830263508Sdim if (N0IsConst && !N1IsConst) 1831263508Sdim return DAG.getNode(ISD::MUL, SDLoc(N), VT, N1, N0); 1832193323Sed // fold (mul x, 0) -> 0 1833263508Sdim if (N1IsConst && ConstValue1 == 0) 1834193323Sed return N1; 1835263508Sdim // We require a splat of the entire scalar bit width for non-contiguous 1836263508Sdim // bit patterns. 1837263508Sdim bool IsFullSplat = 1838263508Sdim ConstValue1.getBitWidth() == VT.getScalarType().getSizeInBits(); 1839263508Sdim // fold (mul x, 1) -> x 1840263508Sdim if (N1IsConst && ConstValue1 == 1 && IsFullSplat) 1841263508Sdim return N0; 1842193323Sed // fold (mul x, -1) -> 0-x 1843263508Sdim if (N1IsConst && ConstValue1.isAllOnesValue()) 1844263508Sdim return DAG.getNode(ISD::SUB, SDLoc(N), VT, 1845193323Sed DAG.getConstant(0, VT), N0); 1846193323Sed // fold (mul x, (1 << c)) -> x << c 1847263508Sdim if (N1IsConst && ConstValue1.isPowerOf2() && IsFullSplat) 1848263508Sdim return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, 1849263508Sdim DAG.getConstant(ConstValue1.logBase2(), 1850219077Sdim getShiftAmountTy(N0.getValueType()))); 1851193323Sed // fold (mul x, -(1 << c)) -> -(x << c) or (-x) << c 1852263508Sdim if (N1IsConst && (-ConstValue1).isPowerOf2() && IsFullSplat) { 1853263508Sdim unsigned Log2Val = (-ConstValue1).logBase2(); 1854193323Sed // FIXME: If the input is something that is easily negated (e.g. a 1855193323Sed // single-use add), we should put the negate there. 1856263508Sdim return DAG.getNode(ISD::SUB, SDLoc(N), VT, 1857193323Sed DAG.getConstant(0, VT), 1858263508Sdim DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, 1859219077Sdim DAG.getConstant(Log2Val, 1860219077Sdim getShiftAmountTy(N0.getValueType())))); 1861193323Sed } 1862263508Sdim 1863263508Sdim APInt Val; 1864193323Sed // (mul (shl X, c1), c2) -> (mul X, c2 << c1) 1865263508Sdim if (N1IsConst && N0.getOpcode() == ISD::SHL && 1866263508Sdim (isConstantSplatVector(N0.getOperand(1).getNode(), Val) || 1867263508Sdim isa<ConstantSDNode>(N0.getOperand(1)))) { 1868263508Sdim SDValue C3 = DAG.getNode(ISD::SHL, SDLoc(N), VT, 1869193323Sed N1, N0.getOperand(1)); 1870193323Sed AddToWorkList(C3.getNode()); 1871263508Sdim return DAG.getNode(ISD::MUL, SDLoc(N), VT, 1872193323Sed N0.getOperand(0), C3); 1873193323Sed } 1874193323Sed 1875193323Sed // Change (mul (shl X, C), Y) -> (shl (mul X, Y), C) when the shift has one 1876193323Sed // use. 1877193323Sed { 1878193323Sed SDValue Sh(0,0), Y(0,0); 1879193323Sed // Check for both (mul (shl X, C), Y) and (mul Y, (shl X, C)). 1880263508Sdim if (N0.getOpcode() == ISD::SHL && 1881263508Sdim (isConstantSplatVector(N0.getOperand(1).getNode(), Val) || 1882263508Sdim isa<ConstantSDNode>(N0.getOperand(1))) && 1883193323Sed N0.getNode()->hasOneUse()) { 1884193323Sed Sh = N0; Y = N1; 1885193323Sed } else if (N1.getOpcode() == ISD::SHL && 1886193323Sed isa<ConstantSDNode>(N1.getOperand(1)) && 1887193323Sed N1.getNode()->hasOneUse()) { 1888193323Sed Sh = N1; Y = N0; 1889193323Sed } 1890193323Sed 1891193323Sed if (Sh.getNode()) { 1892263508Sdim SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT, 1893193323Sed Sh.getOperand(0), Y); 1894263508Sdim return DAG.getNode(ISD::SHL, SDLoc(N), VT, 1895193323Sed Mul, Sh.getOperand(1)); 1896193323Sed } 1897193323Sed } 1898193323Sed 1899193323Sed // fold (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2) 1900263508Sdim if (N1IsConst && N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse() && 1901263508Sdim (isConstantSplatVector(N0.getOperand(1).getNode(), Val) || 1902263508Sdim isa<ConstantSDNode>(N0.getOperand(1)))) 1903263508Sdim return DAG.getNode(ISD::ADD, SDLoc(N), VT, 1904263508Sdim DAG.getNode(ISD::MUL, SDLoc(N0), VT, 1905193323Sed N0.getOperand(0), N1), 1906263508Sdim DAG.getNode(ISD::MUL, SDLoc(N1), VT, 1907193323Sed N0.getOperand(1), N1)); 1908193323Sed 1909193323Sed // reassociate mul 1910263508Sdim SDValue RMUL = ReassociateOps(ISD::MUL, SDLoc(N), N0, N1); 1911193323Sed if (RMUL.getNode() != 0) 1912193323Sed return RMUL; 1913193323Sed 1914193323Sed return SDValue(); 1915193323Sed} 1916193323Sed 1917193323SedSDValue DAGCombiner::visitSDIV(SDNode *N) { 1918193323Sed SDValue N0 = N->getOperand(0); 1919193323Sed SDValue N1 = N->getOperand(1); 1920193323Sed ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0.getNode()); 1921193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode()); 1922198090Srdivacky EVT VT = N->getValueType(0); 1923193323Sed 1924193323Sed // fold vector ops 1925193323Sed if (VT.isVector()) { 1926193323Sed SDValue FoldedVOp = SimplifyVBinOp(N); 1927193323Sed if (FoldedVOp.getNode()) return FoldedVOp; 1928193323Sed } 1929193323Sed 1930193323Sed // fold (sdiv c1, c2) -> c1/c2 1931193323Sed if (N0C && N1C && !N1C->isNullValue()) 1932193323Sed return DAG.FoldConstantArithmetic(ISD::SDIV, VT, N0C, N1C); 1933193323Sed // fold (sdiv X, 1) -> X 1934234353Sdim if (N1C && N1C->getAPIntValue() == 1LL) 1935193323Sed return N0; 1936193323Sed // fold (sdiv X, -1) -> 0-X 1937193323Sed if (N1C && N1C->isAllOnesValue()) 1938263508Sdim return DAG.getNode(ISD::SUB, SDLoc(N), VT, 1939193323Sed DAG.getConstant(0, VT), N0); 1940193323Sed // If we know the sign bits of both operands are zero, strength reduce to a 1941193323Sed // udiv instead. Handles (X&15) /s 4 -> X&15 >> 2 1942193323Sed if (!VT.isVector()) { 1943193323Sed if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0)) 1944263508Sdim return DAG.getNode(ISD::UDIV, SDLoc(N), N1.getValueType(), 1945193323Sed N0, N1); 1946193323Sed } 1947193323Sed // fold (sdiv X, pow2) -> simple ops after legalize 1948234353Sdim if (N1C && !N1C->isNullValue() && 1949234353Sdim (N1C->getAPIntValue().isPowerOf2() || 1950234353Sdim (-N1C->getAPIntValue()).isPowerOf2())) { 1951193323Sed // If dividing by powers of two is cheap, then don't perform the following 1952193323Sed // fold. 1953193323Sed if (TLI.isPow2DivCheap()) 1954193323Sed return SDValue(); 1955193323Sed 1956234353Sdim unsigned lg2 = N1C->getAPIntValue().countTrailingZeros(); 1957193323Sed 1958193323Sed // Splat the sign bit into the register 1959263508Sdim SDValue SGN = DAG.getNode(ISD::SRA, SDLoc(N), VT, N0, 1960193323Sed DAG.getConstant(VT.getSizeInBits()-1, 1961219077Sdim getShiftAmountTy(N0.getValueType()))); 1962193323Sed AddToWorkList(SGN.getNode()); 1963193323Sed 1964193323Sed // Add (N0 < 0) ? abs2 - 1 : 0; 1965263508Sdim SDValue SRL = DAG.getNode(ISD::SRL, SDLoc(N), VT, SGN, 1966193323Sed DAG.getConstant(VT.getSizeInBits() - lg2, 1967219077Sdim getShiftAmountTy(SGN.getValueType()))); 1968263508Sdim SDValue ADD = DAG.getNode(ISD::ADD, SDLoc(N), VT, N0, SRL); 1969193323Sed AddToWorkList(SRL.getNode()); 1970193323Sed AddToWorkList(ADD.getNode()); // Divide by pow2 1971263508Sdim SDValue SRA = DAG.getNode(ISD::SRA, SDLoc(N), VT, ADD, 1972219077Sdim DAG.getConstant(lg2, getShiftAmountTy(ADD.getValueType()))); 1973193323Sed 1974193323Sed // If we're dividing by a positive value, we're done. Otherwise, we must 1975193323Sed // negate the result. 1976234353Sdim if (N1C->getAPIntValue().isNonNegative()) 1977193323Sed return SRA; 1978193323Sed 1979193323Sed AddToWorkList(SRA.getNode()); 1980263508Sdim return DAG.getNode(ISD::SUB, SDLoc(N), VT, 1981193323Sed DAG.getConstant(0, VT), SRA); 1982193323Sed } 1983193323Sed 1984193323Sed // if integer divide is expensive and we satisfy the requirements, emit an 1985193323Sed // alternate sequence. 1986234353Sdim if (N1C && !N1C->isNullValue() && !TLI.isIntDivCheap()) { 1987193323Sed SDValue Op = BuildSDIV(N); 1988193323Sed if (Op.getNode()) return Op; 1989193323Sed } 1990193323Sed 1991193323Sed // undef / X -> 0 1992193323Sed if (N0.getOpcode() == ISD::UNDEF) 1993193323Sed return DAG.getConstant(0, VT); 1994193323Sed // X / undef -> undef 1995193323Sed if (N1.getOpcode() == ISD::UNDEF) 1996193323Sed return N1; 1997193323Sed 1998193323Sed return SDValue(); 1999193323Sed} 2000193323Sed 2001193323SedSDValue DAGCombiner::visitUDIV(SDNode *N) { 2002193323Sed SDValue N0 = N->getOperand(0); 2003193323Sed SDValue N1 = N->getOperand(1); 2004193323Sed ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0.getNode()); 2005193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode()); 2006198090Srdivacky EVT VT = N->getValueType(0); 2007193323Sed 2008193323Sed // fold vector ops 2009193323Sed if (VT.isVector()) { 2010193323Sed SDValue FoldedVOp = SimplifyVBinOp(N); 2011193323Sed if (FoldedVOp.getNode()) return FoldedVOp; 2012193323Sed } 2013193323Sed 2014193323Sed // fold (udiv c1, c2) -> c1/c2 2015193323Sed if (N0C && N1C && !N1C->isNullValue()) 2016193323Sed return DAG.FoldConstantArithmetic(ISD::UDIV, VT, N0C, N1C); 2017193323Sed // fold (udiv x, (1 << c)) -> x >>u c 2018193323Sed if (N1C && N1C->getAPIntValue().isPowerOf2()) 2019263508Sdim return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, 2020193323Sed DAG.getConstant(N1C->getAPIntValue().logBase2(), 2021219077Sdim getShiftAmountTy(N0.getValueType()))); 2022193323Sed // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2 2023193323Sed if (N1.getOpcode() == ISD::SHL) { 2024193323Sed if (ConstantSDNode *SHC = dyn_cast<ConstantSDNode>(N1.getOperand(0))) { 2025193323Sed if (SHC->getAPIntValue().isPowerOf2()) { 2026198090Srdivacky EVT ADDVT = N1.getOperand(1).getValueType(); 2027263508Sdim SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N), ADDVT, 2028193323Sed N1.getOperand(1), 2029193323Sed DAG.getConstant(SHC->getAPIntValue() 2030193323Sed .logBase2(), 2031193323Sed ADDVT)); 2032193323Sed AddToWorkList(Add.getNode()); 2033263508Sdim return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, Add); 2034193323Sed } 2035193323Sed } 2036193323Sed } 2037193323Sed // fold (udiv x, c) -> alternate 2038193323Sed if (N1C && !N1C->isNullValue() && !TLI.isIntDivCheap()) { 2039193323Sed SDValue Op = BuildUDIV(N); 2040193323Sed if (Op.getNode()) return Op; 2041193323Sed } 2042193323Sed 2043193323Sed // undef / X -> 0 2044193323Sed if (N0.getOpcode() == ISD::UNDEF) 2045193323Sed return DAG.getConstant(0, VT); 2046193323Sed // X / undef -> undef 2047193323Sed if (N1.getOpcode() == ISD::UNDEF) 2048193323Sed return N1; 2049193323Sed 2050193323Sed return SDValue(); 2051193323Sed} 2052193323Sed 2053193323SedSDValue DAGCombiner::visitSREM(SDNode *N) { 2054193323Sed SDValue N0 = N->getOperand(0); 2055193323Sed SDValue N1 = N->getOperand(1); 2056193323Sed ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0); 2057193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 2058198090Srdivacky EVT VT = N->getValueType(0); 2059193323Sed 2060193323Sed // fold (srem c1, c2) -> c1%c2 2061193323Sed if (N0C && N1C && !N1C->isNullValue()) 2062193323Sed return DAG.FoldConstantArithmetic(ISD::SREM, VT, N0C, N1C); 2063193323Sed // If we know the sign bits of both operands are zero, strength reduce to a 2064193323Sed // urem instead. Handles (X & 0x0FFFFFFF) %s 16 -> X&15 2065193323Sed if (!VT.isVector()) { 2066193323Sed if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0)) 2067263508Sdim return DAG.getNode(ISD::UREM, SDLoc(N), VT, N0, N1); 2068193323Sed } 2069193323Sed 2070193323Sed // If X/C can be simplified by the division-by-constant logic, lower 2071193323Sed // X%C to the equivalent of X-X/C*C. 2072193323Sed if (N1C && !N1C->isNullValue()) { 2073263508Sdim SDValue Div = DAG.getNode(ISD::SDIV, SDLoc(N), VT, N0, N1); 2074193323Sed AddToWorkList(Div.getNode()); 2075193323Sed SDValue OptimizedDiv = combine(Div.getNode()); 2076193323Sed if (OptimizedDiv.getNode() && OptimizedDiv.getNode() != Div.getNode()) { 2077263508Sdim SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT, 2078193323Sed OptimizedDiv, N1); 2079263508Sdim SDValue Sub = DAG.getNode(ISD::SUB, SDLoc(N), VT, N0, Mul); 2080193323Sed AddToWorkList(Mul.getNode()); 2081193323Sed return Sub; 2082193323Sed } 2083193323Sed } 2084193323Sed 2085193323Sed // undef % X -> 0 2086193323Sed if (N0.getOpcode() == ISD::UNDEF) 2087193323Sed return DAG.getConstant(0, VT); 2088193323Sed // X % undef -> undef 2089193323Sed if (N1.getOpcode() == ISD::UNDEF) 2090193323Sed return N1; 2091193323Sed 2092193323Sed return SDValue(); 2093193323Sed} 2094193323Sed 2095193323SedSDValue DAGCombiner::visitUREM(SDNode *N) { 2096193323Sed SDValue N0 = N->getOperand(0); 2097193323Sed SDValue N1 = N->getOperand(1); 2098193323Sed ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0); 2099193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 2100198090Srdivacky EVT VT = N->getValueType(0); 2101193323Sed 2102193323Sed // fold (urem c1, c2) -> c1%c2 2103193323Sed if (N0C && N1C && !N1C->isNullValue()) 2104193323Sed return DAG.FoldConstantArithmetic(ISD::UREM, VT, N0C, N1C); 2105193323Sed // fold (urem x, pow2) -> (and x, pow2-1) 2106193323Sed if (N1C && !N1C->isNullValue() && N1C->getAPIntValue().isPowerOf2()) 2107263508Sdim return DAG.getNode(ISD::AND, SDLoc(N), VT, N0, 2108193323Sed DAG.getConstant(N1C->getAPIntValue()-1,VT)); 2109193323Sed // fold (urem x, (shl pow2, y)) -> (and x, (add (shl pow2, y), -1)) 2110193323Sed if (N1.getOpcode() == ISD::SHL) { 2111193323Sed if (ConstantSDNode *SHC = dyn_cast<ConstantSDNode>(N1.getOperand(0))) { 2112193323Sed if (SHC->getAPIntValue().isPowerOf2()) { 2113193323Sed SDValue Add = 2114263508Sdim DAG.getNode(ISD::ADD, SDLoc(N), VT, N1, 2115193323Sed DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), 2116193323Sed VT)); 2117193323Sed AddToWorkList(Add.getNode()); 2118263508Sdim return DAG.getNode(ISD::AND, SDLoc(N), VT, N0, Add); 2119193323Sed } 2120193323Sed } 2121193323Sed } 2122193323Sed 2123193323Sed // If X/C can be simplified by the division-by-constant logic, lower 2124193323Sed // X%C to the equivalent of X-X/C*C. 2125193323Sed if (N1C && !N1C->isNullValue()) { 2126263508Sdim SDValue Div = DAG.getNode(ISD::UDIV, SDLoc(N), VT, N0, N1); 2127193323Sed AddToWorkList(Div.getNode()); 2128193323Sed SDValue OptimizedDiv = combine(Div.getNode()); 2129193323Sed if (OptimizedDiv.getNode() && OptimizedDiv.getNode() != Div.getNode()) { 2130263508Sdim SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT, 2131193323Sed OptimizedDiv, N1); 2132263508Sdim SDValue Sub = DAG.getNode(ISD::SUB, SDLoc(N), VT, N0, Mul); 2133193323Sed AddToWorkList(Mul.getNode()); 2134193323Sed return Sub; 2135193323Sed } 2136193323Sed } 2137193323Sed 2138193323Sed // undef % X -> 0 2139193323Sed if (N0.getOpcode() == ISD::UNDEF) 2140193323Sed return DAG.getConstant(0, VT); 2141193323Sed // X % undef -> undef 2142193323Sed if (N1.getOpcode() == ISD::UNDEF) 2143193323Sed return N1; 2144193323Sed 2145193323Sed return SDValue(); 2146193323Sed} 2147193323Sed 2148193323SedSDValue DAGCombiner::visitMULHS(SDNode *N) { 2149193323Sed SDValue N0 = N->getOperand(0); 2150193323Sed SDValue N1 = N->getOperand(1); 2151193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 2152198090Srdivacky EVT VT = N->getValueType(0); 2153263508Sdim SDLoc DL(N); 2154193323Sed 2155193323Sed // fold (mulhs x, 0) -> 0 2156193323Sed if (N1C && N1C->isNullValue()) 2157193323Sed return N1; 2158193323Sed // fold (mulhs x, 1) -> (sra x, size(x)-1) 2159193323Sed if (N1C && N1C->getAPIntValue() == 1) 2160263508Sdim return DAG.getNode(ISD::SRA, SDLoc(N), N0.getValueType(), N0, 2161193323Sed DAG.getConstant(N0.getValueType().getSizeInBits() - 1, 2162219077Sdim getShiftAmountTy(N0.getValueType()))); 2163193323Sed // fold (mulhs x, undef) -> 0 2164193323Sed if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) 2165193323Sed return DAG.getConstant(0, VT); 2166193323Sed 2167218893Sdim // If the type twice as wide is legal, transform the mulhs to a wider multiply 2168218893Sdim // plus a shift. 2169218893Sdim if (VT.isSimple() && !VT.isVector()) { 2170218893Sdim MVT Simple = VT.getSimpleVT(); 2171218893Sdim unsigned SimpleSize = Simple.getSizeInBits(); 2172218893Sdim EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2); 2173218893Sdim if (TLI.isOperationLegal(ISD::MUL, NewVT)) { 2174218893Sdim N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0); 2175218893Sdim N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1); 2176218893Sdim N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1); 2177218893Sdim N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1, 2178219077Sdim DAG.getConstant(SimpleSize, getShiftAmountTy(N1.getValueType()))); 2179218893Sdim return DAG.getNode(ISD::TRUNCATE, DL, VT, N1); 2180218893Sdim } 2181218893Sdim } 2182219077Sdim 2183193323Sed return SDValue(); 2184193323Sed} 2185193323Sed 2186193323SedSDValue DAGCombiner::visitMULHU(SDNode *N) { 2187193323Sed SDValue N0 = N->getOperand(0); 2188193323Sed SDValue N1 = N->getOperand(1); 2189193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 2190198090Srdivacky EVT VT = N->getValueType(0); 2191263508Sdim SDLoc DL(N); 2192193323Sed 2193193323Sed // fold (mulhu x, 0) -> 0 2194193323Sed if (N1C && N1C->isNullValue()) 2195193323Sed return N1; 2196193323Sed // fold (mulhu x, 1) -> 0 2197193323Sed if (N1C && N1C->getAPIntValue() == 1) 2198193323Sed return DAG.getConstant(0, N0.getValueType()); 2199193323Sed // fold (mulhu x, undef) -> 0 2200193323Sed if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) 2201193323Sed return DAG.getConstant(0, VT); 2202193323Sed 2203218893Sdim // If the type twice as wide is legal, transform the mulhu to a wider multiply 2204218893Sdim // plus a shift. 2205218893Sdim if (VT.isSimple() && !VT.isVector()) { 2206218893Sdim MVT Simple = VT.getSimpleVT(); 2207218893Sdim unsigned SimpleSize = Simple.getSizeInBits(); 2208218893Sdim EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2); 2209218893Sdim if (TLI.isOperationLegal(ISD::MUL, NewVT)) { 2210218893Sdim N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0); 2211218893Sdim N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1); 2212218893Sdim N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1); 2213218893Sdim N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1, 2214219077Sdim DAG.getConstant(SimpleSize, getShiftAmountTy(N1.getValueType()))); 2215218893Sdim return DAG.getNode(ISD::TRUNCATE, DL, VT, N1); 2216218893Sdim } 2217218893Sdim } 2218219077Sdim 2219193323Sed return SDValue(); 2220193323Sed} 2221193323Sed 2222193323Sed/// SimplifyNodeWithTwoResults - Perform optimizations common to nodes that 2223193323Sed/// compute two values. LoOp and HiOp give the opcodes for the two computations 2224193323Sed/// that are being performed. Return true if a simplification was made. 2225193323Sed/// 2226193323SedSDValue DAGCombiner::SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp, 2227193323Sed unsigned HiOp) { 2228193323Sed // If the high half is not needed, just compute the low half. 2229193323Sed bool HiExists = N->hasAnyUseOfValue(1); 2230193323Sed if (!HiExists && 2231193323Sed (!LegalOperations || 2232193323Sed TLI.isOperationLegal(LoOp, N->getValueType(0)))) { 2233263508Sdim SDValue Res = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), 2234193323Sed N->op_begin(), N->getNumOperands()); 2235193323Sed return CombineTo(N, Res, Res); 2236193323Sed } 2237193323Sed 2238193323Sed // If the low half is not needed, just compute the high half. 2239193323Sed bool LoExists = N->hasAnyUseOfValue(0); 2240193323Sed if (!LoExists && 2241193323Sed (!LegalOperations || 2242193323Sed TLI.isOperationLegal(HiOp, N->getValueType(1)))) { 2243263508Sdim SDValue Res = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), 2244193323Sed N->op_begin(), N->getNumOperands()); 2245193323Sed return CombineTo(N, Res, Res); 2246193323Sed } 2247193323Sed 2248193323Sed // If both halves are used, return as it is. 2249193323Sed if (LoExists && HiExists) 2250193323Sed return SDValue(); 2251193323Sed 2252193323Sed // If the two computed results can be simplified separately, separate them. 2253193323Sed if (LoExists) { 2254263508Sdim SDValue Lo = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), 2255193323Sed N->op_begin(), N->getNumOperands()); 2256193323Sed AddToWorkList(Lo.getNode()); 2257193323Sed SDValue LoOpt = combine(Lo.getNode()); 2258193323Sed if (LoOpt.getNode() && LoOpt.getNode() != Lo.getNode() && 2259193323Sed (!LegalOperations || 2260193323Sed TLI.isOperationLegal(LoOpt.getOpcode(), LoOpt.getValueType()))) 2261193323Sed return CombineTo(N, LoOpt, LoOpt); 2262193323Sed } 2263193323Sed 2264193323Sed if (HiExists) { 2265263508Sdim SDValue Hi = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), 2266193323Sed N->op_begin(), N->getNumOperands()); 2267193323Sed AddToWorkList(Hi.getNode()); 2268193323Sed SDValue HiOpt = combine(Hi.getNode()); 2269193323Sed if (HiOpt.getNode() && HiOpt != Hi && 2270193323Sed (!LegalOperations || 2271193323Sed TLI.isOperationLegal(HiOpt.getOpcode(), HiOpt.getValueType()))) 2272193323Sed return CombineTo(N, HiOpt, HiOpt); 2273193323Sed } 2274193323Sed 2275193323Sed return SDValue(); 2276193323Sed} 2277193323Sed 2278193323SedSDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) { 2279193323Sed SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS); 2280193323Sed if (Res.getNode()) return Res; 2281193323Sed 2282218893Sdim EVT VT = N->getValueType(0); 2283263508Sdim SDLoc DL(N); 2284218893Sdim 2285218893Sdim // If the type twice as wide is legal, transform the mulhu to a wider multiply 2286218893Sdim // plus a shift. 2287218893Sdim if (VT.isSimple() && !VT.isVector()) { 2288218893Sdim MVT Simple = VT.getSimpleVT(); 2289218893Sdim unsigned SimpleSize = Simple.getSizeInBits(); 2290218893Sdim EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2); 2291218893Sdim if (TLI.isOperationLegal(ISD::MUL, NewVT)) { 2292218893Sdim SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(0)); 2293218893Sdim SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(1)); 2294218893Sdim Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi); 2295218893Sdim // Compute the high part as N1. 2296218893Sdim Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo, 2297219077Sdim DAG.getConstant(SimpleSize, getShiftAmountTy(Lo.getValueType()))); 2298218893Sdim Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi); 2299218893Sdim // Compute the low part as N0. 2300218893Sdim Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo); 2301218893Sdim return CombineTo(N, Lo, Hi); 2302218893Sdim } 2303218893Sdim } 2304219077Sdim 2305193323Sed return SDValue(); 2306193323Sed} 2307193323Sed 2308193323SedSDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) { 2309193323Sed SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU); 2310193323Sed if (Res.getNode()) return Res; 2311193323Sed 2312218893Sdim EVT VT = N->getValueType(0); 2313263508Sdim SDLoc DL(N); 2314219077Sdim 2315218893Sdim // If the type twice as wide is legal, transform the mulhu to a wider multiply 2316218893Sdim // plus a shift. 2317218893Sdim if (VT.isSimple() && !VT.isVector()) { 2318218893Sdim MVT Simple = VT.getSimpleVT(); 2319218893Sdim unsigned SimpleSize = Simple.getSizeInBits(); 2320218893Sdim EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2); 2321218893Sdim if (TLI.isOperationLegal(ISD::MUL, NewVT)) { 2322218893Sdim SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(0)); 2323218893Sdim SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(1)); 2324218893Sdim Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi); 2325218893Sdim // Compute the high part as N1. 2326218893Sdim Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo, 2327219077Sdim DAG.getConstant(SimpleSize, getShiftAmountTy(Lo.getValueType()))); 2328218893Sdim Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi); 2329218893Sdim // Compute the low part as N0. 2330218893Sdim Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo); 2331218893Sdim return CombineTo(N, Lo, Hi); 2332218893Sdim } 2333218893Sdim } 2334219077Sdim 2335193323Sed return SDValue(); 2336193323Sed} 2337193323Sed 2338223017SdimSDValue DAGCombiner::visitSMULO(SDNode *N) { 2339223017Sdim // (smulo x, 2) -> (saddo x, x) 2340223017Sdim if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1))) 2341223017Sdim if (C2->getAPIntValue() == 2) 2342263508Sdim return DAG.getNode(ISD::SADDO, SDLoc(N), N->getVTList(), 2343223017Sdim N->getOperand(0), N->getOperand(0)); 2344223017Sdim 2345223017Sdim return SDValue(); 2346223017Sdim} 2347223017Sdim 2348223017SdimSDValue DAGCombiner::visitUMULO(SDNode *N) { 2349223017Sdim // (umulo x, 2) -> (uaddo x, x) 2350223017Sdim if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1))) 2351223017Sdim if (C2->getAPIntValue() == 2) 2352263508Sdim return DAG.getNode(ISD::UADDO, SDLoc(N), N->getVTList(), 2353223017Sdim N->getOperand(0), N->getOperand(0)); 2354223017Sdim 2355223017Sdim return SDValue(); 2356223017Sdim} 2357223017Sdim 2358193323SedSDValue DAGCombiner::visitSDIVREM(SDNode *N) { 2359193323Sed SDValue Res = SimplifyNodeWithTwoResults(N, ISD::SDIV, ISD::SREM); 2360193323Sed if (Res.getNode()) return Res; 2361193323Sed 2362193323Sed return SDValue(); 2363193323Sed} 2364193323Sed 2365193323SedSDValue DAGCombiner::visitUDIVREM(SDNode *N) { 2366193323Sed SDValue Res = SimplifyNodeWithTwoResults(N, ISD::UDIV, ISD::UREM); 2367193323Sed if (Res.getNode()) return Res; 2368193323Sed 2369193323Sed return SDValue(); 2370193323Sed} 2371193323Sed 2372193323Sed/// SimplifyBinOpWithSameOpcodeHands - If this is a binary operator with 2373193323Sed/// two operands of the same opcode, try to simplify it. 2374193323SedSDValue DAGCombiner::SimplifyBinOpWithSameOpcodeHands(SDNode *N) { 2375193323Sed SDValue N0 = N->getOperand(0), N1 = N->getOperand(1); 2376198090Srdivacky EVT VT = N0.getValueType(); 2377193323Sed assert(N0.getOpcode() == N1.getOpcode() && "Bad input!"); 2378193323Sed 2379202375Srdivacky // Bail early if none of these transforms apply. 2380202375Srdivacky if (N0.getNode()->getNumOperands() == 0) return SDValue(); 2381202375Srdivacky 2382193323Sed // For each of OP in AND/OR/XOR: 2383193323Sed // fold (OP (zext x), (zext y)) -> (zext (OP x, y)) 2384193323Sed // fold (OP (sext x), (sext y)) -> (sext (OP x, y)) 2385193323Sed // fold (OP (aext x), (aext y)) -> (aext (OP x, y)) 2386210299Sed // fold (OP (trunc x), (trunc y)) -> (trunc (OP x, y)) (if trunc isn't free) 2387200581Srdivacky // 2388200581Srdivacky // do not sink logical op inside of a vector extend, since it may combine 2389200581Srdivacky // into a vsetcc. 2390202375Srdivacky EVT Op0VT = N0.getOperand(0).getValueType(); 2391202375Srdivacky if ((N0.getOpcode() == ISD::ZERO_EXTEND || 2392193323Sed N0.getOpcode() == ISD::SIGN_EXTEND || 2393207618Srdivacky // Avoid infinite looping with PromoteIntBinOp. 2394207618Srdivacky (N0.getOpcode() == ISD::ANY_EXTEND && 2395207618Srdivacky (!LegalTypes || TLI.isTypeDesirableForOp(N->getOpcode(), Op0VT))) || 2396210299Sed (N0.getOpcode() == ISD::TRUNCATE && 2397210299Sed (!TLI.isZExtFree(VT, Op0VT) || 2398210299Sed !TLI.isTruncateFree(Op0VT, VT)) && 2399210299Sed TLI.isTypeLegal(Op0VT))) && 2400200581Srdivacky !VT.isVector() && 2401202375Srdivacky Op0VT == N1.getOperand(0).getValueType() && 2402202375Srdivacky (!LegalOperations || TLI.isOperationLegal(N->getOpcode(), Op0VT))) { 2403263508Sdim SDValue ORNode = DAG.getNode(N->getOpcode(), SDLoc(N0), 2404193323Sed N0.getOperand(0).getValueType(), 2405193323Sed N0.getOperand(0), N1.getOperand(0)); 2406193323Sed AddToWorkList(ORNode.getNode()); 2407263508Sdim return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, ORNode); 2408193323Sed } 2409193323Sed 2410193323Sed // For each of OP in SHL/SRL/SRA/AND... 2411193323Sed // fold (and (OP x, z), (OP y, z)) -> (OP (and x, y), z) 2412193323Sed // fold (or (OP x, z), (OP y, z)) -> (OP (or x, y), z) 2413193323Sed // fold (xor (OP x, z), (OP y, z)) -> (OP (xor x, y), z) 2414193323Sed if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL || 2415193323Sed N0.getOpcode() == ISD::SRA || N0.getOpcode() == ISD::AND) && 2416193323Sed N0.getOperand(1) == N1.getOperand(1)) { 2417263508Sdim SDValue ORNode = DAG.getNode(N->getOpcode(), SDLoc(N0), 2418193323Sed N0.getOperand(0).getValueType(), 2419193323Sed N0.getOperand(0), N1.getOperand(0)); 2420193323Sed AddToWorkList(ORNode.getNode()); 2421263508Sdim return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, 2422193323Sed ORNode, N0.getOperand(1)); 2423193323Sed } 2424193323Sed 2425234353Sdim // Simplify xor/and/or (bitcast(A), bitcast(B)) -> bitcast(op (A,B)) 2426234353Sdim // Only perform this optimization after type legalization and before 2427234353Sdim // LegalizeVectorOprs. LegalizeVectorOprs promotes vector operations by 2428234353Sdim // adding bitcasts. For example (xor v4i32) is promoted to (v2i64), and 2429234353Sdim // we don't want to undo this promotion. 2430234353Sdim // We also handle SCALAR_TO_VECTOR because xor/or/and operations are cheaper 2431234353Sdim // on scalars. 2432243830Sdim if ((N0.getOpcode() == ISD::BITCAST || 2433243830Sdim N0.getOpcode() == ISD::SCALAR_TO_VECTOR) && 2434243830Sdim Level == AfterLegalizeTypes) { 2435234353Sdim SDValue In0 = N0.getOperand(0); 2436234353Sdim SDValue In1 = N1.getOperand(0); 2437234353Sdim EVT In0Ty = In0.getValueType(); 2438234353Sdim EVT In1Ty = In1.getValueType(); 2439263508Sdim SDLoc DL(N); 2440243830Sdim // If both incoming values are integers, and the original types are the 2441243830Sdim // same. 2442234353Sdim if (In0Ty.isInteger() && In1Ty.isInteger() && In0Ty == In1Ty) { 2443243830Sdim SDValue Op = DAG.getNode(N->getOpcode(), DL, In0Ty, In0, In1); 2444243830Sdim SDValue BC = DAG.getNode(N0.getOpcode(), DL, VT, Op); 2445234353Sdim AddToWorkList(Op.getNode()); 2446234353Sdim return BC; 2447234353Sdim } 2448234353Sdim } 2449234353Sdim 2450234353Sdim // Xor/and/or are indifferent to the swizzle operation (shuffle of one value). 2451234353Sdim // Simplify xor/and/or (shuff(A), shuff(B)) -> shuff(op (A,B)) 2452234353Sdim // If both shuffles use the same mask, and both shuffle within a single 2453234353Sdim // vector, then it is worthwhile to move the swizzle after the operation. 2454234353Sdim // The type-legalizer generates this pattern when loading illegal 2455234353Sdim // vector types from memory. In many cases this allows additional shuffle 2456234353Sdim // optimizations. 2457234353Sdim if (N0.getOpcode() == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG && 2458234353Sdim N0.getOperand(1).getOpcode() == ISD::UNDEF && 2459234353Sdim N1.getOperand(1).getOpcode() == ISD::UNDEF) { 2460234353Sdim ShuffleVectorSDNode *SVN0 = cast<ShuffleVectorSDNode>(N0); 2461234353Sdim ShuffleVectorSDNode *SVN1 = cast<ShuffleVectorSDNode>(N1); 2462234353Sdim 2463234353Sdim assert(N0.getOperand(0).getValueType() == N1.getOperand(1).getValueType() && 2464234353Sdim "Inputs to shuffles are not the same type"); 2465234353Sdim 2466234353Sdim unsigned NumElts = VT.getVectorNumElements(); 2467234353Sdim 2468234353Sdim // Check that both shuffles use the same mask. The masks are known to be of 2469234353Sdim // the same length because the result vector type is the same. 2470234353Sdim bool SameMask = true; 2471234353Sdim for (unsigned i = 0; i != NumElts; ++i) { 2472234353Sdim int Idx0 = SVN0->getMaskElt(i); 2473234353Sdim int Idx1 = SVN1->getMaskElt(i); 2474234353Sdim if (Idx0 != Idx1) { 2475234353Sdim SameMask = false; 2476234353Sdim break; 2477234353Sdim } 2478234353Sdim } 2479234353Sdim 2480234353Sdim if (SameMask) { 2481263508Sdim SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N), VT, 2482234353Sdim N0.getOperand(0), N1.getOperand(0)); 2483234353Sdim AddToWorkList(Op.getNode()); 2484263508Sdim return DAG.getVectorShuffle(VT, SDLoc(N), Op, 2485234353Sdim DAG.getUNDEF(VT), &SVN0->getMask()[0]); 2486234353Sdim } 2487234353Sdim } 2488234353Sdim 2489193323Sed return SDValue(); 2490193323Sed} 2491193323Sed 2492193323SedSDValue DAGCombiner::visitAND(SDNode *N) { 2493193323Sed SDValue N0 = N->getOperand(0); 2494193323Sed SDValue N1 = N->getOperand(1); 2495193323Sed SDValue LL, LR, RL, RR, CC0, CC1; 2496193323Sed ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0); 2497193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 2498198090Srdivacky EVT VT = N1.getValueType(); 2499204792Srdivacky unsigned BitWidth = VT.getScalarType().getSizeInBits(); 2500193323Sed 2501193323Sed // fold vector ops 2502193323Sed if (VT.isVector()) { 2503193323Sed SDValue FoldedVOp = SimplifyVBinOp(N); 2504193323Sed if (FoldedVOp.getNode()) return FoldedVOp; 2505249423Sdim 2506249423Sdim // fold (and x, 0) -> 0, vector edition 2507249423Sdim if (ISD::isBuildVectorAllZeros(N0.getNode())) 2508249423Sdim return N0; 2509249423Sdim if (ISD::isBuildVectorAllZeros(N1.getNode())) 2510249423Sdim return N1; 2511249423Sdim 2512249423Sdim // fold (and x, -1) -> x, vector edition 2513249423Sdim if (ISD::isBuildVectorAllOnes(N0.getNode())) 2514249423Sdim return N1; 2515249423Sdim if (ISD::isBuildVectorAllOnes(N1.getNode())) 2516249423Sdim return N0; 2517193323Sed } 2518193323Sed 2519193323Sed // fold (and x, undef) -> 0 2520193323Sed if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) 2521193323Sed return DAG.getConstant(0, VT); 2522193323Sed // fold (and c1, c2) -> c1&c2 2523193323Sed if (N0C && N1C) 2524193323Sed return DAG.FoldConstantArithmetic(ISD::AND, VT, N0C, N1C); 2525193323Sed // canonicalize constant to RHS 2526193323Sed if (N0C && !N1C) 2527263508Sdim return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0); 2528193323Sed // fold (and x, -1) -> x 2529193323Sed if (N1C && N1C->isAllOnesValue()) 2530193323Sed return N0; 2531193323Sed // if (and x, c) is known to be zero, return 0 2532193323Sed if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0), 2533193323Sed APInt::getAllOnesValue(BitWidth))) 2534193323Sed return DAG.getConstant(0, VT); 2535193323Sed // reassociate and 2536263508Sdim SDValue RAND = ReassociateOps(ISD::AND, SDLoc(N), N0, N1); 2537193323Sed if (RAND.getNode() != 0) 2538193323Sed return RAND; 2539204642Srdivacky // fold (and (or x, C), D) -> D if (C & D) == D 2540193323Sed if (N1C && N0.getOpcode() == ISD::OR) 2541193323Sed if (ConstantSDNode *ORI = dyn_cast<ConstantSDNode>(N0.getOperand(1))) 2542193323Sed if ((ORI->getAPIntValue() & N1C->getAPIntValue()) == N1C->getAPIntValue()) 2543193323Sed return N1; 2544193323Sed // fold (and (any_ext V), c) -> (zero_ext V) if 'and' only clears top bits. 2545193323Sed if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) { 2546193323Sed SDValue N0Op0 = N0.getOperand(0); 2547193323Sed APInt Mask = ~N1C->getAPIntValue(); 2548218893Sdim Mask = Mask.trunc(N0Op0.getValueSizeInBits()); 2549193323Sed if (DAG.MaskedValueIsZero(N0Op0, Mask)) { 2550263508Sdim SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), 2551193323Sed N0.getValueType(), N0Op0); 2552193323Sed 2553193323Sed // Replace uses of the AND with uses of the Zero extend node. 2554193323Sed CombineTo(N, Zext); 2555193323Sed 2556193323Sed // We actually want to replace all uses of the any_extend with the 2557193323Sed // zero_extend, to avoid duplicating things. This will later cause this 2558193323Sed // AND to be folded. 2559193323Sed CombineTo(N0.getNode(), Zext); 2560193323Sed return SDValue(N, 0); // Return N so it doesn't get rechecked! 2561193323Sed } 2562193323Sed } 2563263508Sdim // similarly fold (and (X (load ([non_ext|any_ext|zero_ext] V))), c) -> 2564234353Sdim // (X (load ([non_ext|zero_ext] V))) if 'and' only clears top bits which must 2565234353Sdim // already be zero by virtue of the width of the base type of the load. 2566234353Sdim // 2567234353Sdim // the 'X' node here can either be nothing or an extract_vector_elt to catch 2568234353Sdim // more cases. 2569234353Sdim if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT && 2570234353Sdim N0.getOperand(0).getOpcode() == ISD::LOAD) || 2571234353Sdim N0.getOpcode() == ISD::LOAD) { 2572234353Sdim LoadSDNode *Load = cast<LoadSDNode>( (N0.getOpcode() == ISD::LOAD) ? 2573234353Sdim N0 : N0.getOperand(0) ); 2574234353Sdim 2575234353Sdim // Get the constant (if applicable) the zero'th operand is being ANDed with. 2576234353Sdim // This can be a pure constant or a vector splat, in which case we treat the 2577234353Sdim // vector as a scalar and use the splat value. 2578234353Sdim APInt Constant = APInt::getNullValue(1); 2579234353Sdim if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) { 2580234353Sdim Constant = C->getAPIntValue(); 2581234353Sdim } else if (BuildVectorSDNode *Vector = dyn_cast<BuildVectorSDNode>(N1)) { 2582234353Sdim APInt SplatValue, SplatUndef; 2583234353Sdim unsigned SplatBitSize; 2584234353Sdim bool HasAnyUndefs; 2585234353Sdim bool IsSplat = Vector->isConstantSplat(SplatValue, SplatUndef, 2586234353Sdim SplatBitSize, HasAnyUndefs); 2587234353Sdim if (IsSplat) { 2588234353Sdim // Undef bits can contribute to a possible optimisation if set, so 2589234353Sdim // set them. 2590234353Sdim SplatValue |= SplatUndef; 2591234353Sdim 2592234353Sdim // The splat value may be something like "0x00FFFFFF", which means 0 for 2593234353Sdim // the first vector value and FF for the rest, repeating. We need a mask 2594234353Sdim // that will apply equally to all members of the vector, so AND all the 2595234353Sdim // lanes of the constant together. 2596234353Sdim EVT VT = Vector->getValueType(0); 2597234353Sdim unsigned BitWidth = VT.getVectorElementType().getSizeInBits(); 2598243830Sdim 2599243830Sdim // If the splat value has been compressed to a bitlength lower 2600243830Sdim // than the size of the vector lane, we need to re-expand it to 2601243830Sdim // the lane size. 2602243830Sdim if (BitWidth > SplatBitSize) 2603243830Sdim for (SplatValue = SplatValue.zextOrTrunc(BitWidth); 2604243830Sdim SplatBitSize < BitWidth; 2605243830Sdim SplatBitSize = SplatBitSize * 2) 2606243830Sdim SplatValue |= SplatValue.shl(SplatBitSize); 2607243830Sdim 2608234353Sdim Constant = APInt::getAllOnesValue(BitWidth); 2609243830Sdim for (unsigned i = 0, n = SplatBitSize/BitWidth; i < n; ++i) 2610234353Sdim Constant &= SplatValue.lshr(i*BitWidth).zextOrTrunc(BitWidth); 2611234353Sdim } 2612234353Sdim } 2613234353Sdim 2614234353Sdim // If we want to change an EXTLOAD to a ZEXTLOAD, ensure a ZEXTLOAD is 2615234353Sdim // actually legal and isn't going to get expanded, else this is a false 2616234353Sdim // optimisation. 2617234353Sdim bool CanZextLoadProfitably = TLI.isLoadExtLegal(ISD::ZEXTLOAD, 2618234353Sdim Load->getMemoryVT()); 2619234353Sdim 2620234353Sdim // Resize the constant to the same size as the original memory access before 2621234353Sdim // extension. If it is still the AllOnesValue then this AND is completely 2622234353Sdim // unneeded. 2623234353Sdim Constant = 2624234353Sdim Constant.zextOrTrunc(Load->getMemoryVT().getScalarType().getSizeInBits()); 2625234353Sdim 2626234353Sdim bool B; 2627234353Sdim switch (Load->getExtensionType()) { 2628234353Sdim default: B = false; break; 2629234353Sdim case ISD::EXTLOAD: B = CanZextLoadProfitably; break; 2630234353Sdim case ISD::ZEXTLOAD: 2631234353Sdim case ISD::NON_EXTLOAD: B = true; break; 2632234353Sdim } 2633234353Sdim 2634234353Sdim if (B && Constant.isAllOnesValue()) { 2635234353Sdim // If the load type was an EXTLOAD, convert to ZEXTLOAD in order to 2636234353Sdim // preserve semantics once we get rid of the AND. 2637234353Sdim SDValue NewLoad(Load, 0); 2638234353Sdim if (Load->getExtensionType() == ISD::EXTLOAD) { 2639234353Sdim NewLoad = DAG.getLoad(Load->getAddressingMode(), ISD::ZEXTLOAD, 2640263508Sdim Load->getValueType(0), SDLoc(Load), 2641234353Sdim Load->getChain(), Load->getBasePtr(), 2642234353Sdim Load->getOffset(), Load->getMemoryVT(), 2643234353Sdim Load->getMemOperand()); 2644234353Sdim // Replace uses of the EXTLOAD with the new ZEXTLOAD. 2645239462Sdim if (Load->getNumValues() == 3) { 2646239462Sdim // PRE/POST_INC loads have 3 values. 2647239462Sdim SDValue To[] = { NewLoad.getValue(0), NewLoad.getValue(1), 2648239462Sdim NewLoad.getValue(2) }; 2649239462Sdim CombineTo(Load, To, 3, true); 2650239462Sdim } else { 2651239462Sdim CombineTo(Load, NewLoad.getValue(0), NewLoad.getValue(1)); 2652239462Sdim } 2653234353Sdim } 2654234353Sdim 2655234353Sdim // Fold the AND away, taking care not to fold to the old load node if we 2656234353Sdim // replaced it. 2657234353Sdim CombineTo(N, (N0.getNode() == Load) ? NewLoad : N0); 2658234353Sdim 2659234353Sdim return SDValue(N, 0); // Return N so it doesn't get rechecked! 2660234353Sdim } 2661234353Sdim } 2662193323Sed // fold (and (setcc x), (setcc y)) -> (setcc (and x, y)) 2663193323Sed if (isSetCCEquivalent(N0, LL, LR, CC0) && isSetCCEquivalent(N1, RL, RR, CC1)){ 2664193323Sed ISD::CondCode Op0 = cast<CondCodeSDNode>(CC0)->get(); 2665193323Sed ISD::CondCode Op1 = cast<CondCodeSDNode>(CC1)->get(); 2666193323Sed 2667193323Sed if (LR == RR && isa<ConstantSDNode>(LR) && Op0 == Op1 && 2668193323Sed LL.getValueType().isInteger()) { 2669193323Sed // fold (and (seteq X, 0), (seteq Y, 0)) -> (seteq (or X, Y), 0) 2670193323Sed if (cast<ConstantSDNode>(LR)->isNullValue() && Op1 == ISD::SETEQ) { 2671263508Sdim SDValue ORNode = DAG.getNode(ISD::OR, SDLoc(N0), 2672193323Sed LR.getValueType(), LL, RL); 2673193323Sed AddToWorkList(ORNode.getNode()); 2674263508Sdim return DAG.getSetCC(SDLoc(N), VT, ORNode, LR, Op1); 2675193323Sed } 2676193323Sed // fold (and (seteq X, -1), (seteq Y, -1)) -> (seteq (and X, Y), -1) 2677193323Sed if (cast<ConstantSDNode>(LR)->isAllOnesValue() && Op1 == ISD::SETEQ) { 2678263508Sdim SDValue ANDNode = DAG.getNode(ISD::AND, SDLoc(N0), 2679193323Sed LR.getValueType(), LL, RL); 2680193323Sed AddToWorkList(ANDNode.getNode()); 2681263508Sdim return DAG.getSetCC(SDLoc(N), VT, ANDNode, LR, Op1); 2682193323Sed } 2683193323Sed // fold (and (setgt X, -1), (setgt Y, -1)) -> (setgt (or X, Y), -1) 2684193323Sed if (cast<ConstantSDNode>(LR)->isAllOnesValue() && Op1 == ISD::SETGT) { 2685263508Sdim SDValue ORNode = DAG.getNode(ISD::OR, SDLoc(N0), 2686193323Sed LR.getValueType(), LL, RL); 2687193323Sed AddToWorkList(ORNode.getNode()); 2688263508Sdim return DAG.getSetCC(SDLoc(N), VT, ORNode, LR, Op1); 2689193323Sed } 2690193323Sed } 2691263508Sdim // Simplify (and (setne X, 0), (setne X, -1)) -> (setuge (add X, 1), 2) 2692263508Sdim if (LL == RL && isa<ConstantSDNode>(LR) && isa<ConstantSDNode>(RR) && 2693263508Sdim Op0 == Op1 && LL.getValueType().isInteger() && 2694263508Sdim Op0 == ISD::SETNE && ((cast<ConstantSDNode>(LR)->isNullValue() && 2695263508Sdim cast<ConstantSDNode>(RR)->isAllOnesValue()) || 2696263508Sdim (cast<ConstantSDNode>(LR)->isAllOnesValue() && 2697263508Sdim cast<ConstantSDNode>(RR)->isNullValue()))) { 2698263508Sdim SDValue ADDNode = DAG.getNode(ISD::ADD, SDLoc(N0), LL.getValueType(), 2699263508Sdim LL, DAG.getConstant(1, LL.getValueType())); 2700263508Sdim AddToWorkList(ADDNode.getNode()); 2701263508Sdim return DAG.getSetCC(SDLoc(N), VT, ADDNode, 2702263508Sdim DAG.getConstant(2, LL.getValueType()), ISD::SETUGE); 2703263508Sdim } 2704193323Sed // canonicalize equivalent to ll == rl 2705193323Sed if (LL == RR && LR == RL) { 2706193323Sed Op1 = ISD::getSetCCSwappedOperands(Op1); 2707193323Sed std::swap(RL, RR); 2708193323Sed } 2709193323Sed if (LL == RL && LR == RR) { 2710193323Sed bool isInteger = LL.getValueType().isInteger(); 2711193323Sed ISD::CondCode Result = ISD::getSetCCAndOperation(Op0, Op1, isInteger); 2712193323Sed if (Result != ISD::SETCC_INVALID && 2713249423Sdim (!LegalOperations || 2714249423Sdim (TLI.isCondCodeLegal(Result, LL.getSimpleValueType()) && 2715249423Sdim TLI.isOperationLegal(ISD::SETCC, 2716263508Sdim getSetCCResultType(N0.getSimpleValueType()))))) 2717263508Sdim return DAG.getSetCC(SDLoc(N), N0.getValueType(), 2718193323Sed LL, LR, Result); 2719193323Sed } 2720193323Sed } 2721193323Sed 2722193323Sed // Simplify: (and (op x...), (op y...)) -> (op (and x, y)) 2723193323Sed if (N0.getOpcode() == N1.getOpcode()) { 2724193323Sed SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N); 2725193323Sed if (Tmp.getNode()) return Tmp; 2726193323Sed } 2727193323Sed 2728193323Sed // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1) 2729193323Sed // fold (and (sra)) -> (and (srl)) when possible. 2730193323Sed if (!VT.isVector() && 2731193323Sed SimplifyDemandedBits(SDValue(N, 0))) 2732193323Sed return SDValue(N, 0); 2733202375Srdivacky 2734193323Sed // fold (zext_inreg (extload x)) -> (zextload x) 2735193323Sed if (ISD::isEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) { 2736193323Sed LoadSDNode *LN0 = cast<LoadSDNode>(N0); 2737198090Srdivacky EVT MemVT = LN0->getMemoryVT(); 2738193323Sed // If we zero all the possible extended bits, then we can turn this into 2739193323Sed // a zextload if we are running before legalize or the operation is legal. 2740204792Srdivacky unsigned BitWidth = N1.getValueType().getScalarType().getSizeInBits(); 2741193323Sed if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth, 2742204792Srdivacky BitWidth - MemVT.getScalarType().getSizeInBits())) && 2743193323Sed ((!LegalOperations && !LN0->isVolatile()) || 2744198090Srdivacky TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT))) { 2745263508Sdim SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, 2746193323Sed LN0->getChain(), LN0->getBasePtr(), 2747263508Sdim MemVT, LN0->getMemOperand()); 2748193323Sed AddToWorkList(N); 2749193323Sed CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1)); 2750193323Sed return SDValue(N, 0); // Return N so it doesn't get rechecked! 2751193323Sed } 2752193323Sed } 2753193323Sed // fold (zext_inreg (sextload x)) -> (zextload x) iff load has one use 2754193323Sed if (ISD::isSEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) && 2755193323Sed N0.hasOneUse()) { 2756193323Sed LoadSDNode *LN0 = cast<LoadSDNode>(N0); 2757198090Srdivacky EVT MemVT = LN0->getMemoryVT(); 2758193323Sed // If we zero all the possible extended bits, then we can turn this into 2759193323Sed // a zextload if we are running before legalize or the operation is legal. 2760204792Srdivacky unsigned BitWidth = N1.getValueType().getScalarType().getSizeInBits(); 2761193323Sed if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth, 2762204792Srdivacky BitWidth - MemVT.getScalarType().getSizeInBits())) && 2763193323Sed ((!LegalOperations && !LN0->isVolatile()) || 2764198090Srdivacky TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT))) { 2765263508Sdim SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, 2766263508Sdim LN0->getChain(), LN0->getBasePtr(), 2767263508Sdim MemVT, LN0->getMemOperand()); 2768193323Sed AddToWorkList(N); 2769193323Sed CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1)); 2770193323Sed return SDValue(N, 0); // Return N so it doesn't get rechecked! 2771193323Sed } 2772193323Sed } 2773193323Sed 2774193323Sed // fold (and (load x), 255) -> (zextload x, i8) 2775193323Sed // fold (and (extload x, i16), 255) -> (zextload x, i8) 2776202375Srdivacky // fold (and (any_ext (extload x, i16)), 255) -> (zextload x, i8) 2777202375Srdivacky if (N1C && (N0.getOpcode() == ISD::LOAD || 2778202375Srdivacky (N0.getOpcode() == ISD::ANY_EXTEND && 2779202375Srdivacky N0.getOperand(0).getOpcode() == ISD::LOAD))) { 2780202375Srdivacky bool HasAnyExt = N0.getOpcode() == ISD::ANY_EXTEND; 2781202375Srdivacky LoadSDNode *LN0 = HasAnyExt 2782202375Srdivacky ? cast<LoadSDNode>(N0.getOperand(0)) 2783202375Srdivacky : cast<LoadSDNode>(N0); 2784193323Sed if (LN0->getExtensionType() != ISD::SEXTLOAD && 2785263508Sdim LN0->isUnindexed() && N0.hasOneUse() && SDValue(LN0, 0).hasOneUse()) { 2786193323Sed uint32_t ActiveBits = N1C->getAPIntValue().getActiveBits(); 2787202375Srdivacky if (ActiveBits > 0 && APIntOps::isMask(ActiveBits, N1C->getAPIntValue())){ 2788202375Srdivacky EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits); 2789202375Srdivacky EVT LoadedVT = LN0->getMemoryVT(); 2790193323Sed 2791202375Srdivacky if (ExtVT == LoadedVT && 2792202375Srdivacky (!LegalOperations || TLI.isLoadExtLegal(ISD::ZEXTLOAD, ExtVT))) { 2793202375Srdivacky EVT LoadResultTy = HasAnyExt ? LN0->getValueType(0) : VT; 2794218893Sdim 2795218893Sdim SDValue NewLoad = 2796263508Sdim DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN0), LoadResultTy, 2797263508Sdim LN0->getChain(), LN0->getBasePtr(), ExtVT, 2798263508Sdim LN0->getMemOperand()); 2799202375Srdivacky AddToWorkList(N); 2800202375Srdivacky CombineTo(LN0, NewLoad, NewLoad.getValue(1)); 2801202375Srdivacky return SDValue(N, 0); // Return N so it doesn't get rechecked! 2802202375Srdivacky } 2803218893Sdim 2804202375Srdivacky // Do not change the width of a volatile load. 2805202375Srdivacky // Do not generate loads of non-round integer types since these can 2806202375Srdivacky // be expensive (and would be wrong if the type is not byte sized). 2807202375Srdivacky if (!LN0->isVolatile() && LoadedVT.bitsGT(ExtVT) && ExtVT.isRound() && 2808202375Srdivacky (!LegalOperations || TLI.isLoadExtLegal(ISD::ZEXTLOAD, ExtVT))) { 2809202375Srdivacky EVT PtrType = LN0->getOperand(1).getValueType(); 2810193323Sed 2811202375Srdivacky unsigned Alignment = LN0->getAlignment(); 2812202375Srdivacky SDValue NewPtr = LN0->getBasePtr(); 2813193323Sed 2814202375Srdivacky // For big endian targets, we need to add an offset to the pointer 2815202375Srdivacky // to load the correct bytes. For little endian systems, we merely 2816202375Srdivacky // need to read fewer bytes from the same pointer. 2817202375Srdivacky if (TLI.isBigEndian()) { 2818202375Srdivacky unsigned LVTStoreBytes = LoadedVT.getStoreSize(); 2819202375Srdivacky unsigned EVTStoreBytes = ExtVT.getStoreSize(); 2820202375Srdivacky unsigned PtrOff = LVTStoreBytes - EVTStoreBytes; 2821263508Sdim NewPtr = DAG.getNode(ISD::ADD, SDLoc(LN0), PtrType, 2822202375Srdivacky NewPtr, DAG.getConstant(PtrOff, PtrType)); 2823202375Srdivacky Alignment = MinAlign(Alignment, PtrOff); 2824202375Srdivacky } 2825193323Sed 2826202375Srdivacky AddToWorkList(NewPtr.getNode()); 2827218893Sdim 2828202375Srdivacky EVT LoadResultTy = HasAnyExt ? LN0->getValueType(0) : VT; 2829202375Srdivacky SDValue Load = 2830263508Sdim DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN0), LoadResultTy, 2831202375Srdivacky LN0->getChain(), NewPtr, 2832218893Sdim LN0->getPointerInfo(), 2833203954Srdivacky ExtVT, LN0->isVolatile(), LN0->isNonTemporal(), 2834263508Sdim Alignment, LN0->getTBAAInfo()); 2835202375Srdivacky AddToWorkList(N); 2836202375Srdivacky CombineTo(LN0, Load, Load.getValue(1)); 2837202375Srdivacky return SDValue(N, 0); // Return N so it doesn't get rechecked! 2838193323Sed } 2839193323Sed } 2840193323Sed } 2841193323Sed } 2842193323Sed 2843239462Sdim if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::SRL && 2844239462Sdim VT.getSizeInBits() <= 64) { 2845239462Sdim if (ConstantSDNode *ADDI = dyn_cast<ConstantSDNode>(N0.getOperand(1))) { 2846239462Sdim APInt ADDC = ADDI->getAPIntValue(); 2847239462Sdim if (!TLI.isLegalAddImmediate(ADDC.getSExtValue())) { 2848239462Sdim // Look for (and (add x, c1), (lshr y, c2)). If C1 wasn't a legal 2849239462Sdim // immediate for an add, but it is legal if its top c2 bits are set, 2850239462Sdim // transform the ADD so the immediate doesn't need to be materialized 2851239462Sdim // in a register. 2852239462Sdim if (ConstantSDNode *SRLI = dyn_cast<ConstantSDNode>(N1.getOperand(1))) { 2853239462Sdim APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(), 2854239462Sdim SRLI->getZExtValue()); 2855239462Sdim if (DAG.MaskedValueIsZero(N0.getOperand(1), Mask)) { 2856239462Sdim ADDC |= Mask; 2857239462Sdim if (TLI.isLegalAddImmediate(ADDC.getSExtValue())) { 2858239462Sdim SDValue NewAdd = 2859263508Sdim DAG.getNode(ISD::ADD, SDLoc(N0), VT, 2860239462Sdim N0.getOperand(0), DAG.getConstant(ADDC, VT)); 2861239462Sdim CombineTo(N0.getNode(), NewAdd); 2862239462Sdim return SDValue(N, 0); // Return N so it doesn't get rechecked! 2863239462Sdim } 2864239462Sdim } 2865239462Sdim } 2866239462Sdim } 2867239462Sdim } 2868239462Sdim } 2869239462Sdim 2870263508Sdim // fold (and (or (srl N, 8), (shl N, 8)), 0xffff) -> (srl (bswap N), const) 2871263508Sdim if (N1C && N1C->getAPIntValue() == 0xffff && N0.getOpcode() == ISD::OR) { 2872263508Sdim SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0), 2873263508Sdim N0.getOperand(1), false); 2874263508Sdim if (BSwap.getNode()) 2875263508Sdim return BSwap; 2876263508Sdim } 2877263508Sdim 2878193323Sed return SDValue(); 2879193323Sed} 2880193323Sed 2881224145Sdim/// MatchBSwapHWord - Match (a >> 8) | (a << 8) as (bswap a) >> 16 2882224145Sdim/// 2883224145SdimSDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1, 2884224145Sdim bool DemandHighBits) { 2885224145Sdim if (!LegalOperations) 2886224145Sdim return SDValue(); 2887224145Sdim 2888224145Sdim EVT VT = N->getValueType(0); 2889224145Sdim if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16) 2890224145Sdim return SDValue(); 2891224145Sdim if (!TLI.isOperationLegal(ISD::BSWAP, VT)) 2892224145Sdim return SDValue(); 2893224145Sdim 2894224145Sdim // Recognize (and (shl a, 8), 0xff), (and (srl a, 8), 0xff00) 2895224145Sdim bool LookPassAnd0 = false; 2896224145Sdim bool LookPassAnd1 = false; 2897224145Sdim if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::SRL) 2898224145Sdim std::swap(N0, N1); 2899224145Sdim if (N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL) 2900224145Sdim std::swap(N0, N1); 2901224145Sdim if (N0.getOpcode() == ISD::AND) { 2902224145Sdim if (!N0.getNode()->hasOneUse()) 2903224145Sdim return SDValue(); 2904224145Sdim ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1)); 2905224145Sdim if (!N01C || N01C->getZExtValue() != 0xFF00) 2906224145Sdim return SDValue(); 2907224145Sdim N0 = N0.getOperand(0); 2908224145Sdim LookPassAnd0 = true; 2909224145Sdim } 2910224145Sdim 2911224145Sdim if (N1.getOpcode() == ISD::AND) { 2912224145Sdim if (!N1.getNode()->hasOneUse()) 2913224145Sdim return SDValue(); 2914224145Sdim ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1)); 2915224145Sdim if (!N11C || N11C->getZExtValue() != 0xFF) 2916224145Sdim return SDValue(); 2917224145Sdim N1 = N1.getOperand(0); 2918224145Sdim LookPassAnd1 = true; 2919224145Sdim } 2920224145Sdim 2921224145Sdim if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL) 2922224145Sdim std::swap(N0, N1); 2923224145Sdim if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL) 2924224145Sdim return SDValue(); 2925224145Sdim if (!N0.getNode()->hasOneUse() || 2926224145Sdim !N1.getNode()->hasOneUse()) 2927224145Sdim return SDValue(); 2928224145Sdim 2929224145Sdim ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1)); 2930224145Sdim ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1)); 2931224145Sdim if (!N01C || !N11C) 2932224145Sdim return SDValue(); 2933224145Sdim if (N01C->getZExtValue() != 8 || N11C->getZExtValue() != 8) 2934224145Sdim return SDValue(); 2935224145Sdim 2936224145Sdim // Look for (shl (and a, 0xff), 8), (srl (and a, 0xff00), 8) 2937224145Sdim SDValue N00 = N0->getOperand(0); 2938224145Sdim if (!LookPassAnd0 && N00.getOpcode() == ISD::AND) { 2939224145Sdim if (!N00.getNode()->hasOneUse()) 2940224145Sdim return SDValue(); 2941224145Sdim ConstantSDNode *N001C = dyn_cast<ConstantSDNode>(N00.getOperand(1)); 2942224145Sdim if (!N001C || N001C->getZExtValue() != 0xFF) 2943224145Sdim return SDValue(); 2944224145Sdim N00 = N00.getOperand(0); 2945224145Sdim LookPassAnd0 = true; 2946224145Sdim } 2947224145Sdim 2948224145Sdim SDValue N10 = N1->getOperand(0); 2949224145Sdim if (!LookPassAnd1 && N10.getOpcode() == ISD::AND) { 2950224145Sdim if (!N10.getNode()->hasOneUse()) 2951224145Sdim return SDValue(); 2952224145Sdim ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N10.getOperand(1)); 2953224145Sdim if (!N101C || N101C->getZExtValue() != 0xFF00) 2954224145Sdim return SDValue(); 2955224145Sdim N10 = N10.getOperand(0); 2956224145Sdim LookPassAnd1 = true; 2957224145Sdim } 2958224145Sdim 2959224145Sdim if (N00 != N10) 2960224145Sdim return SDValue(); 2961224145Sdim 2962263508Sdim // Make sure everything beyond the low halfword gets set to zero since the SRL 2963263508Sdim // 16 will clear the top bits. 2964224145Sdim unsigned OpSizeInBits = VT.getSizeInBits(); 2965263508Sdim if (DemandHighBits && OpSizeInBits > 16) { 2966263508Sdim // If the left-shift isn't masked out then the only way this is a bswap is 2967263508Sdim // if all bits beyond the low 8 are 0. In that case the entire pattern 2968263508Sdim // reduces to a left shift anyway: leave it for other parts of the combiner. 2969263508Sdim if (!LookPassAnd0) 2970263508Sdim return SDValue(); 2971224145Sdim 2972263508Sdim // However, if the right shift isn't masked out then it might be because 2973263508Sdim // it's not needed. See if we can spot that too. 2974263508Sdim if (!LookPassAnd1 && 2975263508Sdim !DAG.MaskedValueIsZero( 2976263508Sdim N10, APInt::getHighBitsSet(OpSizeInBits, OpSizeInBits - 16))) 2977263508Sdim return SDValue(); 2978263508Sdim } 2979263508Sdim 2980263508Sdim SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00); 2981224145Sdim if (OpSizeInBits > 16) 2982263508Sdim Res = DAG.getNode(ISD::SRL, SDLoc(N), VT, Res, 2983224145Sdim DAG.getConstant(OpSizeInBits-16, getShiftAmountTy(VT))); 2984224145Sdim return Res; 2985224145Sdim} 2986224145Sdim 2987224145Sdim/// isBSwapHWordElement - Return true if the specified node is an element 2988224145Sdim/// that makes up a 32-bit packed halfword byteswap. i.e. 2989224145Sdim/// ((x&0xff)<<8)|((x&0xff00)>>8)|((x&0x00ff0000)<<8)|((x&0xff000000)>>8) 2990263508Sdimstatic bool isBSwapHWordElement(SDValue N, SmallVectorImpl<SDNode *> &Parts) { 2991224145Sdim if (!N.getNode()->hasOneUse()) 2992224145Sdim return false; 2993224145Sdim 2994224145Sdim unsigned Opc = N.getOpcode(); 2995224145Sdim if (Opc != ISD::AND && Opc != ISD::SHL && Opc != ISD::SRL) 2996224145Sdim return false; 2997224145Sdim 2998224145Sdim ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N.getOperand(1)); 2999224145Sdim if (!N1C) 3000224145Sdim return false; 3001224145Sdim 3002224145Sdim unsigned Num; 3003224145Sdim switch (N1C->getZExtValue()) { 3004224145Sdim default: 3005224145Sdim return false; 3006224145Sdim case 0xFF: Num = 0; break; 3007224145Sdim case 0xFF00: Num = 1; break; 3008224145Sdim case 0xFF0000: Num = 2; break; 3009224145Sdim case 0xFF000000: Num = 3; break; 3010224145Sdim } 3011224145Sdim 3012224145Sdim // Look for (x & 0xff) << 8 as well as ((x << 8) & 0xff00). 3013224145Sdim SDValue N0 = N.getOperand(0); 3014224145Sdim if (Opc == ISD::AND) { 3015224145Sdim if (Num == 0 || Num == 2) { 3016224145Sdim // (x >> 8) & 0xff 3017224145Sdim // (x >> 8) & 0xff0000 3018224145Sdim if (N0.getOpcode() != ISD::SRL) 3019224145Sdim return false; 3020224145Sdim ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1)); 3021224145Sdim if (!C || C->getZExtValue() != 8) 3022224145Sdim return false; 3023224145Sdim } else { 3024224145Sdim // (x << 8) & 0xff00 3025224145Sdim // (x << 8) & 0xff000000 3026224145Sdim if (N0.getOpcode() != ISD::SHL) 3027224145Sdim return false; 3028224145Sdim ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1)); 3029224145Sdim if (!C || C->getZExtValue() != 8) 3030224145Sdim return false; 3031224145Sdim } 3032224145Sdim } else if (Opc == ISD::SHL) { 3033224145Sdim // (x & 0xff) << 8 3034224145Sdim // (x & 0xff0000) << 8 3035224145Sdim if (Num != 0 && Num != 2) 3036224145Sdim return false; 3037224145Sdim ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1)); 3038224145Sdim if (!C || C->getZExtValue() != 8) 3039224145Sdim return false; 3040224145Sdim } else { // Opc == ISD::SRL 3041224145Sdim // (x & 0xff00) >> 8 3042224145Sdim // (x & 0xff000000) >> 8 3043224145Sdim if (Num != 1 && Num != 3) 3044224145Sdim return false; 3045224145Sdim ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1)); 3046224145Sdim if (!C || C->getZExtValue() != 8) 3047224145Sdim return false; 3048224145Sdim } 3049224145Sdim 3050224145Sdim if (Parts[Num]) 3051224145Sdim return false; 3052224145Sdim 3053224145Sdim Parts[Num] = N0.getOperand(0).getNode(); 3054224145Sdim return true; 3055224145Sdim} 3056224145Sdim 3057224145Sdim/// MatchBSwapHWord - Match a 32-bit packed halfword bswap. That is 3058224145Sdim/// ((x&0xff)<<8)|((x&0xff00)>>8)|((x&0x00ff0000)<<8)|((x&0xff000000)>>8) 3059224145Sdim/// => (rotl (bswap x), 16) 3060224145SdimSDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) { 3061224145Sdim if (!LegalOperations) 3062224145Sdim return SDValue(); 3063224145Sdim 3064224145Sdim EVT VT = N->getValueType(0); 3065224145Sdim if (VT != MVT::i32) 3066224145Sdim return SDValue(); 3067224145Sdim if (!TLI.isOperationLegal(ISD::BSWAP, VT)) 3068224145Sdim return SDValue(); 3069224145Sdim 3070224145Sdim SmallVector<SDNode*,4> Parts(4, (SDNode*)0); 3071224145Sdim // Look for either 3072224145Sdim // (or (or (and), (and)), (or (and), (and))) 3073224145Sdim // (or (or (or (and), (and)), (and)), (and)) 3074224145Sdim if (N0.getOpcode() != ISD::OR) 3075224145Sdim return SDValue(); 3076224145Sdim SDValue N00 = N0.getOperand(0); 3077224145Sdim SDValue N01 = N0.getOperand(1); 3078224145Sdim 3079249423Sdim if (N1.getOpcode() == ISD::OR && 3080249423Sdim N00.getNumOperands() == 2 && N01.getNumOperands() == 2) { 3081224145Sdim // (or (or (and), (and)), (or (and), (and))) 3082224145Sdim SDValue N000 = N00.getOperand(0); 3083224145Sdim if (!isBSwapHWordElement(N000, Parts)) 3084224145Sdim return SDValue(); 3085224145Sdim 3086224145Sdim SDValue N001 = N00.getOperand(1); 3087224145Sdim if (!isBSwapHWordElement(N001, Parts)) 3088224145Sdim return SDValue(); 3089224145Sdim SDValue N010 = N01.getOperand(0); 3090224145Sdim if (!isBSwapHWordElement(N010, Parts)) 3091224145Sdim return SDValue(); 3092224145Sdim SDValue N011 = N01.getOperand(1); 3093224145Sdim if (!isBSwapHWordElement(N011, Parts)) 3094224145Sdim return SDValue(); 3095224145Sdim } else { 3096224145Sdim // (or (or (or (and), (and)), (and)), (and)) 3097224145Sdim if (!isBSwapHWordElement(N1, Parts)) 3098224145Sdim return SDValue(); 3099224145Sdim if (!isBSwapHWordElement(N01, Parts)) 3100224145Sdim return SDValue(); 3101224145Sdim if (N00.getOpcode() != ISD::OR) 3102224145Sdim return SDValue(); 3103224145Sdim SDValue N000 = N00.getOperand(0); 3104224145Sdim if (!isBSwapHWordElement(N000, Parts)) 3105224145Sdim return SDValue(); 3106224145Sdim SDValue N001 = N00.getOperand(1); 3107224145Sdim if (!isBSwapHWordElement(N001, Parts)) 3108224145Sdim return SDValue(); 3109224145Sdim } 3110224145Sdim 3111224145Sdim // Make sure the parts are all coming from the same node. 3112224145Sdim if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3]) 3113224145Sdim return SDValue(); 3114224145Sdim 3115263508Sdim SDValue BSwap = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, 3116224145Sdim SDValue(Parts[0],0)); 3117224145Sdim 3118263508Sdim // Result of the bswap should be rotated by 16. If it's not legal, then 3119224145Sdim // do (x << 16) | (x >> 16). 3120224145Sdim SDValue ShAmt = DAG.getConstant(16, getShiftAmountTy(VT)); 3121224145Sdim if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT)) 3122263508Sdim return DAG.getNode(ISD::ROTL, SDLoc(N), VT, BSwap, ShAmt); 3123243830Sdim if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT)) 3124263508Sdim return DAG.getNode(ISD::ROTR, SDLoc(N), VT, BSwap, ShAmt); 3125263508Sdim return DAG.getNode(ISD::OR, SDLoc(N), VT, 3126263508Sdim DAG.getNode(ISD::SHL, SDLoc(N), VT, BSwap, ShAmt), 3127263508Sdim DAG.getNode(ISD::SRL, SDLoc(N), VT, BSwap, ShAmt)); 3128224145Sdim} 3129224145Sdim 3130193323SedSDValue DAGCombiner::visitOR(SDNode *N) { 3131193323Sed SDValue N0 = N->getOperand(0); 3132193323Sed SDValue N1 = N->getOperand(1); 3133193323Sed SDValue LL, LR, RL, RR, CC0, CC1; 3134193323Sed ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0); 3135193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 3136198090Srdivacky EVT VT = N1.getValueType(); 3137193323Sed 3138193323Sed // fold vector ops 3139193323Sed if (VT.isVector()) { 3140193323Sed SDValue FoldedVOp = SimplifyVBinOp(N); 3141193323Sed if (FoldedVOp.getNode()) return FoldedVOp; 3142249423Sdim 3143249423Sdim // fold (or x, 0) -> x, vector edition 3144249423Sdim if (ISD::isBuildVectorAllZeros(N0.getNode())) 3145249423Sdim return N1; 3146249423Sdim if (ISD::isBuildVectorAllZeros(N1.getNode())) 3147249423Sdim return N0; 3148249423Sdim 3149249423Sdim // fold (or x, -1) -> -1, vector edition 3150249423Sdim if (ISD::isBuildVectorAllOnes(N0.getNode())) 3151249423Sdim return N0; 3152249423Sdim if (ISD::isBuildVectorAllOnes(N1.getNode())) 3153249423Sdim return N1; 3154193323Sed } 3155193323Sed 3156193323Sed // fold (or x, undef) -> -1 3157210299Sed if (!LegalOperations && 3158210299Sed (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)) { 3159200581Srdivacky EVT EltVT = VT.isVector() ? VT.getVectorElementType() : VT; 3160200581Srdivacky return DAG.getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT); 3161200581Srdivacky } 3162193323Sed // fold (or c1, c2) -> c1|c2 3163193323Sed if (N0C && N1C) 3164193323Sed return DAG.FoldConstantArithmetic(ISD::OR, VT, N0C, N1C); 3165193323Sed // canonicalize constant to RHS 3166193323Sed if (N0C && !N1C) 3167263508Sdim return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0); 3168193323Sed // fold (or x, 0) -> x 3169193323Sed if (N1C && N1C->isNullValue()) 3170193323Sed return N0; 3171193323Sed // fold (or x, -1) -> -1 3172193323Sed if (N1C && N1C->isAllOnesValue()) 3173193323Sed return N1; 3174193323Sed // fold (or x, c) -> c iff (x & ~c) == 0 3175193323Sed if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue())) 3176193323Sed return N1; 3177224145Sdim 3178224145Sdim // Recognize halfword bswaps as (bswap + rotl 16) or (bswap + shl 16) 3179224145Sdim SDValue BSwap = MatchBSwapHWord(N, N0, N1); 3180224145Sdim if (BSwap.getNode() != 0) 3181224145Sdim return BSwap; 3182224145Sdim BSwap = MatchBSwapHWordLow(N, N0, N1); 3183224145Sdim if (BSwap.getNode() != 0) 3184224145Sdim return BSwap; 3185224145Sdim 3186193323Sed // reassociate or 3187263508Sdim SDValue ROR = ReassociateOps(ISD::OR, SDLoc(N), N0, N1); 3188193323Sed if (ROR.getNode() != 0) 3189193323Sed return ROR; 3190193323Sed // Canonicalize (or (and X, c1), c2) -> (and (or X, c2), c1|c2) 3191204642Srdivacky // iff (c1 & c2) == 0. 3192193323Sed if (N1C && N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() && 3193193323Sed isa<ConstantSDNode>(N0.getOperand(1))) { 3194193323Sed ConstantSDNode *C1 = cast<ConstantSDNode>(N0.getOperand(1)); 3195204642Srdivacky if ((C1->getAPIntValue() & N1C->getAPIntValue()) != 0) 3196263508Sdim return DAG.getNode(ISD::AND, SDLoc(N), VT, 3197263508Sdim DAG.getNode(ISD::OR, SDLoc(N0), VT, 3198204642Srdivacky N0.getOperand(0), N1), 3199204642Srdivacky DAG.FoldConstantArithmetic(ISD::OR, VT, N1C, C1)); 3200193323Sed } 3201193323Sed // fold (or (setcc x), (setcc y)) -> (setcc (or x, y)) 3202193323Sed if (isSetCCEquivalent(N0, LL, LR, CC0) && isSetCCEquivalent(N1, RL, RR, CC1)){ 3203193323Sed ISD::CondCode Op0 = cast<CondCodeSDNode>(CC0)->get(); 3204193323Sed ISD::CondCode Op1 = cast<CondCodeSDNode>(CC1)->get(); 3205193323Sed 3206193323Sed if (LR == RR && isa<ConstantSDNode>(LR) && Op0 == Op1 && 3207193323Sed LL.getValueType().isInteger()) { 3208193323Sed // fold (or (setne X, 0), (setne Y, 0)) -> (setne (or X, Y), 0) 3209193323Sed // fold (or (setlt X, 0), (setlt Y, 0)) -> (setne (or X, Y), 0) 3210193323Sed if (cast<ConstantSDNode>(LR)->isNullValue() && 3211193323Sed (Op1 == ISD::SETNE || Op1 == ISD::SETLT)) { 3212263508Sdim SDValue ORNode = DAG.getNode(ISD::OR, SDLoc(LR), 3213193323Sed LR.getValueType(), LL, RL); 3214193323Sed AddToWorkList(ORNode.getNode()); 3215263508Sdim return DAG.getSetCC(SDLoc(N), VT, ORNode, LR, Op1); 3216193323Sed } 3217193323Sed // fold (or (setne X, -1), (setne Y, -1)) -> (setne (and X, Y), -1) 3218193323Sed // fold (or (setgt X, -1), (setgt Y -1)) -> (setgt (and X, Y), -1) 3219193323Sed if (cast<ConstantSDNode>(LR)->isAllOnesValue() && 3220193323Sed (Op1 == ISD::SETNE || Op1 == ISD::SETGT)) { 3221263508Sdim SDValue ANDNode = DAG.getNode(ISD::AND, SDLoc(LR), 3222193323Sed LR.getValueType(), LL, RL); 3223193323Sed AddToWorkList(ANDNode.getNode()); 3224263508Sdim return DAG.getSetCC(SDLoc(N), VT, ANDNode, LR, Op1); 3225193323Sed } 3226193323Sed } 3227193323Sed // canonicalize equivalent to ll == rl 3228193323Sed if (LL == RR && LR == RL) { 3229193323Sed Op1 = ISD::getSetCCSwappedOperands(Op1); 3230193323Sed std::swap(RL, RR); 3231193323Sed } 3232193323Sed if (LL == RL && LR == RR) { 3233193323Sed bool isInteger = LL.getValueType().isInteger(); 3234193323Sed ISD::CondCode Result = ISD::getSetCCOrOperation(Op0, Op1, isInteger); 3235193323Sed if (Result != ISD::SETCC_INVALID && 3236249423Sdim (!LegalOperations || 3237249423Sdim (TLI.isCondCodeLegal(Result, LL.getSimpleValueType()) && 3238249423Sdim TLI.isOperationLegal(ISD::SETCC, 3239263508Sdim getSetCCResultType(N0.getValueType()))))) 3240263508Sdim return DAG.getSetCC(SDLoc(N), N0.getValueType(), 3241193323Sed LL, LR, Result); 3242193323Sed } 3243193323Sed } 3244193323Sed 3245193323Sed // Simplify: (or (op x...), (op y...)) -> (op (or x, y)) 3246193323Sed if (N0.getOpcode() == N1.getOpcode()) { 3247193323Sed SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N); 3248193323Sed if (Tmp.getNode()) return Tmp; 3249193323Sed } 3250193323Sed 3251193323Sed // (or (and X, C1), (and Y, C2)) -> (and (or X, Y), C3) if possible. 3252193323Sed if (N0.getOpcode() == ISD::AND && 3253193323Sed N1.getOpcode() == ISD::AND && 3254193323Sed N0.getOperand(1).getOpcode() == ISD::Constant && 3255193323Sed N1.getOperand(1).getOpcode() == ISD::Constant && 3256193323Sed // Don't increase # computations. 3257193323Sed (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) { 3258193323Sed // We can only do this xform if we know that bits from X that are set in C2 3259193323Sed // but not in C1 are already zero. Likewise for Y. 3260193323Sed const APInt &LHSMask = 3261193323Sed cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue(); 3262193323Sed const APInt &RHSMask = 3263193323Sed cast<ConstantSDNode>(N1.getOperand(1))->getAPIntValue(); 3264193323Sed 3265193323Sed if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) && 3266193323Sed DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) { 3267263508Sdim SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT, 3268193323Sed N0.getOperand(0), N1.getOperand(0)); 3269263508Sdim return DAG.getNode(ISD::AND, SDLoc(N), VT, X, 3270193323Sed DAG.getConstant(LHSMask | RHSMask, VT)); 3271193323Sed } 3272193323Sed } 3273193323Sed 3274193323Sed // See if this is some rotate idiom. 3275263508Sdim if (SDNode *Rot = MatchRotate(N0, N1, SDLoc(N))) 3276193323Sed return SDValue(Rot, 0); 3277193323Sed 3278210299Sed // Simplify the operands using demanded-bits information. 3279210299Sed if (!VT.isVector() && 3280210299Sed SimplifyDemandedBits(SDValue(N, 0))) 3281210299Sed return SDValue(N, 0); 3282210299Sed 3283193323Sed return SDValue(); 3284193323Sed} 3285193323Sed 3286193323Sed/// MatchRotateHalf - Match "(X shl/srl V1) & V2" where V2 may not be present. 3287193323Sedstatic bool MatchRotateHalf(SDValue Op, SDValue &Shift, SDValue &Mask) { 3288193323Sed if (Op.getOpcode() == ISD::AND) { 3289193323Sed if (isa<ConstantSDNode>(Op.getOperand(1))) { 3290193323Sed Mask = Op.getOperand(1); 3291193323Sed Op = Op.getOperand(0); 3292193323Sed } else { 3293193323Sed return false; 3294193323Sed } 3295193323Sed } 3296193323Sed 3297193323Sed if (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) { 3298193323Sed Shift = Op; 3299193323Sed return true; 3300193323Sed } 3301193323Sed 3302193323Sed return false; 3303193323Sed} 3304193323Sed 3305193323Sed// MatchRotate - Handle an 'or' of two operands. If this is one of the many 3306193323Sed// idioms for rotate, and if the target supports rotation instructions, generate 3307193323Sed// a rot[lr]. 3308263508SdimSDNode *DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, SDLoc DL) { 3309193323Sed // Must be a legal type. Expanded 'n promoted things won't work with rotates. 3310198090Srdivacky EVT VT = LHS.getValueType(); 3311193323Sed if (!TLI.isTypeLegal(VT)) return 0; 3312193323Sed 3313193323Sed // The target must have at least one rotate flavor. 3314193323Sed bool HasROTL = TLI.isOperationLegalOrCustom(ISD::ROTL, VT); 3315193323Sed bool HasROTR = TLI.isOperationLegalOrCustom(ISD::ROTR, VT); 3316193323Sed if (!HasROTL && !HasROTR) return 0; 3317193323Sed 3318193323Sed // Match "(X shl/srl V1) & V2" where V2 may not be present. 3319193323Sed SDValue LHSShift; // The shift. 3320193323Sed SDValue LHSMask; // AND value if any. 3321193323Sed if (!MatchRotateHalf(LHS, LHSShift, LHSMask)) 3322193323Sed return 0; // Not part of a rotate. 3323193323Sed 3324193323Sed SDValue RHSShift; // The shift. 3325193323Sed SDValue RHSMask; // AND value if any. 3326193323Sed if (!MatchRotateHalf(RHS, RHSShift, RHSMask)) 3327193323Sed return 0; // Not part of a rotate. 3328193323Sed 3329193323Sed if (LHSShift.getOperand(0) != RHSShift.getOperand(0)) 3330193323Sed return 0; // Not shifting the same value. 3331193323Sed 3332193323Sed if (LHSShift.getOpcode() == RHSShift.getOpcode()) 3333193323Sed return 0; // Shifts must disagree. 3334193323Sed 3335193323Sed // Canonicalize shl to left side in a shl/srl pair. 3336193323Sed if (RHSShift.getOpcode() == ISD::SHL) { 3337193323Sed std::swap(LHS, RHS); 3338193323Sed std::swap(LHSShift, RHSShift); 3339193323Sed std::swap(LHSMask , RHSMask ); 3340193323Sed } 3341193323Sed 3342193323Sed unsigned OpSizeInBits = VT.getSizeInBits(); 3343193323Sed SDValue LHSShiftArg = LHSShift.getOperand(0); 3344193323Sed SDValue LHSShiftAmt = LHSShift.getOperand(1); 3345193323Sed SDValue RHSShiftAmt = RHSShift.getOperand(1); 3346193323Sed 3347193323Sed // fold (or (shl x, C1), (srl x, C2)) -> (rotl x, C1) 3348193323Sed // fold (or (shl x, C1), (srl x, C2)) -> (rotr x, C2) 3349193323Sed if (LHSShiftAmt.getOpcode() == ISD::Constant && 3350193323Sed RHSShiftAmt.getOpcode() == ISD::Constant) { 3351193323Sed uint64_t LShVal = cast<ConstantSDNode>(LHSShiftAmt)->getZExtValue(); 3352193323Sed uint64_t RShVal = cast<ConstantSDNode>(RHSShiftAmt)->getZExtValue(); 3353193323Sed if ((LShVal + RShVal) != OpSizeInBits) 3354193323Sed return 0; 3355193323Sed 3356243830Sdim SDValue Rot = DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT, 3357243830Sdim LHSShiftArg, HasROTL ? LHSShiftAmt : RHSShiftAmt); 3358193323Sed 3359193323Sed // If there is an AND of either shifted operand, apply it to the result. 3360193323Sed if (LHSMask.getNode() || RHSMask.getNode()) { 3361193323Sed APInt Mask = APInt::getAllOnesValue(OpSizeInBits); 3362193323Sed 3363193323Sed if (LHSMask.getNode()) { 3364193323Sed APInt RHSBits = APInt::getLowBitsSet(OpSizeInBits, LShVal); 3365193323Sed Mask &= cast<ConstantSDNode>(LHSMask)->getAPIntValue() | RHSBits; 3366193323Sed } 3367193323Sed if (RHSMask.getNode()) { 3368193323Sed APInt LHSBits = APInt::getHighBitsSet(OpSizeInBits, RShVal); 3369193323Sed Mask &= cast<ConstantSDNode>(RHSMask)->getAPIntValue() | LHSBits; 3370193323Sed } 3371193323Sed 3372193323Sed Rot = DAG.getNode(ISD::AND, DL, VT, Rot, DAG.getConstant(Mask, VT)); 3373193323Sed } 3374193323Sed 3375193323Sed return Rot.getNode(); 3376193323Sed } 3377193323Sed 3378193323Sed // If there is a mask here, and we have a variable shift, we can't be sure 3379193323Sed // that we're masking out the right stuff. 3380193323Sed if (LHSMask.getNode() || RHSMask.getNode()) 3381193323Sed return 0; 3382193323Sed 3383263508Sdim // If the shift amount is sign/zext/any-extended just peel it off. 3384263508Sdim SDValue LExtOp0 = LHSShiftAmt; 3385263508Sdim SDValue RExtOp0 = RHSShiftAmt; 3386243830Sdim if ((LHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND || 3387243830Sdim LHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND || 3388243830Sdim LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND || 3389243830Sdim LHSShiftAmt.getOpcode() == ISD::TRUNCATE) && 3390243830Sdim (RHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND || 3391243830Sdim RHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND || 3392243830Sdim RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND || 3393243830Sdim RHSShiftAmt.getOpcode() == ISD::TRUNCATE)) { 3394263508Sdim LExtOp0 = LHSShiftAmt.getOperand(0); 3395263508Sdim RExtOp0 = RHSShiftAmt.getOperand(0); 3396193323Sed } 3397193323Sed 3398263508Sdim if (RExtOp0.getOpcode() == ISD::SUB && RExtOp0.getOperand(1) == LExtOp0) { 3399263508Sdim // fold (or (shl x, (*ext y)), (srl x, (*ext (sub 32, y)))) -> 3400263508Sdim // (rotl x, y) 3401263508Sdim // fold (or (shl x, (*ext y)), (srl x, (*ext (sub 32, y)))) -> 3402263508Sdim // (rotr x, (sub 32, y)) 3403263508Sdim if (ConstantSDNode *SUBC = 3404263508Sdim dyn_cast<ConstantSDNode>(RExtOp0.getOperand(0))) 3405263508Sdim if (SUBC->getAPIntValue() == OpSizeInBits) 3406263508Sdim return DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT, LHSShiftArg, 3407263508Sdim HasROTL ? LHSShiftAmt : RHSShiftAmt).getNode(); 3408263508Sdim } else if (LExtOp0.getOpcode() == ISD::SUB && 3409263508Sdim RExtOp0 == LExtOp0.getOperand(1)) { 3410263508Sdim // fold (or (shl x, (*ext (sub 32, y))), (srl x, (*ext y))) -> 3411263508Sdim // (rotr x, y) 3412263508Sdim // fold (or (shl x, (*ext (sub 32, y))), (srl x, (*ext y))) -> 3413263508Sdim // (rotl x, (sub 32, y)) 3414263508Sdim if (ConstantSDNode *SUBC = 3415263508Sdim dyn_cast<ConstantSDNode>(LExtOp0.getOperand(0))) 3416263508Sdim if (SUBC->getAPIntValue() == OpSizeInBits) 3417263508Sdim return DAG.getNode(HasROTR ? ISD::ROTR : ISD::ROTL, DL, VT, LHSShiftArg, 3418263508Sdim HasROTR ? RHSShiftAmt : LHSShiftAmt).getNode(); 3419263508Sdim } 3420263508Sdim 3421193323Sed return 0; 3422193323Sed} 3423193323Sed 3424193323SedSDValue DAGCombiner::visitXOR(SDNode *N) { 3425193323Sed SDValue N0 = N->getOperand(0); 3426193323Sed SDValue N1 = N->getOperand(1); 3427193323Sed SDValue LHS, RHS, CC; 3428193323Sed ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0); 3429193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 3430198090Srdivacky EVT VT = N0.getValueType(); 3431193323Sed 3432193323Sed // fold vector ops 3433193323Sed if (VT.isVector()) { 3434193323Sed SDValue FoldedVOp = SimplifyVBinOp(N); 3435193323Sed if (FoldedVOp.getNode()) return FoldedVOp; 3436249423Sdim 3437249423Sdim // fold (xor x, 0) -> x, vector edition 3438249423Sdim if (ISD::isBuildVectorAllZeros(N0.getNode())) 3439249423Sdim return N1; 3440249423Sdim if (ISD::isBuildVectorAllZeros(N1.getNode())) 3441249423Sdim return N0; 3442193323Sed } 3443193323Sed 3444193323Sed // fold (xor undef, undef) -> 0. This is a common idiom (misuse). 3445193323Sed if (N0.getOpcode() == ISD::UNDEF && N1.getOpcode() == ISD::UNDEF) 3446193323Sed return DAG.getConstant(0, VT); 3447193323Sed // fold (xor x, undef) -> undef 3448193323Sed if (N0.getOpcode() == ISD::UNDEF) 3449193323Sed return N0; 3450193323Sed if (N1.getOpcode() == ISD::UNDEF) 3451193323Sed return N1; 3452193323Sed // fold (xor c1, c2) -> c1^c2 3453193323Sed if (N0C && N1C) 3454193323Sed return DAG.FoldConstantArithmetic(ISD::XOR, VT, N0C, N1C); 3455193323Sed // canonicalize constant to RHS 3456193323Sed if (N0C && !N1C) 3457263508Sdim return DAG.getNode(ISD::XOR, SDLoc(N), VT, N1, N0); 3458193323Sed // fold (xor x, 0) -> x 3459193323Sed if (N1C && N1C->isNullValue()) 3460193323Sed return N0; 3461193323Sed // reassociate xor 3462263508Sdim SDValue RXOR = ReassociateOps(ISD::XOR, SDLoc(N), N0, N1); 3463193323Sed if (RXOR.getNode() != 0) 3464193323Sed return RXOR; 3465193323Sed 3466193323Sed // fold !(x cc y) -> (x !cc y) 3467193323Sed if (N1C && N1C->getAPIntValue() == 1 && isSetCCEquivalent(N0, LHS, RHS, CC)) { 3468193323Sed bool isInt = LHS.getValueType().isInteger(); 3469193323Sed ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(), 3470193323Sed isInt); 3471193323Sed 3472249423Sdim if (!LegalOperations || 3473249423Sdim TLI.isCondCodeLegal(NotCC, LHS.getSimpleValueType())) { 3474193323Sed switch (N0.getOpcode()) { 3475193323Sed default: 3476198090Srdivacky llvm_unreachable("Unhandled SetCC Equivalent!"); 3477193323Sed case ISD::SETCC: 3478263508Sdim return DAG.getSetCC(SDLoc(N), VT, LHS, RHS, NotCC); 3479193323Sed case ISD::SELECT_CC: 3480263508Sdim return DAG.getSelectCC(SDLoc(N), LHS, RHS, N0.getOperand(2), 3481193323Sed N0.getOperand(3), NotCC); 3482193323Sed } 3483193323Sed } 3484193323Sed } 3485193323Sed 3486193323Sed // fold (not (zext (setcc x, y))) -> (zext (not (setcc x, y))) 3487193323Sed if (N1C && N1C->getAPIntValue() == 1 && N0.getOpcode() == ISD::ZERO_EXTEND && 3488193323Sed N0.getNode()->hasOneUse() && 3489193323Sed isSetCCEquivalent(N0.getOperand(0), LHS, RHS, CC)){ 3490193323Sed SDValue V = N0.getOperand(0); 3491263508Sdim V = DAG.getNode(ISD::XOR, SDLoc(N0), V.getValueType(), V, 3492193323Sed DAG.getConstant(1, V.getValueType())); 3493193323Sed AddToWorkList(V.getNode()); 3494263508Sdim return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, V); 3495193323Sed } 3496193323Sed 3497193323Sed // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are setcc 3498193323Sed if (N1C && N1C->getAPIntValue() == 1 && VT == MVT::i1 && 3499193323Sed (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) { 3500193323Sed SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1); 3501193323Sed if (isOneUseSetCC(RHS) || isOneUseSetCC(LHS)) { 3502193323Sed unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND; 3503263508Sdim LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS 3504263508Sdim RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS 3505193323Sed AddToWorkList(LHS.getNode()); AddToWorkList(RHS.getNode()); 3506263508Sdim return DAG.getNode(NewOpcode, SDLoc(N), VT, LHS, RHS); 3507193323Sed } 3508193323Sed } 3509193323Sed // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are constants 3510193323Sed if (N1C && N1C->isAllOnesValue() && 3511193323Sed (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) { 3512193323Sed SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1); 3513193323Sed if (isa<ConstantSDNode>(RHS) || isa<ConstantSDNode>(LHS)) { 3514193323Sed unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND; 3515263508Sdim LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS 3516263508Sdim RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS 3517193323Sed AddToWorkList(LHS.getNode()); AddToWorkList(RHS.getNode()); 3518263508Sdim return DAG.getNode(NewOpcode, SDLoc(N), VT, LHS, RHS); 3519193323Sed } 3520193323Sed } 3521263508Sdim // fold (xor (and x, y), y) -> (and (not x), y) 3522263508Sdim if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() && 3523263508Sdim N0->getOperand(1) == N1) { 3524263508Sdim SDValue X = N0->getOperand(0); 3525263508Sdim SDValue NotX = DAG.getNOT(SDLoc(X), X, VT); 3526263508Sdim AddToWorkList(NotX.getNode()); 3527263508Sdim return DAG.getNode(ISD::AND, SDLoc(N), VT, NotX, N1); 3528263508Sdim } 3529193323Sed // fold (xor (xor x, c1), c2) -> (xor x, (xor c1, c2)) 3530193323Sed if (N1C && N0.getOpcode() == ISD::XOR) { 3531193323Sed ConstantSDNode *N00C = dyn_cast<ConstantSDNode>(N0.getOperand(0)); 3532193323Sed ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1)); 3533193323Sed if (N00C) 3534263508Sdim return DAG.getNode(ISD::XOR, SDLoc(N), VT, N0.getOperand(1), 3535193323Sed DAG.getConstant(N1C->getAPIntValue() ^ 3536193323Sed N00C->getAPIntValue(), VT)); 3537193323Sed if (N01C) 3538263508Sdim return DAG.getNode(ISD::XOR, SDLoc(N), VT, N0.getOperand(0), 3539193323Sed DAG.getConstant(N1C->getAPIntValue() ^ 3540193323Sed N01C->getAPIntValue(), VT)); 3541193323Sed } 3542193323Sed // fold (xor x, x) -> 0 3543218893Sdim if (N0 == N1) 3544263508Sdim return tryFoldToZero(SDLoc(N), TLI, VT, DAG, LegalOperations, LegalTypes); 3545193323Sed 3546193323Sed // Simplify: xor (op x...), (op y...) -> (op (xor x, y)) 3547193323Sed if (N0.getOpcode() == N1.getOpcode()) { 3548193323Sed SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N); 3549193323Sed if (Tmp.getNode()) return Tmp; 3550193323Sed } 3551193323Sed 3552193323Sed // Simplify the expression using non-local knowledge. 3553193323Sed if (!VT.isVector() && 3554193323Sed SimplifyDemandedBits(SDValue(N, 0))) 3555193323Sed return SDValue(N, 0); 3556193323Sed 3557193323Sed return SDValue(); 3558193323Sed} 3559193323Sed 3560193323Sed/// visitShiftByConstant - Handle transforms common to the three shifts, when 3561193323Sed/// the shift amount is a constant. 3562193323SedSDValue DAGCombiner::visitShiftByConstant(SDNode *N, unsigned Amt) { 3563193323Sed SDNode *LHS = N->getOperand(0).getNode(); 3564193323Sed if (!LHS->hasOneUse()) return SDValue(); 3565193323Sed 3566193323Sed // We want to pull some binops through shifts, so that we have (and (shift)) 3567193323Sed // instead of (shift (and)), likewise for add, or, xor, etc. This sort of 3568193323Sed // thing happens with address calculations, so it's important to canonicalize 3569193323Sed // it. 3570193323Sed bool HighBitSet = false; // Can we transform this if the high bit is set? 3571193323Sed 3572193323Sed switch (LHS->getOpcode()) { 3573193323Sed default: return SDValue(); 3574193323Sed case ISD::OR: 3575193323Sed case ISD::XOR: 3576193323Sed HighBitSet = false; // We can only transform sra if the high bit is clear. 3577193323Sed break; 3578193323Sed case ISD::AND: 3579193323Sed HighBitSet = true; // We can only transform sra if the high bit is set. 3580193323Sed break; 3581193323Sed case ISD::ADD: 3582193323Sed if (N->getOpcode() != ISD::SHL) 3583193323Sed return SDValue(); // only shl(add) not sr[al](add). 3584193323Sed HighBitSet = false; // We can only transform sra if the high bit is clear. 3585193323Sed break; 3586193323Sed } 3587193323Sed 3588193323Sed // We require the RHS of the binop to be a constant as well. 3589193323Sed ConstantSDNode *BinOpCst = dyn_cast<ConstantSDNode>(LHS->getOperand(1)); 3590193323Sed if (!BinOpCst) return SDValue(); 3591193323Sed 3592193323Sed // FIXME: disable this unless the input to the binop is a shift by a constant. 3593193323Sed // If it is not a shift, it pessimizes some common cases like: 3594193323Sed // 3595193323Sed // void foo(int *X, int i) { X[i & 1235] = 1; } 3596193323Sed // int bar(int *X, int i) { return X[i & 255]; } 3597193323Sed SDNode *BinOpLHSVal = LHS->getOperand(0).getNode(); 3598193323Sed if ((BinOpLHSVal->getOpcode() != ISD::SHL && 3599193323Sed BinOpLHSVal->getOpcode() != ISD::SRA && 3600193323Sed BinOpLHSVal->getOpcode() != ISD::SRL) || 3601193323Sed !isa<ConstantSDNode>(BinOpLHSVal->getOperand(1))) 3602193323Sed return SDValue(); 3603193323Sed 3604198090Srdivacky EVT VT = N->getValueType(0); 3605193323Sed 3606193323Sed // If this is a signed shift right, and the high bit is modified by the 3607193323Sed // logical operation, do not perform the transformation. The highBitSet 3608193323Sed // boolean indicates the value of the high bit of the constant which would 3609193323Sed // cause it to be modified for this operation. 3610193323Sed if (N->getOpcode() == ISD::SRA) { 3611193323Sed bool BinOpRHSSignSet = BinOpCst->getAPIntValue().isNegative(); 3612193323Sed if (BinOpRHSSignSet != HighBitSet) 3613193323Sed return SDValue(); 3614193323Sed } 3615193323Sed 3616193323Sed // Fold the constants, shifting the binop RHS by the shift amount. 3617263508Sdim SDValue NewRHS = DAG.getNode(N->getOpcode(), SDLoc(LHS->getOperand(1)), 3618193323Sed N->getValueType(0), 3619193323Sed LHS->getOperand(1), N->getOperand(1)); 3620193323Sed 3621193323Sed // Create the new shift. 3622218893Sdim SDValue NewShift = DAG.getNode(N->getOpcode(), 3623263508Sdim SDLoc(LHS->getOperand(0)), 3624193323Sed VT, LHS->getOperand(0), N->getOperand(1)); 3625193323Sed 3626193323Sed // Create the new binop. 3627263508Sdim return DAG.getNode(LHS->getOpcode(), SDLoc(N), VT, NewShift, NewRHS); 3628193323Sed} 3629193323Sed 3630193323SedSDValue DAGCombiner::visitSHL(SDNode *N) { 3631193323Sed SDValue N0 = N->getOperand(0); 3632193323Sed SDValue N1 = N->getOperand(1); 3633193323Sed ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0); 3634193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 3635198090Srdivacky EVT VT = N0.getValueType(); 3636200581Srdivacky unsigned OpSizeInBits = VT.getScalarType().getSizeInBits(); 3637193323Sed 3638263508Sdim // fold vector ops 3639263508Sdim if (VT.isVector()) { 3640263508Sdim SDValue FoldedVOp = SimplifyVBinOp(N); 3641263508Sdim if (FoldedVOp.getNode()) return FoldedVOp; 3642263508Sdim } 3643263508Sdim 3644193323Sed // fold (shl c1, c2) -> c1<<c2 3645193323Sed if (N0C && N1C) 3646193323Sed return DAG.FoldConstantArithmetic(ISD::SHL, VT, N0C, N1C); 3647193323Sed // fold (shl 0, x) -> 0 3648193323Sed if (N0C && N0C->isNullValue()) 3649193323Sed return N0; 3650193323Sed // fold (shl x, c >= size(x)) -> undef 3651193323Sed if (N1C && N1C->getZExtValue() >= OpSizeInBits) 3652193323Sed return DAG.getUNDEF(VT); 3653193323Sed // fold (shl x, 0) -> x 3654193323Sed if (N1C && N1C->isNullValue()) 3655193323Sed return N0; 3656224145Sdim // fold (shl undef, x) -> 0 3657224145Sdim if (N0.getOpcode() == ISD::UNDEF) 3658224145Sdim return DAG.getConstant(0, VT); 3659193323Sed // if (shl x, c) is known to be zero, return 0 3660193323Sed if (DAG.MaskedValueIsZero(SDValue(N, 0), 3661200581Srdivacky APInt::getAllOnesValue(OpSizeInBits))) 3662193323Sed return DAG.getConstant(0, VT); 3663193323Sed // fold (shl x, (trunc (and y, c))) -> (shl x, (and (trunc y), (trunc c))). 3664193323Sed if (N1.getOpcode() == ISD::TRUNCATE && 3665193323Sed N1.getOperand(0).getOpcode() == ISD::AND && 3666193323Sed N1.hasOneUse() && N1.getOperand(0).hasOneUse()) { 3667193323Sed SDValue N101 = N1.getOperand(0).getOperand(1); 3668193323Sed if (ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N101)) { 3669198090Srdivacky EVT TruncVT = N1.getValueType(); 3670193323Sed SDValue N100 = N1.getOperand(0).getOperand(0); 3671193323Sed APInt TruncC = N101C->getAPIntValue(); 3672218893Sdim TruncC = TruncC.trunc(TruncVT.getSizeInBits()); 3673263508Sdim return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, 3674263508Sdim DAG.getNode(ISD::AND, SDLoc(N), TruncVT, 3675193323Sed DAG.getNode(ISD::TRUNCATE, 3676263508Sdim SDLoc(N), 3677193323Sed TruncVT, N100), 3678193323Sed DAG.getConstant(TruncC, TruncVT))); 3679193323Sed } 3680193323Sed } 3681193323Sed 3682193323Sed if (N1C && SimplifyDemandedBits(SDValue(N, 0))) 3683193323Sed return SDValue(N, 0); 3684193323Sed 3685193323Sed // fold (shl (shl x, c1), c2) -> 0 or (shl x, (add c1, c2)) 3686193323Sed if (N1C && N0.getOpcode() == ISD::SHL && 3687193323Sed N0.getOperand(1).getOpcode() == ISD::Constant) { 3688193323Sed uint64_t c1 = cast<ConstantSDNode>(N0.getOperand(1))->getZExtValue(); 3689193323Sed uint64_t c2 = N1C->getZExtValue(); 3690218893Sdim if (c1 + c2 >= OpSizeInBits) 3691193323Sed return DAG.getConstant(0, VT); 3692263508Sdim return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0.getOperand(0), 3693193323Sed DAG.getConstant(c1 + c2, N1.getValueType())); 3694193323Sed } 3695218893Sdim 3696218893Sdim // fold (shl (ext (shl x, c1)), c2) -> (ext (shl x, (add c1, c2))) 3697218893Sdim // For this to be valid, the second form must not preserve any of the bits 3698218893Sdim // that are shifted out by the inner shift in the first form. This means 3699218893Sdim // the outer shift size must be >= the number of bits added by the ext. 3700218893Sdim // As a corollary, we don't care what kind of ext it is. 3701218893Sdim if (N1C && (N0.getOpcode() == ISD::ZERO_EXTEND || 3702218893Sdim N0.getOpcode() == ISD::ANY_EXTEND || 3703218893Sdim N0.getOpcode() == ISD::SIGN_EXTEND) && 3704218893Sdim N0.getOperand(0).getOpcode() == ISD::SHL && 3705218893Sdim isa<ConstantSDNode>(N0.getOperand(0)->getOperand(1))) { 3706219077Sdim uint64_t c1 = 3707218893Sdim cast<ConstantSDNode>(N0.getOperand(0)->getOperand(1))->getZExtValue(); 3708218893Sdim uint64_t c2 = N1C->getZExtValue(); 3709218893Sdim EVT InnerShiftVT = N0.getOperand(0).getValueType(); 3710218893Sdim uint64_t InnerShiftSize = InnerShiftVT.getScalarType().getSizeInBits(); 3711218893Sdim if (c2 >= OpSizeInBits - InnerShiftSize) { 3712218893Sdim if (c1 + c2 >= OpSizeInBits) 3713218893Sdim return DAG.getConstant(0, VT); 3714263508Sdim return DAG.getNode(ISD::SHL, SDLoc(N0), VT, 3715263508Sdim DAG.getNode(N0.getOpcode(), SDLoc(N0), VT, 3716218893Sdim N0.getOperand(0)->getOperand(0)), 3717218893Sdim DAG.getConstant(c1 + c2, N1.getValueType())); 3718218893Sdim } 3719218893Sdim } 3720218893Sdim 3721263508Sdim // fold (shl (zext (srl x, C)), C) -> (zext (shl (srl x, C), C)) 3722263508Sdim // Only fold this if the inner zext has no other uses to avoid increasing 3723263508Sdim // the total number of instructions. 3724263508Sdim if (N1C && N0.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse() && 3725263508Sdim N0.getOperand(0).getOpcode() == ISD::SRL && 3726263508Sdim isa<ConstantSDNode>(N0.getOperand(0)->getOperand(1))) { 3727263508Sdim uint64_t c1 = 3728263508Sdim cast<ConstantSDNode>(N0.getOperand(0)->getOperand(1))->getZExtValue(); 3729263508Sdim if (c1 < VT.getSizeInBits()) { 3730263508Sdim uint64_t c2 = N1C->getZExtValue(); 3731263508Sdim if (c1 == c2) { 3732263508Sdim SDValue NewOp0 = N0.getOperand(0); 3733263508Sdim EVT CountVT = NewOp0.getOperand(1).getValueType(); 3734263508Sdim SDValue NewSHL = DAG.getNode(ISD::SHL, SDLoc(N), NewOp0.getValueType(), 3735263508Sdim NewOp0, DAG.getConstant(c2, CountVT)); 3736263508Sdim AddToWorkList(NewSHL.getNode()); 3737263508Sdim return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL); 3738263508Sdim } 3739263508Sdim } 3740263508Sdim } 3741263508Sdim 3742223017Sdim // fold (shl (srl x, c1), c2) -> (and (shl x, (sub c2, c1), MASK) or 3743223017Sdim // (and (srl x, (sub c1, c2), MASK) 3744234353Sdim // Only fold this if the inner shift has no other uses -- if it does, folding 3745234353Sdim // this will increase the total number of instructions. 3746234353Sdim if (N1C && N0.getOpcode() == ISD::SRL && N0.hasOneUse() && 3747193323Sed N0.getOperand(1).getOpcode() == ISD::Constant) { 3748193323Sed uint64_t c1 = cast<ConstantSDNode>(N0.getOperand(1))->getZExtValue(); 3749198090Srdivacky if (c1 < VT.getSizeInBits()) { 3750198090Srdivacky uint64_t c2 = N1C->getZExtValue(); 3751223017Sdim APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(), 3752223017Sdim VT.getSizeInBits() - c1); 3753223017Sdim SDValue Shift; 3754223017Sdim if (c2 > c1) { 3755223017Sdim Mask = Mask.shl(c2-c1); 3756263508Sdim Shift = DAG.getNode(ISD::SHL, SDLoc(N), VT, N0.getOperand(0), 3757223017Sdim DAG.getConstant(c2-c1, N1.getValueType())); 3758223017Sdim } else { 3759223017Sdim Mask = Mask.lshr(c1-c2); 3760263508Sdim Shift = DAG.getNode(ISD::SRL, SDLoc(N), VT, N0.getOperand(0), 3761223017Sdim DAG.getConstant(c1-c2, N1.getValueType())); 3762223017Sdim } 3763263508Sdim return DAG.getNode(ISD::AND, SDLoc(N0), VT, Shift, 3764223017Sdim DAG.getConstant(Mask, VT)); 3765198090Srdivacky } 3766193323Sed } 3767193323Sed // fold (shl (sra x, c1), c1) -> (and x, (shl -1, c1)) 3768198090Srdivacky if (N1C && N0.getOpcode() == ISD::SRA && N1 == N0.getOperand(1)) { 3769198090Srdivacky SDValue HiBitsMask = 3770198090Srdivacky DAG.getConstant(APInt::getHighBitsSet(VT.getSizeInBits(), 3771198090Srdivacky VT.getSizeInBits() - 3772198090Srdivacky N1C->getZExtValue()), 3773198090Srdivacky VT); 3774263508Sdim return DAG.getNode(ISD::AND, SDLoc(N), VT, N0.getOperand(0), 3775198090Srdivacky HiBitsMask); 3776198090Srdivacky } 3777193323Sed 3778207618Srdivacky if (N1C) { 3779207618Srdivacky SDValue NewSHL = visitShiftByConstant(N, N1C->getZExtValue()); 3780207618Srdivacky if (NewSHL.getNode()) 3781207618Srdivacky return NewSHL; 3782207618Srdivacky } 3783207618Srdivacky 3784207618Srdivacky return SDValue(); 3785193323Sed} 3786193323Sed 3787193323SedSDValue DAGCombiner::visitSRA(SDNode *N) { 3788193323Sed SDValue N0 = N->getOperand(0); 3789193323Sed SDValue N1 = N->getOperand(1); 3790193323Sed ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0); 3791193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 3792198090Srdivacky EVT VT = N0.getValueType(); 3793200581Srdivacky unsigned OpSizeInBits = VT.getScalarType().getSizeInBits(); 3794193323Sed 3795263508Sdim // fold vector ops 3796263508Sdim if (VT.isVector()) { 3797263508Sdim SDValue FoldedVOp = SimplifyVBinOp(N); 3798263508Sdim if (FoldedVOp.getNode()) return FoldedVOp; 3799263508Sdim } 3800263508Sdim 3801193323Sed // fold (sra c1, c2) -> (sra c1, c2) 3802193323Sed if (N0C && N1C) 3803193323Sed return DAG.FoldConstantArithmetic(ISD::SRA, VT, N0C, N1C); 3804193323Sed // fold (sra 0, x) -> 0 3805193323Sed if (N0C && N0C->isNullValue()) 3806193323Sed return N0; 3807193323Sed // fold (sra -1, x) -> -1 3808193323Sed if (N0C && N0C->isAllOnesValue()) 3809193323Sed return N0; 3810193323Sed // fold (sra x, (setge c, size(x))) -> undef 3811200581Srdivacky if (N1C && N1C->getZExtValue() >= OpSizeInBits) 3812193323Sed return DAG.getUNDEF(VT); 3813193323Sed // fold (sra x, 0) -> x 3814193323Sed if (N1C && N1C->isNullValue()) 3815193323Sed return N0; 3816193323Sed // fold (sra (shl x, c1), c1) -> sext_inreg for some c1 and target supports 3817193323Sed // sext_inreg. 3818193323Sed if (N1C && N0.getOpcode() == ISD::SHL && N1 == N0.getOperand(1)) { 3819200581Srdivacky unsigned LowBits = OpSizeInBits - (unsigned)N1C->getZExtValue(); 3820202375Srdivacky EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), LowBits); 3821202375Srdivacky if (VT.isVector()) 3822202375Srdivacky ExtVT = EVT::getVectorVT(*DAG.getContext(), 3823202375Srdivacky ExtVT, VT.getVectorNumElements()); 3824202375Srdivacky if ((!LegalOperations || 3825202375Srdivacky TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, ExtVT))) 3826263508Sdim return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, 3827202375Srdivacky N0.getOperand(0), DAG.getValueType(ExtVT)); 3828193323Sed } 3829193323Sed 3830193323Sed // fold (sra (sra x, c1), c2) -> (sra x, (add c1, c2)) 3831193323Sed if (N1C && N0.getOpcode() == ISD::SRA) { 3832193323Sed if (ConstantSDNode *C1 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) { 3833193323Sed unsigned Sum = N1C->getZExtValue() + C1->getZExtValue(); 3834200581Srdivacky if (Sum >= OpSizeInBits) Sum = OpSizeInBits-1; 3835263508Sdim return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0.getOperand(0), 3836193323Sed DAG.getConstant(Sum, N1C->getValueType(0))); 3837193323Sed } 3838193323Sed } 3839193323Sed 3840193323Sed // fold (sra (shl X, m), (sub result_size, n)) 3841193323Sed // -> (sign_extend (trunc (shl X, (sub (sub result_size, n), m)))) for 3842193323Sed // result_size - n != m. 3843193323Sed // If truncate is free for the target sext(shl) is likely to result in better 3844193323Sed // code. 3845193323Sed if (N0.getOpcode() == ISD::SHL) { 3846193323Sed // Get the two constanst of the shifts, CN0 = m, CN = n. 3847193323Sed const ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1)); 3848193323Sed if (N01C && N1C) { 3849193323Sed // Determine what the truncate's result bitsize and type would be. 3850198090Srdivacky EVT TruncVT = 3851218893Sdim EVT::getIntegerVT(*DAG.getContext(), 3852218893Sdim OpSizeInBits - N1C->getZExtValue()); 3853193323Sed // Determine the residual right-shift amount. 3854193323Sed signed ShiftAmt = N1C->getZExtValue() - N01C->getZExtValue(); 3855193323Sed 3856193323Sed // If the shift is not a no-op (in which case this should be just a sign 3857193323Sed // extend already), the truncated to type is legal, sign_extend is legal 3858203954Srdivacky // on that type, and the truncate to that type is both legal and free, 3859193323Sed // perform the transform. 3860193323Sed if ((ShiftAmt > 0) && 3861193323Sed TLI.isOperationLegalOrCustom(ISD::SIGN_EXTEND, TruncVT) && 3862193323Sed TLI.isOperationLegalOrCustom(ISD::TRUNCATE, VT) && 3863193323Sed TLI.isTruncateFree(VT, TruncVT)) { 3864193323Sed 3865219077Sdim SDValue Amt = DAG.getConstant(ShiftAmt, 3866219077Sdim getShiftAmountTy(N0.getOperand(0).getValueType())); 3867263508Sdim SDValue Shift = DAG.getNode(ISD::SRL, SDLoc(N0), VT, 3868193323Sed N0.getOperand(0), Amt); 3869263508Sdim SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), TruncVT, 3870193323Sed Shift); 3871263508Sdim return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), 3872193323Sed N->getValueType(0), Trunc); 3873193323Sed } 3874193323Sed } 3875193323Sed } 3876193323Sed 3877193323Sed // fold (sra x, (trunc (and y, c))) -> (sra x, (and (trunc y), (trunc c))). 3878193323Sed if (N1.getOpcode() == ISD::TRUNCATE && 3879193323Sed N1.getOperand(0).getOpcode() == ISD::AND && 3880193323Sed N1.hasOneUse() && N1.getOperand(0).hasOneUse()) { 3881193323Sed SDValue N101 = N1.getOperand(0).getOperand(1); 3882193323Sed if (ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N101)) { 3883198090Srdivacky EVT TruncVT = N1.getValueType(); 3884193323Sed SDValue N100 = N1.getOperand(0).getOperand(0); 3885193323Sed APInt TruncC = N101C->getAPIntValue(); 3886218893Sdim TruncC = TruncC.trunc(TruncVT.getScalarType().getSizeInBits()); 3887263508Sdim return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0, 3888263508Sdim DAG.getNode(ISD::AND, SDLoc(N), 3889193323Sed TruncVT, 3890193323Sed DAG.getNode(ISD::TRUNCATE, 3891263508Sdim SDLoc(N), 3892193323Sed TruncVT, N100), 3893193323Sed DAG.getConstant(TruncC, TruncVT))); 3894193323Sed } 3895193323Sed } 3896193323Sed 3897218893Sdim // fold (sra (trunc (sr x, c1)), c2) -> (trunc (sra x, c1+c2)) 3898218893Sdim // if c1 is equal to the number of bits the trunc removes 3899218893Sdim if (N0.getOpcode() == ISD::TRUNCATE && 3900218893Sdim (N0.getOperand(0).getOpcode() == ISD::SRL || 3901218893Sdim N0.getOperand(0).getOpcode() == ISD::SRA) && 3902218893Sdim N0.getOperand(0).hasOneUse() && 3903218893Sdim N0.getOperand(0).getOperand(1).hasOneUse() && 3904218893Sdim N1C && isa<ConstantSDNode>(N0.getOperand(0).getOperand(1))) { 3905218893Sdim EVT LargeVT = N0.getOperand(0).getValueType(); 3906218893Sdim ConstantSDNode *LargeShiftAmt = 3907218893Sdim cast<ConstantSDNode>(N0.getOperand(0).getOperand(1)); 3908218893Sdim 3909218893Sdim if (LargeVT.getScalarType().getSizeInBits() - OpSizeInBits == 3910218893Sdim LargeShiftAmt->getZExtValue()) { 3911218893Sdim SDValue Amt = 3912218893Sdim DAG.getConstant(LargeShiftAmt->getZExtValue() + N1C->getZExtValue(), 3913219077Sdim getShiftAmountTy(N0.getOperand(0).getOperand(0).getValueType())); 3914263508Sdim SDValue SRA = DAG.getNode(ISD::SRA, SDLoc(N), LargeVT, 3915218893Sdim N0.getOperand(0).getOperand(0), Amt); 3916263508Sdim return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, SRA); 3917218893Sdim } 3918218893Sdim } 3919218893Sdim 3920193323Sed // Simplify, based on bits shifted out of the LHS. 3921193323Sed if (N1C && SimplifyDemandedBits(SDValue(N, 0))) 3922193323Sed return SDValue(N, 0); 3923193323Sed 3924193323Sed 3925193323Sed // If the sign bit is known to be zero, switch this to a SRL. 3926193323Sed if (DAG.SignBitIsZero(N0)) 3927263508Sdim return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, N1); 3928193323Sed 3929207618Srdivacky if (N1C) { 3930207618Srdivacky SDValue NewSRA = visitShiftByConstant(N, N1C->getZExtValue()); 3931207618Srdivacky if (NewSRA.getNode()) 3932207618Srdivacky return NewSRA; 3933207618Srdivacky } 3934207618Srdivacky 3935207618Srdivacky return SDValue(); 3936193323Sed} 3937193323Sed 3938193323SedSDValue DAGCombiner::visitSRL(SDNode *N) { 3939193323Sed SDValue N0 = N->getOperand(0); 3940193323Sed SDValue N1 = N->getOperand(1); 3941193323Sed ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0); 3942193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 3943198090Srdivacky EVT VT = N0.getValueType(); 3944200581Srdivacky unsigned OpSizeInBits = VT.getScalarType().getSizeInBits(); 3945193323Sed 3946263508Sdim // fold vector ops 3947263508Sdim if (VT.isVector()) { 3948263508Sdim SDValue FoldedVOp = SimplifyVBinOp(N); 3949263508Sdim if (FoldedVOp.getNode()) return FoldedVOp; 3950263508Sdim } 3951263508Sdim 3952193323Sed // fold (srl c1, c2) -> c1 >>u c2 3953193323Sed if (N0C && N1C) 3954193323Sed return DAG.FoldConstantArithmetic(ISD::SRL, VT, N0C, N1C); 3955193323Sed // fold (srl 0, x) -> 0 3956193323Sed if (N0C && N0C->isNullValue()) 3957193323Sed return N0; 3958193323Sed // fold (srl x, c >= size(x)) -> undef 3959193323Sed if (N1C && N1C->getZExtValue() >= OpSizeInBits) 3960193323Sed return DAG.getUNDEF(VT); 3961193323Sed // fold (srl x, 0) -> x 3962193323Sed if (N1C && N1C->isNullValue()) 3963193323Sed return N0; 3964193323Sed // if (srl x, c) is known to be zero, return 0 3965193323Sed if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0), 3966193323Sed APInt::getAllOnesValue(OpSizeInBits))) 3967193323Sed return DAG.getConstant(0, VT); 3968193323Sed 3969193323Sed // fold (srl (srl x, c1), c2) -> 0 or (srl x, (add c1, c2)) 3970193323Sed if (N1C && N0.getOpcode() == ISD::SRL && 3971193323Sed N0.getOperand(1).getOpcode() == ISD::Constant) { 3972193323Sed uint64_t c1 = cast<ConstantSDNode>(N0.getOperand(1))->getZExtValue(); 3973193323Sed uint64_t c2 = N1C->getZExtValue(); 3974218893Sdim if (c1 + c2 >= OpSizeInBits) 3975193323Sed return DAG.getConstant(0, VT); 3976263508Sdim return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0.getOperand(0), 3977193323Sed DAG.getConstant(c1 + c2, N1.getValueType())); 3978193323Sed } 3979218893Sdim 3980218893Sdim // fold (srl (trunc (srl x, c1)), c2) -> 0 or (trunc (srl x, (add c1, c2))) 3981218893Sdim if (N1C && N0.getOpcode() == ISD::TRUNCATE && 3982218893Sdim N0.getOperand(0).getOpcode() == ISD::SRL && 3983218893Sdim isa<ConstantSDNode>(N0.getOperand(0)->getOperand(1))) { 3984219077Sdim uint64_t c1 = 3985218893Sdim cast<ConstantSDNode>(N0.getOperand(0)->getOperand(1))->getZExtValue(); 3986218893Sdim uint64_t c2 = N1C->getZExtValue(); 3987218893Sdim EVT InnerShiftVT = N0.getOperand(0).getValueType(); 3988218893Sdim EVT ShiftCountVT = N0.getOperand(0)->getOperand(1).getValueType(); 3989218893Sdim uint64_t InnerShiftSize = InnerShiftVT.getScalarType().getSizeInBits(); 3990218893Sdim // This is only valid if the OpSizeInBits + c1 = size of inner shift. 3991218893Sdim if (c1 + OpSizeInBits == InnerShiftSize) { 3992218893Sdim if (c1 + c2 >= InnerShiftSize) 3993218893Sdim return DAG.getConstant(0, VT); 3994263508Sdim return DAG.getNode(ISD::TRUNCATE, SDLoc(N0), VT, 3995263508Sdim DAG.getNode(ISD::SRL, SDLoc(N0), InnerShiftVT, 3996218893Sdim N0.getOperand(0)->getOperand(0), 3997218893Sdim DAG.getConstant(c1 + c2, ShiftCountVT))); 3998218893Sdim } 3999218893Sdim } 4000218893Sdim 4001207618Srdivacky // fold (srl (shl x, c), c) -> (and x, cst2) 4002207618Srdivacky if (N1C && N0.getOpcode() == ISD::SHL && N0.getOperand(1) == N1 && 4003207618Srdivacky N0.getValueSizeInBits() <= 64) { 4004207618Srdivacky uint64_t ShAmt = N1C->getZExtValue()+64-N0.getValueSizeInBits(); 4005263508Sdim return DAG.getNode(ISD::AND, SDLoc(N), VT, N0.getOperand(0), 4006207618Srdivacky DAG.getConstant(~0ULL >> ShAmt, VT)); 4007207618Srdivacky } 4008193323Sed 4009263508Sdim // fold (srl (anyextend x), c) -> (and (anyextend (srl x, c)), mask) 4010193323Sed if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) { 4011193323Sed // Shifting in all undef bits? 4012198090Srdivacky EVT SmallVT = N0.getOperand(0).getValueType(); 4013193323Sed if (N1C->getZExtValue() >= SmallVT.getSizeInBits()) 4014193323Sed return DAG.getUNDEF(VT); 4015193323Sed 4016207618Srdivacky if (!LegalTypes || TLI.isTypeDesirableForOp(ISD::SRL, SmallVT)) { 4017221345Sdim uint64_t ShiftAmt = N1C->getZExtValue(); 4018263508Sdim SDValue SmallShift = DAG.getNode(ISD::SRL, SDLoc(N0), SmallVT, 4019221345Sdim N0.getOperand(0), 4020221345Sdim DAG.getConstant(ShiftAmt, getShiftAmountTy(SmallVT))); 4021207618Srdivacky AddToWorkList(SmallShift.getNode()); 4022263508Sdim APInt Mask = APInt::getAllOnesValue(VT.getSizeInBits()).lshr(ShiftAmt); 4023263508Sdim return DAG.getNode(ISD::AND, SDLoc(N), VT, 4024263508Sdim DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, SmallShift), 4025263508Sdim DAG.getConstant(Mask, VT)); 4026207618Srdivacky } 4027193323Sed } 4028193323Sed 4029193323Sed // fold (srl (sra X, Y), 31) -> (srl X, 31). This srl only looks at the sign 4030193323Sed // bit, which is unmodified by sra. 4031193323Sed if (N1C && N1C->getZExtValue() + 1 == VT.getSizeInBits()) { 4032193323Sed if (N0.getOpcode() == ISD::SRA) 4033263508Sdim return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0.getOperand(0), N1); 4034193323Sed } 4035193323Sed 4036193323Sed // fold (srl (ctlz x), "5") -> x iff x has one bit set (the low bit). 4037193323Sed if (N1C && N0.getOpcode() == ISD::CTLZ && 4038193323Sed N1C->getAPIntValue() == Log2_32(VT.getSizeInBits())) { 4039193323Sed APInt KnownZero, KnownOne; 4040234353Sdim DAG.ComputeMaskedBits(N0.getOperand(0), KnownZero, KnownOne); 4041193323Sed 4042193323Sed // If any of the input bits are KnownOne, then the input couldn't be all 4043193323Sed // zeros, thus the result of the srl will always be zero. 4044193323Sed if (KnownOne.getBoolValue()) return DAG.getConstant(0, VT); 4045193323Sed 4046193323Sed // If all of the bits input the to ctlz node are known to be zero, then 4047193323Sed // the result of the ctlz is "32" and the result of the shift is one. 4048234353Sdim APInt UnknownBits = ~KnownZero; 4049193323Sed if (UnknownBits == 0) return DAG.getConstant(1, VT); 4050193323Sed 4051193323Sed // Otherwise, check to see if there is exactly one bit input to the ctlz. 4052193323Sed if ((UnknownBits & (UnknownBits - 1)) == 0) { 4053193323Sed // Okay, we know that only that the single bit specified by UnknownBits 4054193323Sed // could be set on input to the CTLZ node. If this bit is set, the SRL 4055193323Sed // will return 0, if it is clear, it returns 1. Change the CTLZ/SRL pair 4056193323Sed // to an SRL/XOR pair, which is likely to simplify more. 4057193323Sed unsigned ShAmt = UnknownBits.countTrailingZeros(); 4058193323Sed SDValue Op = N0.getOperand(0); 4059193323Sed 4060193323Sed if (ShAmt) { 4061263508Sdim Op = DAG.getNode(ISD::SRL, SDLoc(N0), VT, Op, 4062219077Sdim DAG.getConstant(ShAmt, getShiftAmountTy(Op.getValueType()))); 4063193323Sed AddToWorkList(Op.getNode()); 4064193323Sed } 4065193323Sed 4066263508Sdim return DAG.getNode(ISD::XOR, SDLoc(N), VT, 4067193323Sed Op, DAG.getConstant(1, VT)); 4068193323Sed } 4069193323Sed } 4070193323Sed 4071193323Sed // fold (srl x, (trunc (and y, c))) -> (srl x, (and (trunc y), (trunc c))). 4072193323Sed if (N1.getOpcode() == ISD::TRUNCATE && 4073193323Sed N1.getOperand(0).getOpcode() == ISD::AND && 4074193323Sed N1.hasOneUse() && N1.getOperand(0).hasOneUse()) { 4075193323Sed SDValue N101 = N1.getOperand(0).getOperand(1); 4076193323Sed if (ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N101)) { 4077198090Srdivacky EVT TruncVT = N1.getValueType(); 4078193323Sed SDValue N100 = N1.getOperand(0).getOperand(0); 4079193323Sed APInt TruncC = N101C->getAPIntValue(); 4080218893Sdim TruncC = TruncC.trunc(TruncVT.getSizeInBits()); 4081263508Sdim return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, 4082263508Sdim DAG.getNode(ISD::AND, SDLoc(N), 4083193323Sed TruncVT, 4084193323Sed DAG.getNode(ISD::TRUNCATE, 4085263508Sdim SDLoc(N), 4086193323Sed TruncVT, N100), 4087193323Sed DAG.getConstant(TruncC, TruncVT))); 4088193323Sed } 4089193323Sed } 4090193323Sed 4091193323Sed // fold operands of srl based on knowledge that the low bits are not 4092193323Sed // demanded. 4093193323Sed if (N1C && SimplifyDemandedBits(SDValue(N, 0))) 4094193323Sed return SDValue(N, 0); 4095193323Sed 4096201360Srdivacky if (N1C) { 4097201360Srdivacky SDValue NewSRL = visitShiftByConstant(N, N1C->getZExtValue()); 4098201360Srdivacky if (NewSRL.getNode()) 4099201360Srdivacky return NewSRL; 4100201360Srdivacky } 4101201360Srdivacky 4102210299Sed // Attempt to convert a srl of a load into a narrower zero-extending load. 4103210299Sed SDValue NarrowLoad = ReduceLoadWidth(N); 4104210299Sed if (NarrowLoad.getNode()) 4105210299Sed return NarrowLoad; 4106210299Sed 4107201360Srdivacky // Here is a common situation. We want to optimize: 4108201360Srdivacky // 4109201360Srdivacky // %a = ... 4110201360Srdivacky // %b = and i32 %a, 2 4111201360Srdivacky // %c = srl i32 %b, 1 4112201360Srdivacky // brcond i32 %c ... 4113201360Srdivacky // 4114201360Srdivacky // into 4115218893Sdim // 4116201360Srdivacky // %a = ... 4117201360Srdivacky // %b = and %a, 2 4118201360Srdivacky // %c = setcc eq %b, 0 4119201360Srdivacky // brcond %c ... 4120201360Srdivacky // 4121201360Srdivacky // However when after the source operand of SRL is optimized into AND, the SRL 4122201360Srdivacky // itself may not be optimized further. Look for it and add the BRCOND into 4123201360Srdivacky // the worklist. 4124202375Srdivacky if (N->hasOneUse()) { 4125202375Srdivacky SDNode *Use = *N->use_begin(); 4126202375Srdivacky if (Use->getOpcode() == ISD::BRCOND) 4127202375Srdivacky AddToWorkList(Use); 4128202375Srdivacky else if (Use->getOpcode() == ISD::TRUNCATE && Use->hasOneUse()) { 4129202375Srdivacky // Also look pass the truncate. 4130202375Srdivacky Use = *Use->use_begin(); 4131202375Srdivacky if (Use->getOpcode() == ISD::BRCOND) 4132202375Srdivacky AddToWorkList(Use); 4133202375Srdivacky } 4134202375Srdivacky } 4135201360Srdivacky 4136201360Srdivacky return SDValue(); 4137193323Sed} 4138193323Sed 4139193323SedSDValue DAGCombiner::visitCTLZ(SDNode *N) { 4140193323Sed SDValue N0 = N->getOperand(0); 4141198090Srdivacky EVT VT = N->getValueType(0); 4142193323Sed 4143193323Sed // fold (ctlz c1) -> c2 4144193323Sed if (isa<ConstantSDNode>(N0)) 4145263508Sdim return DAG.getNode(ISD::CTLZ, SDLoc(N), VT, N0); 4146193323Sed return SDValue(); 4147193323Sed} 4148193323Sed 4149234353SdimSDValue DAGCombiner::visitCTLZ_ZERO_UNDEF(SDNode *N) { 4150234353Sdim SDValue N0 = N->getOperand(0); 4151234353Sdim EVT VT = N->getValueType(0); 4152234353Sdim 4153234353Sdim // fold (ctlz_zero_undef c1) -> c2 4154234353Sdim if (isa<ConstantSDNode>(N0)) 4155263508Sdim return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SDLoc(N), VT, N0); 4156234353Sdim return SDValue(); 4157234353Sdim} 4158234353Sdim 4159193323SedSDValue DAGCombiner::visitCTTZ(SDNode *N) { 4160193323Sed SDValue N0 = N->getOperand(0); 4161198090Srdivacky EVT VT = N->getValueType(0); 4162193323Sed 4163193323Sed // fold (cttz c1) -> c2 4164193323Sed if (isa<ConstantSDNode>(N0)) 4165263508Sdim return DAG.getNode(ISD::CTTZ, SDLoc(N), VT, N0); 4166193323Sed return SDValue(); 4167193323Sed} 4168193323Sed 4169234353SdimSDValue DAGCombiner::visitCTTZ_ZERO_UNDEF(SDNode *N) { 4170234353Sdim SDValue N0 = N->getOperand(0); 4171234353Sdim EVT VT = N->getValueType(0); 4172234353Sdim 4173234353Sdim // fold (cttz_zero_undef c1) -> c2 4174234353Sdim if (isa<ConstantSDNode>(N0)) 4175263508Sdim return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0); 4176234353Sdim return SDValue(); 4177234353Sdim} 4178234353Sdim 4179193323SedSDValue DAGCombiner::visitCTPOP(SDNode *N) { 4180193323Sed SDValue N0 = N->getOperand(0); 4181198090Srdivacky EVT VT = N->getValueType(0); 4182193323Sed 4183193323Sed // fold (ctpop c1) -> c2 4184193323Sed if (isa<ConstantSDNode>(N0)) 4185263508Sdim return DAG.getNode(ISD::CTPOP, SDLoc(N), VT, N0); 4186193323Sed return SDValue(); 4187193323Sed} 4188193323Sed 4189193323SedSDValue DAGCombiner::visitSELECT(SDNode *N) { 4190193323Sed SDValue N0 = N->getOperand(0); 4191193323Sed SDValue N1 = N->getOperand(1); 4192193323Sed SDValue N2 = N->getOperand(2); 4193193323Sed ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0); 4194193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 4195193323Sed ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2); 4196198090Srdivacky EVT VT = N->getValueType(0); 4197198090Srdivacky EVT VT0 = N0.getValueType(); 4198193323Sed 4199193323Sed // fold (select C, X, X) -> X 4200193323Sed if (N1 == N2) 4201193323Sed return N1; 4202193323Sed // fold (select true, X, Y) -> X 4203193323Sed if (N0C && !N0C->isNullValue()) 4204193323Sed return N1; 4205193323Sed // fold (select false, X, Y) -> Y 4206193323Sed if (N0C && N0C->isNullValue()) 4207193323Sed return N2; 4208193323Sed // fold (select C, 1, X) -> (or C, X) 4209193323Sed if (VT == MVT::i1 && N1C && N1C->getAPIntValue() == 1) 4210263508Sdim return DAG.getNode(ISD::OR, SDLoc(N), VT, N0, N2); 4211193323Sed // fold (select C, 0, 1) -> (xor C, 1) 4212193323Sed if (VT.isInteger() && 4213193323Sed (VT0 == MVT::i1 || 4214193323Sed (VT0.isInteger() && 4215243830Sdim TLI.getBooleanContents(false) == 4216243830Sdim TargetLowering::ZeroOrOneBooleanContent)) && 4217193323Sed N1C && N2C && N1C->isNullValue() && N2C->getAPIntValue() == 1) { 4218193323Sed SDValue XORNode; 4219193323Sed if (VT == VT0) 4220263508Sdim return DAG.getNode(ISD::XOR, SDLoc(N), VT0, 4221193323Sed N0, DAG.getConstant(1, VT0)); 4222263508Sdim XORNode = DAG.getNode(ISD::XOR, SDLoc(N0), VT0, 4223193323Sed N0, DAG.getConstant(1, VT0)); 4224193323Sed AddToWorkList(XORNode.getNode()); 4225193323Sed if (VT.bitsGT(VT0)) 4226263508Sdim return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, XORNode); 4227263508Sdim return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, XORNode); 4228193323Sed } 4229193323Sed // fold (select C, 0, X) -> (and (not C), X) 4230193323Sed if (VT == VT0 && VT == MVT::i1 && N1C && N1C->isNullValue()) { 4231263508Sdim SDValue NOTNode = DAG.getNOT(SDLoc(N0), N0, VT); 4232193323Sed AddToWorkList(NOTNode.getNode()); 4233263508Sdim return DAG.getNode(ISD::AND, SDLoc(N), VT, NOTNode, N2); 4234193323Sed } 4235193323Sed // fold (select C, X, 1) -> (or (not C), X) 4236193323Sed if (VT == VT0 && VT == MVT::i1 && N2C && N2C->getAPIntValue() == 1) { 4237263508Sdim SDValue NOTNode = DAG.getNOT(SDLoc(N0), N0, VT); 4238193323Sed AddToWorkList(NOTNode.getNode()); 4239263508Sdim return DAG.getNode(ISD::OR, SDLoc(N), VT, NOTNode, N1); 4240193323Sed } 4241193323Sed // fold (select C, X, 0) -> (and C, X) 4242193323Sed if (VT == MVT::i1 && N2C && N2C->isNullValue()) 4243263508Sdim return DAG.getNode(ISD::AND, SDLoc(N), VT, N0, N1); 4244193323Sed // fold (select X, X, Y) -> (or X, Y) 4245193323Sed // fold (select X, 1, Y) -> (or X, Y) 4246193323Sed if (VT == MVT::i1 && (N0 == N1 || (N1C && N1C->getAPIntValue() == 1))) 4247263508Sdim return DAG.getNode(ISD::OR, SDLoc(N), VT, N0, N2); 4248193323Sed // fold (select X, Y, X) -> (and X, Y) 4249193323Sed // fold (select X, Y, 0) -> (and X, Y) 4250193323Sed if (VT == MVT::i1 && (N0 == N2 || (N2C && N2C->getAPIntValue() == 0))) 4251263508Sdim return DAG.getNode(ISD::AND, SDLoc(N), VT, N0, N1); 4252193323Sed 4253193323Sed // If we can fold this based on the true/false value, do so. 4254193323Sed if (SimplifySelectOps(N, N1, N2)) 4255193323Sed return SDValue(N, 0); // Don't revisit N. 4256193323Sed 4257193323Sed // fold selects based on a setcc into other things, such as min/max/abs 4258193323Sed if (N0.getOpcode() == ISD::SETCC) { 4259193323Sed // FIXME: 4260193323Sed // Check against MVT::Other for SELECT_CC, which is a workaround for targets 4261193323Sed // having to say they don't support SELECT_CC on every type the DAG knows 4262193323Sed // about, since there is no way to mark an opcode illegal at all value types 4263198090Srdivacky if (TLI.isOperationLegalOrCustom(ISD::SELECT_CC, MVT::Other) && 4264198090Srdivacky TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT)) 4265263508Sdim return DAG.getNode(ISD::SELECT_CC, SDLoc(N), VT, 4266193323Sed N0.getOperand(0), N0.getOperand(1), 4267193323Sed N1, N2, N0.getOperand(2)); 4268263508Sdim return SimplifySelect(SDLoc(N), N0, N1, N2); 4269193323Sed } 4270193323Sed 4271193323Sed return SDValue(); 4272193323Sed} 4273193323Sed 4274263508Sdimstatic 4275263508Sdimstd::pair<SDValue, SDValue> SplitVSETCC(const SDNode *N, SelectionDAG &DAG) { 4276263508Sdim SDLoc DL(N); 4277263508Sdim EVT LoVT, HiVT; 4278263508Sdim llvm::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 4279263508Sdim 4280263508Sdim // Split the inputs. 4281263508Sdim SDValue Lo, Hi, LL, LH, RL, RH; 4282263508Sdim llvm::tie(LL, LH) = DAG.SplitVectorOperand(N, 0); 4283263508Sdim llvm::tie(RL, RH) = DAG.SplitVectorOperand(N, 1); 4284263508Sdim 4285263508Sdim Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2)); 4286263508Sdim Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2)); 4287263508Sdim 4288263508Sdim return std::make_pair(Lo, Hi); 4289263508Sdim} 4290263508Sdim 4291251662SdimSDValue DAGCombiner::visitVSELECT(SDNode *N) { 4292251662Sdim SDValue N0 = N->getOperand(0); 4293251662Sdim SDValue N1 = N->getOperand(1); 4294251662Sdim SDValue N2 = N->getOperand(2); 4295263508Sdim SDLoc DL(N); 4296251662Sdim 4297251662Sdim // Canonicalize integer abs. 4298251662Sdim // vselect (setg[te] X, 0), X, -X -> 4299251662Sdim // vselect (setgt X, -1), X, -X -> 4300251662Sdim // vselect (setl[te] X, 0), -X, X -> 4301251662Sdim // Y = sra (X, size(X)-1); xor (add (X, Y), Y) 4302251662Sdim if (N0.getOpcode() == ISD::SETCC) { 4303251662Sdim SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1); 4304251662Sdim ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); 4305251662Sdim bool isAbs = false; 4306251662Sdim bool RHSIsAllZeros = ISD::isBuildVectorAllZeros(RHS.getNode()); 4307251662Sdim 4308251662Sdim if (((RHSIsAllZeros && (CC == ISD::SETGT || CC == ISD::SETGE)) || 4309251662Sdim (ISD::isBuildVectorAllOnes(RHS.getNode()) && CC == ISD::SETGT)) && 4310251662Sdim N1 == LHS && N2.getOpcode() == ISD::SUB && N1 == N2.getOperand(1)) 4311251662Sdim isAbs = ISD::isBuildVectorAllZeros(N2.getOperand(0).getNode()); 4312251662Sdim else if ((RHSIsAllZeros && (CC == ISD::SETLT || CC == ISD::SETLE)) && 4313251662Sdim N2 == LHS && N1.getOpcode() == ISD::SUB && N2 == N1.getOperand(1)) 4314251662Sdim isAbs = ISD::isBuildVectorAllZeros(N1.getOperand(0).getNode()); 4315251662Sdim 4316251662Sdim if (isAbs) { 4317251662Sdim EVT VT = LHS.getValueType(); 4318251662Sdim SDValue Shift = DAG.getNode( 4319251662Sdim ISD::SRA, DL, VT, LHS, 4320251662Sdim DAG.getConstant(VT.getScalarType().getSizeInBits() - 1, VT)); 4321251662Sdim SDValue Add = DAG.getNode(ISD::ADD, DL, VT, LHS, Shift); 4322251662Sdim AddToWorkList(Shift.getNode()); 4323251662Sdim AddToWorkList(Add.getNode()); 4324251662Sdim return DAG.getNode(ISD::XOR, DL, VT, Add, Shift); 4325251662Sdim } 4326251662Sdim } 4327251662Sdim 4328263508Sdim // If the VSELECT result requires splitting and the mask is provided by a 4329263508Sdim // SETCC, then split both nodes and its operands before legalization. This 4330263508Sdim // prevents the type legalizer from unrolling SETCC into scalar comparisons 4331263508Sdim // and enables future optimizations (e.g. min/max pattern matching on X86). 4332263508Sdim if (N0.getOpcode() == ISD::SETCC) { 4333263508Sdim EVT VT = N->getValueType(0); 4334263508Sdim 4335263508Sdim // Check if any splitting is required. 4336263508Sdim if (TLI.getTypeAction(*DAG.getContext(), VT) != 4337263508Sdim TargetLowering::TypeSplitVector) 4338263508Sdim return SDValue(); 4339263508Sdim 4340263508Sdim SDValue Lo, Hi, CCLo, CCHi, LL, LH, RL, RH; 4341263508Sdim llvm::tie(CCLo, CCHi) = SplitVSETCC(N0.getNode(), DAG); 4342263508Sdim llvm::tie(LL, LH) = DAG.SplitVectorOperand(N, 1); 4343263508Sdim llvm::tie(RL, RH) = DAG.SplitVectorOperand(N, 2); 4344263508Sdim 4345263508Sdim Lo = DAG.getNode(N->getOpcode(), DL, LL.getValueType(), CCLo, LL, RL); 4346263508Sdim Hi = DAG.getNode(N->getOpcode(), DL, LH.getValueType(), CCHi, LH, RH); 4347263508Sdim 4348263508Sdim // Add the new VSELECT nodes to the work list in case they need to be split 4349263508Sdim // again. 4350263508Sdim AddToWorkList(Lo.getNode()); 4351263508Sdim AddToWorkList(Hi.getNode()); 4352263508Sdim 4353263508Sdim return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi); 4354263508Sdim } 4355263508Sdim 4356251662Sdim return SDValue(); 4357251662Sdim} 4358251662Sdim 4359193323SedSDValue DAGCombiner::visitSELECT_CC(SDNode *N) { 4360193323Sed SDValue N0 = N->getOperand(0); 4361193323Sed SDValue N1 = N->getOperand(1); 4362193323Sed SDValue N2 = N->getOperand(2); 4363193323Sed SDValue N3 = N->getOperand(3); 4364193323Sed SDValue N4 = N->getOperand(4); 4365193323Sed ISD::CondCode CC = cast<CondCodeSDNode>(N4)->get(); 4366193323Sed 4367193323Sed // fold select_cc lhs, rhs, x, x, cc -> x 4368193323Sed if (N2 == N3) 4369193323Sed return N2; 4370193323Sed 4371193323Sed // Determine if the condition we're dealing with is constant 4372263508Sdim SDValue SCC = SimplifySetCC(getSetCCResultType(N0.getValueType()), 4373263508Sdim N0, N1, CC, SDLoc(N), false); 4374263508Sdim if (SCC.getNode()) { 4375263508Sdim AddToWorkList(SCC.getNode()); 4376193323Sed 4377263508Sdim if (ConstantSDNode *SCCC = dyn_cast<ConstantSDNode>(SCC.getNode())) { 4378263508Sdim if (!SCCC->isNullValue()) 4379263508Sdim return N2; // cond always true -> true val 4380263508Sdim else 4381263508Sdim return N3; // cond always false -> false val 4382263508Sdim } 4383263508Sdim 4384263508Sdim // Fold to a simpler select_cc 4385263508Sdim if (SCC.getOpcode() == ISD::SETCC) 4386263508Sdim return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N2.getValueType(), 4387263508Sdim SCC.getOperand(0), SCC.getOperand(1), N2, N3, 4388263508Sdim SCC.getOperand(2)); 4389193323Sed } 4390193323Sed 4391193323Sed // If we can fold this based on the true/false value, do so. 4392193323Sed if (SimplifySelectOps(N, N2, N3)) 4393193323Sed return SDValue(N, 0); // Don't revisit N. 4394193323Sed 4395193323Sed // fold select_cc into other things, such as min/max/abs 4396263508Sdim return SimplifySelectCC(SDLoc(N), N0, N1, N2, N3, CC); 4397193323Sed} 4398193323Sed 4399193323SedSDValue DAGCombiner::visitSETCC(SDNode *N) { 4400193323Sed return SimplifySetCC(N->getValueType(0), N->getOperand(0), N->getOperand(1), 4401193323Sed cast<CondCodeSDNode>(N->getOperand(2))->get(), 4402263508Sdim SDLoc(N)); 4403193323Sed} 4404193323Sed 4405193323Sed// ExtendUsesToFormExtLoad - Trying to extend uses of a load to enable this: 4406193323Sed// "fold ({s|z|a}ext (load x)) -> ({s|z|a}ext (truncate ({s|z|a}extload x)))" 4407193323Sed// transformation. Returns true if extension are possible and the above 4408193323Sed// mentioned transformation is profitable. 4409193323Sedstatic bool ExtendUsesToFormExtLoad(SDNode *N, SDValue N0, 4410193323Sed unsigned ExtOpc, 4411263508Sdim SmallVectorImpl<SDNode *> &ExtendNodes, 4412193323Sed const TargetLowering &TLI) { 4413193323Sed bool HasCopyToRegUses = false; 4414193323Sed bool isTruncFree = TLI.isTruncateFree(N->getValueType(0), N0.getValueType()); 4415193323Sed for (SDNode::use_iterator UI = N0.getNode()->use_begin(), 4416193323Sed UE = N0.getNode()->use_end(); 4417193323Sed UI != UE; ++UI) { 4418193323Sed SDNode *User = *UI; 4419193323Sed if (User == N) 4420193323Sed continue; 4421193323Sed if (UI.getUse().getResNo() != N0.getResNo()) 4422193323Sed continue; 4423193323Sed // FIXME: Only extend SETCC N, N and SETCC N, c for now. 4424193323Sed if (ExtOpc != ISD::ANY_EXTEND && User->getOpcode() == ISD::SETCC) { 4425193323Sed ISD::CondCode CC = cast<CondCodeSDNode>(User->getOperand(2))->get(); 4426193323Sed if (ExtOpc == ISD::ZERO_EXTEND && ISD::isSignedIntSetCC(CC)) 4427193323Sed // Sign bits will be lost after a zext. 4428193323Sed return false; 4429193323Sed bool Add = false; 4430193323Sed for (unsigned i = 0; i != 2; ++i) { 4431193323Sed SDValue UseOp = User->getOperand(i); 4432193323Sed if (UseOp == N0) 4433193323Sed continue; 4434193323Sed if (!isa<ConstantSDNode>(UseOp)) 4435193323Sed return false; 4436193323Sed Add = true; 4437193323Sed } 4438193323Sed if (Add) 4439193323Sed ExtendNodes.push_back(User); 4440193323Sed continue; 4441193323Sed } 4442193323Sed // If truncates aren't free and there are users we can't 4443193323Sed // extend, it isn't worthwhile. 4444193323Sed if (!isTruncFree) 4445193323Sed return false; 4446193323Sed // Remember if this value is live-out. 4447193323Sed if (User->getOpcode() == ISD::CopyToReg) 4448193323Sed HasCopyToRegUses = true; 4449193323Sed } 4450193323Sed 4451193323Sed if (HasCopyToRegUses) { 4452193323Sed bool BothLiveOut = false; 4453193323Sed for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); 4454193323Sed UI != UE; ++UI) { 4455193323Sed SDUse &Use = UI.getUse(); 4456193323Sed if (Use.getResNo() == 0 && Use.getUser()->getOpcode() == ISD::CopyToReg) { 4457193323Sed BothLiveOut = true; 4458193323Sed break; 4459193323Sed } 4460193323Sed } 4461193323Sed if (BothLiveOut) 4462193323Sed // Both unextended and extended values are live out. There had better be 4463218893Sdim // a good reason for the transformation. 4464193323Sed return ExtendNodes.size(); 4465193323Sed } 4466193323Sed return true; 4467193323Sed} 4468193323Sed 4469263508Sdimvoid DAGCombiner::ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs, 4470263508Sdim SDValue Trunc, SDValue ExtLoad, SDLoc DL, 4471224145Sdim ISD::NodeType ExtType) { 4472224145Sdim // Extend SetCC uses if necessary. 4473224145Sdim for (unsigned i = 0, e = SetCCs.size(); i != e; ++i) { 4474224145Sdim SDNode *SetCC = SetCCs[i]; 4475224145Sdim SmallVector<SDValue, 4> Ops; 4476224145Sdim 4477224145Sdim for (unsigned j = 0; j != 2; ++j) { 4478224145Sdim SDValue SOp = SetCC->getOperand(j); 4479224145Sdim if (SOp == Trunc) 4480224145Sdim Ops.push_back(ExtLoad); 4481224145Sdim else 4482224145Sdim Ops.push_back(DAG.getNode(ExtType, DL, ExtLoad->getValueType(0), SOp)); 4483224145Sdim } 4484224145Sdim 4485224145Sdim Ops.push_back(SetCC->getOperand(2)); 4486224145Sdim CombineTo(SetCC, DAG.getNode(ISD::SETCC, DL, SetCC->getValueType(0), 4487224145Sdim &Ops[0], Ops.size())); 4488224145Sdim } 4489224145Sdim} 4490224145Sdim 4491193323SedSDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) { 4492193323Sed SDValue N0 = N->getOperand(0); 4493198090Srdivacky EVT VT = N->getValueType(0); 4494193323Sed 4495193323Sed // fold (sext c1) -> c1 4496193323Sed if (isa<ConstantSDNode>(N0)) 4497263508Sdim return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N0); 4498193323Sed 4499193323Sed // fold (sext (sext x)) -> (sext x) 4500193323Sed // fold (sext (aext x)) -> (sext x) 4501193323Sed if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) 4502263508Sdim return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, 4503193323Sed N0.getOperand(0)); 4504193323Sed 4505193323Sed if (N0.getOpcode() == ISD::TRUNCATE) { 4506193323Sed // fold (sext (truncate (load x))) -> (sext (smaller load x)) 4507193323Sed // fold (sext (truncate (srl (load x), c))) -> (sext (smaller load (x+c/n))) 4508193323Sed SDValue NarrowLoad = ReduceLoadWidth(N0.getNode()); 4509193323Sed if (NarrowLoad.getNode()) { 4510208599Srdivacky SDNode* oye = N0.getNode()->getOperand(0).getNode(); 4511208599Srdivacky if (NarrowLoad.getNode() != N0.getNode()) { 4512193323Sed CombineTo(N0.getNode(), NarrowLoad); 4513208599Srdivacky // CombineTo deleted the truncate, if needed, but not what's under it. 4514208599Srdivacky AddToWorkList(oye); 4515208599Srdivacky } 4516193323Sed return SDValue(N, 0); // Return N so it doesn't get rechecked! 4517193323Sed } 4518193323Sed 4519193323Sed // See if the value being truncated is already sign extended. If so, just 4520193323Sed // eliminate the trunc/sext pair. 4521193323Sed SDValue Op = N0.getOperand(0); 4522202375Srdivacky unsigned OpBits = Op.getValueType().getScalarType().getSizeInBits(); 4523202375Srdivacky unsigned MidBits = N0.getValueType().getScalarType().getSizeInBits(); 4524202375Srdivacky unsigned DestBits = VT.getScalarType().getSizeInBits(); 4525193323Sed unsigned NumSignBits = DAG.ComputeNumSignBits(Op); 4526193323Sed 4527193323Sed if (OpBits == DestBits) { 4528193323Sed // Op is i32, Mid is i8, and Dest is i32. If Op has more than 24 sign 4529193323Sed // bits, it is already ready. 4530193323Sed if (NumSignBits > DestBits-MidBits) 4531193323Sed return Op; 4532193323Sed } else if (OpBits < DestBits) { 4533193323Sed // Op is i32, Mid is i8, and Dest is i64. If Op has more than 24 sign 4534193323Sed // bits, just sext from i32. 4535193323Sed if (NumSignBits > OpBits-MidBits) 4536263508Sdim return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, Op); 4537193323Sed } else { 4538193323Sed // Op is i64, Mid is i8, and Dest is i32. If Op has more than 56 sign 4539193323Sed // bits, just truncate to i32. 4540193323Sed if (NumSignBits > OpBits-MidBits) 4541263508Sdim return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Op); 4542193323Sed } 4543193323Sed 4544193323Sed // fold (sext (truncate x)) -> (sextinreg x). 4545193323Sed if (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, 4546193323Sed N0.getValueType())) { 4547202375Srdivacky if (OpBits < DestBits) 4548263508Sdim Op = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N0), VT, Op); 4549202375Srdivacky else if (OpBits > DestBits) 4550263508Sdim Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), VT, Op); 4551263508Sdim return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, Op, 4552202375Srdivacky DAG.getValueType(N0.getValueType())); 4553193323Sed } 4554193323Sed } 4555193323Sed 4556193323Sed // fold (sext (load x)) -> (sext (truncate (sextload x))) 4557219077Sdim // None of the supported targets knows how to perform load and sign extend 4558221345Sdim // on vectors in one instruction. We only perform this transformation on 4559221345Sdim // scalars. 4560219077Sdim if (ISD::isNON_EXTLoad(N0.getNode()) && !VT.isVector() && 4561193323Sed ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) || 4562193323Sed TLI.isLoadExtLegal(ISD::SEXTLOAD, N0.getValueType()))) { 4563193323Sed bool DoXform = true; 4564193323Sed SmallVector<SDNode*, 4> SetCCs; 4565193323Sed if (!N0.hasOneUse()) 4566193323Sed DoXform = ExtendUsesToFormExtLoad(N, N0, ISD::SIGN_EXTEND, SetCCs, TLI); 4567193323Sed if (DoXform) { 4568193323Sed LoadSDNode *LN0 = cast<LoadSDNode>(N0); 4569263508Sdim SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT, 4570193323Sed LN0->getChain(), 4571263508Sdim LN0->getBasePtr(), N0.getValueType(), 4572263508Sdim LN0->getMemOperand()); 4573193323Sed CombineTo(N, ExtLoad); 4574263508Sdim SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), 4575193323Sed N0.getValueType(), ExtLoad); 4576193323Sed CombineTo(N0.getNode(), Trunc, ExtLoad.getValue(1)); 4577263508Sdim ExtendSetCCUses(SetCCs, Trunc, ExtLoad, SDLoc(N), 4578224145Sdim ISD::SIGN_EXTEND); 4579193323Sed return SDValue(N, 0); // Return N so it doesn't get rechecked! 4580193323Sed } 4581193323Sed } 4582193323Sed 4583193323Sed // fold (sext (sextload x)) -> (sext (truncate (sextload x))) 4584193323Sed // fold (sext ( extload x)) -> (sext (truncate (sextload x))) 4585193323Sed if ((ISD::isSEXTLoad(N0.getNode()) || ISD::isEXTLoad(N0.getNode())) && 4586193323Sed ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) { 4587193323Sed LoadSDNode *LN0 = cast<LoadSDNode>(N0); 4588198090Srdivacky EVT MemVT = LN0->getMemoryVT(); 4589193323Sed if ((!LegalOperations && !LN0->isVolatile()) || 4590198090Srdivacky TLI.isLoadExtLegal(ISD::SEXTLOAD, MemVT)) { 4591263508Sdim SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT, 4592193323Sed LN0->getChain(), 4593263508Sdim LN0->getBasePtr(), MemVT, 4594263508Sdim LN0->getMemOperand()); 4595193323Sed CombineTo(N, ExtLoad); 4596193323Sed CombineTo(N0.getNode(), 4597263508Sdim DAG.getNode(ISD::TRUNCATE, SDLoc(N0), 4598193323Sed N0.getValueType(), ExtLoad), 4599193323Sed ExtLoad.getValue(1)); 4600193323Sed return SDValue(N, 0); // Return N so it doesn't get rechecked! 4601193323Sed } 4602193323Sed } 4603193323Sed 4604224145Sdim // fold (sext (and/or/xor (load x), cst)) -> 4605224145Sdim // (and/or/xor (sextload x), (sext cst)) 4606224145Sdim if ((N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR || 4607224145Sdim N0.getOpcode() == ISD::XOR) && 4608224145Sdim isa<LoadSDNode>(N0.getOperand(0)) && 4609224145Sdim N0.getOperand(1).getOpcode() == ISD::Constant && 4610224145Sdim TLI.isLoadExtLegal(ISD::SEXTLOAD, N0.getValueType()) && 4611224145Sdim (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) { 4612224145Sdim LoadSDNode *LN0 = cast<LoadSDNode>(N0.getOperand(0)); 4613224145Sdim if (LN0->getExtensionType() != ISD::ZEXTLOAD) { 4614224145Sdim bool DoXform = true; 4615224145Sdim SmallVector<SDNode*, 4> SetCCs; 4616224145Sdim if (!N0.hasOneUse()) 4617224145Sdim DoXform = ExtendUsesToFormExtLoad(N, N0.getOperand(0), ISD::SIGN_EXTEND, 4618224145Sdim SetCCs, TLI); 4619224145Sdim if (DoXform) { 4620263508Sdim SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(LN0), VT, 4621224145Sdim LN0->getChain(), LN0->getBasePtr(), 4622224145Sdim LN0->getMemoryVT(), 4623263508Sdim LN0->getMemOperand()); 4624224145Sdim APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue(); 4625224145Sdim Mask = Mask.sext(VT.getSizeInBits()); 4626263508Sdim SDValue And = DAG.getNode(N0.getOpcode(), SDLoc(N), VT, 4627224145Sdim ExtLoad, DAG.getConstant(Mask, VT)); 4628224145Sdim SDValue Trunc = DAG.getNode(ISD::TRUNCATE, 4629263508Sdim SDLoc(N0.getOperand(0)), 4630224145Sdim N0.getOperand(0).getValueType(), ExtLoad); 4631224145Sdim CombineTo(N, And); 4632224145Sdim CombineTo(N0.getOperand(0).getNode(), Trunc, ExtLoad.getValue(1)); 4633263508Sdim ExtendSetCCUses(SetCCs, Trunc, ExtLoad, SDLoc(N), 4634224145Sdim ISD::SIGN_EXTEND); 4635224145Sdim return SDValue(N, 0); // Return N so it doesn't get rechecked! 4636224145Sdim } 4637224145Sdim } 4638224145Sdim } 4639224145Sdim 4640193323Sed if (N0.getOpcode() == ISD::SETCC) { 4641198090Srdivacky // sext(setcc) -> sext_in_reg(vsetcc) for vectors. 4642207618Srdivacky // Only do this before legalize for now. 4643251662Sdim if (VT.isVector() && !LegalOperations && 4644263508Sdim TLI.getBooleanContents(true) == 4645251662Sdim TargetLowering::ZeroOrNegativeOneBooleanContent) { 4646207618Srdivacky EVT N0VT = N0.getOperand(0).getValueType(); 4647234353Sdim // On some architectures (such as SSE/NEON/etc) the SETCC result type is 4648234353Sdim // of the same size as the compared operands. Only optimize sext(setcc()) 4649234353Sdim // if this is the case. 4650263508Sdim EVT SVT = getSetCCResultType(N0VT); 4651234353Sdim 4652234353Sdim // We know that the # elements of the results is the same as the 4653234353Sdim // # elements of the compare (and the # elements of the compare result 4654234353Sdim // for that matter). Check to see that they are the same size. If so, 4655234353Sdim // we know that the element size of the sext'd result matches the 4656234353Sdim // element size of the compare operands. 4657234353Sdim if (VT.getSizeInBits() == SVT.getSizeInBits()) 4658263508Sdim return DAG.getSetCC(SDLoc(N), VT, N0.getOperand(0), 4659210299Sed N0.getOperand(1), 4660210299Sed cast<CondCodeSDNode>(N0.getOperand(2))->get()); 4661263508Sdim 4662207618Srdivacky // If the desired elements are smaller or larger than the source 4663207618Srdivacky // elements we can use a matching integer vector type and then 4664207618Srdivacky // truncate/sign extend 4665263508Sdim EVT MatchingVectorType = N0VT.changeVectorElementTypeToInteger(); 4666243830Sdim if (SVT == MatchingVectorType) { 4667263508Sdim SDValue VsetCC = DAG.getSetCC(SDLoc(N), MatchingVectorType, 4668243830Sdim N0.getOperand(0), N0.getOperand(1), 4669243830Sdim cast<CondCodeSDNode>(N0.getOperand(2))->get()); 4670263508Sdim return DAG.getSExtOrTrunc(VsetCC, SDLoc(N), VT); 4671207618Srdivacky } 4672198090Srdivacky } 4673207618Srdivacky 4674198090Srdivacky // sext(setcc x, y, cc) -> (select_cc x, y, -1, 0, cc) 4675207618Srdivacky unsigned ElementWidth = VT.getScalarType().getSizeInBits(); 4676198090Srdivacky SDValue NegOne = 4677207618Srdivacky DAG.getConstant(APInt::getAllOnesValue(ElementWidth), VT); 4678193323Sed SDValue SCC = 4679263508Sdim SimplifySelectCC(SDLoc(N), N0.getOperand(0), N0.getOperand(1), 4680198090Srdivacky NegOne, DAG.getConstant(0, VT), 4681193323Sed cast<CondCodeSDNode>(N0.getOperand(2))->get(), true); 4682193323Sed if (SCC.getNode()) return SCC; 4683263508Sdim if (!VT.isVector() && 4684263508Sdim (!LegalOperations || 4685263508Sdim TLI.isOperationLegal(ISD::SETCC, getSetCCResultType(VT)))) { 4686263508Sdim return DAG.getSelect(SDLoc(N), VT, 4687263508Sdim DAG.getSetCC(SDLoc(N), 4688263508Sdim getSetCCResultType(VT), 4689263508Sdim N0.getOperand(0), N0.getOperand(1), 4690263508Sdim cast<CondCodeSDNode>(N0.getOperand(2))->get()), 4691263508Sdim NegOne, DAG.getConstant(0, VT)); 4692263508Sdim } 4693218893Sdim } 4694193323Sed 4695193323Sed // fold (sext x) -> (zext x) if the sign bit is known zero. 4696193323Sed if ((!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, VT)) && 4697193323Sed DAG.SignBitIsZero(N0)) 4698263508Sdim return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, N0); 4699193323Sed 4700193323Sed return SDValue(); 4701193323Sed} 4702193323Sed 4703234353Sdim// isTruncateOf - If N is a truncate of some other value, return true, record 4704234353Sdim// the value being truncated in Op and which of Op's bits are zero in KnownZero. 4705234353Sdim// This function computes KnownZero to avoid a duplicated call to 4706234353Sdim// ComputeMaskedBits in the caller. 4707234353Sdimstatic bool isTruncateOf(SelectionDAG &DAG, SDValue N, SDValue &Op, 4708234353Sdim APInt &KnownZero) { 4709234353Sdim APInt KnownOne; 4710234353Sdim if (N->getOpcode() == ISD::TRUNCATE) { 4711234353Sdim Op = N->getOperand(0); 4712234353Sdim DAG.ComputeMaskedBits(Op, KnownZero, KnownOne); 4713234353Sdim return true; 4714234353Sdim } 4715234353Sdim 4716234353Sdim if (N->getOpcode() != ISD::SETCC || N->getValueType(0) != MVT::i1 || 4717234353Sdim cast<CondCodeSDNode>(N->getOperand(2))->get() != ISD::SETNE) 4718234353Sdim return false; 4719234353Sdim 4720234353Sdim SDValue Op0 = N->getOperand(0); 4721234353Sdim SDValue Op1 = N->getOperand(1); 4722234353Sdim assert(Op0.getValueType() == Op1.getValueType()); 4723234353Sdim 4724234353Sdim ConstantSDNode *COp0 = dyn_cast<ConstantSDNode>(Op0); 4725234353Sdim ConstantSDNode *COp1 = dyn_cast<ConstantSDNode>(Op1); 4726234353Sdim if (COp0 && COp0->isNullValue()) 4727234353Sdim Op = Op1; 4728234353Sdim else if (COp1 && COp1->isNullValue()) 4729234353Sdim Op = Op0; 4730234353Sdim else 4731234353Sdim return false; 4732234353Sdim 4733234353Sdim DAG.ComputeMaskedBits(Op, KnownZero, KnownOne); 4734234353Sdim 4735234353Sdim if (!(KnownZero | APInt(Op.getValueSizeInBits(), 1)).isAllOnesValue()) 4736234353Sdim return false; 4737234353Sdim 4738234353Sdim return true; 4739234353Sdim} 4740234353Sdim 4741193323SedSDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) { 4742193323Sed SDValue N0 = N->getOperand(0); 4743198090Srdivacky EVT VT = N->getValueType(0); 4744193323Sed 4745193323Sed // fold (zext c1) -> c1 4746193323Sed if (isa<ConstantSDNode>(N0)) 4747263508Sdim return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, N0); 4748193323Sed // fold (zext (zext x)) -> (zext x) 4749193323Sed // fold (zext (aext x)) -> (zext x) 4750193323Sed if (N0.getOpcode() == ISD::ZERO_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) 4751263508Sdim return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, 4752193323Sed N0.getOperand(0)); 4753193323Sed 4754234353Sdim // fold (zext (truncate x)) -> (zext x) or 4755234353Sdim // (zext (truncate x)) -> (truncate x) 4756234353Sdim // This is valid when the truncated bits of x are already zero. 4757234353Sdim // FIXME: We should extend this to work for vectors too. 4758234353Sdim SDValue Op; 4759234353Sdim APInt KnownZero; 4760234353Sdim if (!VT.isVector() && isTruncateOf(DAG, N0, Op, KnownZero)) { 4761234353Sdim APInt TruncatedBits = 4762234353Sdim (Op.getValueSizeInBits() == N0.getValueSizeInBits()) ? 4763234353Sdim APInt(Op.getValueSizeInBits(), 0) : 4764234353Sdim APInt::getBitsSet(Op.getValueSizeInBits(), 4765234353Sdim N0.getValueSizeInBits(), 4766234353Sdim std::min(Op.getValueSizeInBits(), 4767234353Sdim VT.getSizeInBits())); 4768234353Sdim if (TruncatedBits == (KnownZero & TruncatedBits)) { 4769234353Sdim if (VT.bitsGT(Op.getValueType())) 4770263508Sdim return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, Op); 4771234353Sdim if (VT.bitsLT(Op.getValueType())) 4772263508Sdim return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Op); 4773234353Sdim 4774234353Sdim return Op; 4775234353Sdim } 4776234353Sdim } 4777234353Sdim 4778193323Sed // fold (zext (truncate (load x))) -> (zext (smaller load x)) 4779193323Sed // fold (zext (truncate (srl (load x), c))) -> (zext (small load (x+c/n))) 4780193323Sed if (N0.getOpcode() == ISD::TRUNCATE) { 4781193323Sed SDValue NarrowLoad = ReduceLoadWidth(N0.getNode()); 4782193323Sed if (NarrowLoad.getNode()) { 4783208599Srdivacky SDNode* oye = N0.getNode()->getOperand(0).getNode(); 4784208599Srdivacky if (NarrowLoad.getNode() != N0.getNode()) { 4785193323Sed CombineTo(N0.getNode(), NarrowLoad); 4786208599Srdivacky // CombineTo deleted the truncate, if needed, but not what's under it. 4787208599Srdivacky AddToWorkList(oye); 4788208599Srdivacky } 4789221345Sdim return SDValue(N, 0); // Return N so it doesn't get rechecked! 4790193323Sed } 4791193323Sed } 4792193323Sed 4793193323Sed // fold (zext (truncate x)) -> (and x, mask) 4794193323Sed if (N0.getOpcode() == ISD::TRUNCATE && 4795210299Sed (!LegalOperations || TLI.isOperationLegal(ISD::AND, VT))) { 4796218893Sdim 4797218893Sdim // fold (zext (truncate (load x))) -> (zext (smaller load x)) 4798218893Sdim // fold (zext (truncate (srl (load x), c))) -> (zext (smaller load (x+c/n))) 4799218893Sdim SDValue NarrowLoad = ReduceLoadWidth(N0.getNode()); 4800218893Sdim if (NarrowLoad.getNode()) { 4801218893Sdim SDNode* oye = N0.getNode()->getOperand(0).getNode(); 4802218893Sdim if (NarrowLoad.getNode() != N0.getNode()) { 4803218893Sdim CombineTo(N0.getNode(), NarrowLoad); 4804218893Sdim // CombineTo deleted the truncate, if needed, but not what's under it. 4805218893Sdim AddToWorkList(oye); 4806218893Sdim } 4807218893Sdim return SDValue(N, 0); // Return N so it doesn't get rechecked! 4808218893Sdim } 4809218893Sdim 4810193323Sed SDValue Op = N0.getOperand(0); 4811193323Sed if (Op.getValueType().bitsLT(VT)) { 4812263508Sdim Op = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Op); 4813239462Sdim AddToWorkList(Op.getNode()); 4814193323Sed } else if (Op.getValueType().bitsGT(VT)) { 4815263508Sdim Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Op); 4816239462Sdim AddToWorkList(Op.getNode()); 4817193323Sed } 4818263508Sdim return DAG.getZeroExtendInReg(Op, SDLoc(N), 4819200581Srdivacky N0.getValueType().getScalarType()); 4820193323Sed } 4821193323Sed 4822193323Sed // Fold (zext (and (trunc x), cst)) -> (and x, cst), 4823193323Sed // if either of the casts is not free. 4824193323Sed if (N0.getOpcode() == ISD::AND && 4825193323Sed N0.getOperand(0).getOpcode() == ISD::TRUNCATE && 4826193323Sed N0.getOperand(1).getOpcode() == ISD::Constant && 4827193323Sed (!TLI.isTruncateFree(N0.getOperand(0).getOperand(0).getValueType(), 4828193323Sed N0.getValueType()) || 4829193323Sed !TLI.isZExtFree(N0.getValueType(), VT))) { 4830193323Sed SDValue X = N0.getOperand(0).getOperand(0); 4831193323Sed if (X.getValueType().bitsLT(VT)) { 4832263508Sdim X = DAG.getNode(ISD::ANY_EXTEND, SDLoc(X), VT, X); 4833193323Sed } else if (X.getValueType().bitsGT(VT)) { 4834263508Sdim X = DAG.getNode(ISD::TRUNCATE, SDLoc(X), VT, X); 4835193323Sed } 4836193323Sed APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue(); 4837218893Sdim Mask = Mask.zext(VT.getSizeInBits()); 4838263508Sdim return DAG.getNode(ISD::AND, SDLoc(N), VT, 4839193323Sed X, DAG.getConstant(Mask, VT)); 4840193323Sed } 4841193323Sed 4842193323Sed // fold (zext (load x)) -> (zext (truncate (zextload x))) 4843219077Sdim // None of the supported targets knows how to perform load and vector_zext 4844221345Sdim // on vectors in one instruction. We only perform this transformation on 4845221345Sdim // scalars. 4846219077Sdim if (ISD::isNON_EXTLoad(N0.getNode()) && !VT.isVector() && 4847193323Sed ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) || 4848193323Sed TLI.isLoadExtLegal(ISD::ZEXTLOAD, N0.getValueType()))) { 4849193323Sed bool DoXform = true; 4850193323Sed SmallVector<SDNode*, 4> SetCCs; 4851193323Sed if (!N0.hasOneUse()) 4852193323Sed DoXform = ExtendUsesToFormExtLoad(N, N0, ISD::ZERO_EXTEND, SetCCs, TLI); 4853193323Sed if (DoXform) { 4854193323Sed LoadSDNode *LN0 = cast<LoadSDNode>(N0); 4855263508Sdim SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N), VT, 4856193323Sed LN0->getChain(), 4857263508Sdim LN0->getBasePtr(), N0.getValueType(), 4858263508Sdim LN0->getMemOperand()); 4859193323Sed CombineTo(N, ExtLoad); 4860263508Sdim SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), 4861193323Sed N0.getValueType(), ExtLoad); 4862193323Sed CombineTo(N0.getNode(), Trunc, ExtLoad.getValue(1)); 4863193323Sed 4864263508Sdim ExtendSetCCUses(SetCCs, Trunc, ExtLoad, SDLoc(N), 4865224145Sdim ISD::ZERO_EXTEND); 4866224145Sdim return SDValue(N, 0); // Return N so it doesn't get rechecked! 4867224145Sdim } 4868224145Sdim } 4869193323Sed 4870224145Sdim // fold (zext (and/or/xor (load x), cst)) -> 4871224145Sdim // (and/or/xor (zextload x), (zext cst)) 4872224145Sdim if ((N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR || 4873224145Sdim N0.getOpcode() == ISD::XOR) && 4874224145Sdim isa<LoadSDNode>(N0.getOperand(0)) && 4875224145Sdim N0.getOperand(1).getOpcode() == ISD::Constant && 4876224145Sdim TLI.isLoadExtLegal(ISD::ZEXTLOAD, N0.getValueType()) && 4877224145Sdim (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) { 4878224145Sdim LoadSDNode *LN0 = cast<LoadSDNode>(N0.getOperand(0)); 4879224145Sdim if (LN0->getExtensionType() != ISD::SEXTLOAD) { 4880224145Sdim bool DoXform = true; 4881224145Sdim SmallVector<SDNode*, 4> SetCCs; 4882224145Sdim if (!N0.hasOneUse()) 4883224145Sdim DoXform = ExtendUsesToFormExtLoad(N, N0.getOperand(0), ISD::ZERO_EXTEND, 4884224145Sdim SetCCs, TLI); 4885224145Sdim if (DoXform) { 4886263508Sdim SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN0), VT, 4887224145Sdim LN0->getChain(), LN0->getBasePtr(), 4888224145Sdim LN0->getMemoryVT(), 4889263508Sdim LN0->getMemOperand()); 4890224145Sdim APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue(); 4891224145Sdim Mask = Mask.zext(VT.getSizeInBits()); 4892263508Sdim SDValue And = DAG.getNode(N0.getOpcode(), SDLoc(N), VT, 4893224145Sdim ExtLoad, DAG.getConstant(Mask, VT)); 4894224145Sdim SDValue Trunc = DAG.getNode(ISD::TRUNCATE, 4895263508Sdim SDLoc(N0.getOperand(0)), 4896224145Sdim N0.getOperand(0).getValueType(), ExtLoad); 4897224145Sdim CombineTo(N, And); 4898224145Sdim CombineTo(N0.getOperand(0).getNode(), Trunc, ExtLoad.getValue(1)); 4899263508Sdim ExtendSetCCUses(SetCCs, Trunc, ExtLoad, SDLoc(N), 4900224145Sdim ISD::ZERO_EXTEND); 4901224145Sdim return SDValue(N, 0); // Return N so it doesn't get rechecked! 4902193323Sed } 4903193323Sed } 4904193323Sed } 4905193323Sed 4906193323Sed // fold (zext (zextload x)) -> (zext (truncate (zextload x))) 4907193323Sed // fold (zext ( extload x)) -> (zext (truncate (zextload x))) 4908193323Sed if ((ISD::isZEXTLoad(N0.getNode()) || ISD::isEXTLoad(N0.getNode())) && 4909193323Sed ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) { 4910193323Sed LoadSDNode *LN0 = cast<LoadSDNode>(N0); 4911198090Srdivacky EVT MemVT = LN0->getMemoryVT(); 4912193323Sed if ((!LegalOperations && !LN0->isVolatile()) || 4913198090Srdivacky TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT)) { 4914263508Sdim SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N), VT, 4915193323Sed LN0->getChain(), 4916263508Sdim LN0->getBasePtr(), MemVT, 4917263508Sdim LN0->getMemOperand()); 4918193323Sed CombineTo(N, ExtLoad); 4919193323Sed CombineTo(N0.getNode(), 4920263508Sdim DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), 4921193323Sed ExtLoad), 4922193323Sed ExtLoad.getValue(1)); 4923193323Sed return SDValue(N, 0); // Return N so it doesn't get rechecked! 4924193323Sed } 4925193323Sed } 4926193323Sed 4927193323Sed if (N0.getOpcode() == ISD::SETCC) { 4928208599Srdivacky if (!LegalOperations && VT.isVector()) { 4929208599Srdivacky // zext(setcc) -> (and (vsetcc), (1, 1, ...) for vectors. 4930208599Srdivacky // Only do this before legalize for now. 4931208599Srdivacky EVT N0VT = N0.getOperand(0).getValueType(); 4932208599Srdivacky EVT EltVT = VT.getVectorElementType(); 4933208599Srdivacky SmallVector<SDValue,8> OneOps(VT.getVectorNumElements(), 4934208599Srdivacky DAG.getConstant(1, EltVT)); 4935223017Sdim if (VT.getSizeInBits() == N0VT.getSizeInBits()) 4936208599Srdivacky // We know that the # elements of the results is the same as the 4937208599Srdivacky // # elements of the compare (and the # elements of the compare result 4938208599Srdivacky // for that matter). Check to see that they are the same size. If so, 4939208599Srdivacky // we know that the element size of the sext'd result matches the 4940208599Srdivacky // element size of the compare operands. 4941263508Sdim return DAG.getNode(ISD::AND, SDLoc(N), VT, 4942263508Sdim DAG.getSetCC(SDLoc(N), VT, N0.getOperand(0), 4943208599Srdivacky N0.getOperand(1), 4944208599Srdivacky cast<CondCodeSDNode>(N0.getOperand(2))->get()), 4945263508Sdim DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), VT, 4946208599Srdivacky &OneOps[0], OneOps.size())); 4947223017Sdim 4948223017Sdim // If the desired elements are smaller or larger than the source 4949223017Sdim // elements we can use a matching integer vector type and then 4950223017Sdim // truncate/sign extend 4951223017Sdim EVT MatchingElementType = 4952223017Sdim EVT::getIntegerVT(*DAG.getContext(), 4953223017Sdim N0VT.getScalarType().getSizeInBits()); 4954223017Sdim EVT MatchingVectorType = 4955223017Sdim EVT::getVectorVT(*DAG.getContext(), MatchingElementType, 4956223017Sdim N0VT.getVectorNumElements()); 4957223017Sdim SDValue VsetCC = 4958263508Sdim DAG.getSetCC(SDLoc(N), MatchingVectorType, N0.getOperand(0), 4959223017Sdim N0.getOperand(1), 4960223017Sdim cast<CondCodeSDNode>(N0.getOperand(2))->get()); 4961263508Sdim return DAG.getNode(ISD::AND, SDLoc(N), VT, 4962263508Sdim DAG.getSExtOrTrunc(VsetCC, SDLoc(N), VT), 4963263508Sdim DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), VT, 4964223017Sdim &OneOps[0], OneOps.size())); 4965208599Srdivacky } 4966208599Srdivacky 4967208599Srdivacky // zext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc 4968193323Sed SDValue SCC = 4969263508Sdim SimplifySelectCC(SDLoc(N), N0.getOperand(0), N0.getOperand(1), 4970193323Sed DAG.getConstant(1, VT), DAG.getConstant(0, VT), 4971193323Sed cast<CondCodeSDNode>(N0.getOperand(2))->get(), true); 4972193323Sed if (SCC.getNode()) return SCC; 4973193323Sed } 4974193323Sed 4975200581Srdivacky // (zext (shl (zext x), cst)) -> (shl (zext x), cst) 4976200581Srdivacky if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL) && 4977200581Srdivacky isa<ConstantSDNode>(N0.getOperand(1)) && 4978200581Srdivacky N0.getOperand(0).getOpcode() == ISD::ZERO_EXTEND && 4979200581Srdivacky N0.hasOneUse()) { 4980218893Sdim SDValue ShAmt = N0.getOperand(1); 4981218893Sdim unsigned ShAmtVal = cast<ConstantSDNode>(ShAmt)->getZExtValue(); 4982200581Srdivacky if (N0.getOpcode() == ISD::SHL) { 4983218893Sdim SDValue InnerZExt = N0.getOperand(0); 4984200581Srdivacky // If the original shl may be shifting out bits, do not perform this 4985200581Srdivacky // transformation. 4986218893Sdim unsigned KnownZeroBits = InnerZExt.getValueType().getSizeInBits() - 4987218893Sdim InnerZExt.getOperand(0).getValueType().getSizeInBits(); 4988218893Sdim if (ShAmtVal > KnownZeroBits) 4989200581Srdivacky return SDValue(); 4990200581Srdivacky } 4991218893Sdim 4992263508Sdim SDLoc DL(N); 4993219077Sdim 4994219077Sdim // Ensure that the shift amount is wide enough for the shifted value. 4995218893Sdim if (VT.getSizeInBits() >= 256) 4996218893Sdim ShAmt = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, ShAmt); 4997219077Sdim 4998218893Sdim return DAG.getNode(N0.getOpcode(), DL, VT, 4999218893Sdim DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)), 5000218893Sdim ShAmt); 5001200581Srdivacky } 5002200581Srdivacky 5003193323Sed return SDValue(); 5004193323Sed} 5005193323Sed 5006193323SedSDValue DAGCombiner::visitANY_EXTEND(SDNode *N) { 5007193323Sed SDValue N0 = N->getOperand(0); 5008198090Srdivacky EVT VT = N->getValueType(0); 5009193323Sed 5010193323Sed // fold (aext c1) -> c1 5011193323Sed if (isa<ConstantSDNode>(N0)) 5012263508Sdim return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, N0); 5013193323Sed // fold (aext (aext x)) -> (aext x) 5014193323Sed // fold (aext (zext x)) -> (zext x) 5015193323Sed // fold (aext (sext x)) -> (sext x) 5016193323Sed if (N0.getOpcode() == ISD::ANY_EXTEND || 5017193323Sed N0.getOpcode() == ISD::ZERO_EXTEND || 5018193323Sed N0.getOpcode() == ISD::SIGN_EXTEND) 5019263508Sdim return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, N0.getOperand(0)); 5020193323Sed 5021193323Sed // fold (aext (truncate (load x))) -> (aext (smaller load x)) 5022193323Sed // fold (aext (truncate (srl (load x), c))) -> (aext (small load (x+c/n))) 5023193323Sed if (N0.getOpcode() == ISD::TRUNCATE) { 5024193323Sed SDValue NarrowLoad = ReduceLoadWidth(N0.getNode()); 5025193323Sed if (NarrowLoad.getNode()) { 5026208599Srdivacky SDNode* oye = N0.getNode()->getOperand(0).getNode(); 5027208599Srdivacky if (NarrowLoad.getNode() != N0.getNode()) { 5028193323Sed CombineTo(N0.getNode(), NarrowLoad); 5029208599Srdivacky // CombineTo deleted the truncate, if needed, but not what's under it. 5030208599Srdivacky AddToWorkList(oye); 5031208599Srdivacky } 5032221345Sdim return SDValue(N, 0); // Return N so it doesn't get rechecked! 5033193323Sed } 5034193323Sed } 5035193323Sed 5036193323Sed // fold (aext (truncate x)) 5037193323Sed if (N0.getOpcode() == ISD::TRUNCATE) { 5038193323Sed SDValue TruncOp = N0.getOperand(0); 5039193323Sed if (TruncOp.getValueType() == VT) 5040193323Sed return TruncOp; // x iff x size == zext size. 5041193323Sed if (TruncOp.getValueType().bitsGT(VT)) 5042263508Sdim return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, TruncOp); 5043263508Sdim return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, TruncOp); 5044193323Sed } 5045193323Sed 5046193323Sed // Fold (aext (and (trunc x), cst)) -> (and x, cst) 5047193323Sed // if the trunc is not free. 5048193323Sed if (N0.getOpcode() == ISD::AND && 5049193323Sed N0.getOperand(0).getOpcode() == ISD::TRUNCATE && 5050193323Sed N0.getOperand(1).getOpcode() == ISD::Constant && 5051193323Sed !TLI.isTruncateFree(N0.getOperand(0).getOperand(0).getValueType(), 5052193323Sed N0.getValueType())) { 5053193323Sed SDValue X = N0.getOperand(0).getOperand(0); 5054193323Sed if (X.getValueType().bitsLT(VT)) { 5055263508Sdim X = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, X); 5056193323Sed } else if (X.getValueType().bitsGT(VT)) { 5057263508Sdim X = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, X); 5058193323Sed } 5059193323Sed APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue(); 5060218893Sdim Mask = Mask.zext(VT.getSizeInBits()); 5061263508Sdim return DAG.getNode(ISD::AND, SDLoc(N), VT, 5062193323Sed X, DAG.getConstant(Mask, VT)); 5063193323Sed } 5064193323Sed 5065193323Sed // fold (aext (load x)) -> (aext (truncate (extload x))) 5066219077Sdim // None of the supported targets knows how to perform load and any_ext 5067221345Sdim // on vectors in one instruction. We only perform this transformation on 5068221345Sdim // scalars. 5069219077Sdim if (ISD::isNON_EXTLoad(N0.getNode()) && !VT.isVector() && 5070193323Sed ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) || 5071193323Sed TLI.isLoadExtLegal(ISD::EXTLOAD, N0.getValueType()))) { 5072193323Sed bool DoXform = true; 5073193323Sed SmallVector<SDNode*, 4> SetCCs; 5074193323Sed if (!N0.hasOneUse()) 5075193323Sed DoXform = ExtendUsesToFormExtLoad(N, N0, ISD::ANY_EXTEND, SetCCs, TLI); 5076193323Sed if (DoXform) { 5077193323Sed LoadSDNode *LN0 = cast<LoadSDNode>(N0); 5078263508Sdim SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT, 5079193323Sed LN0->getChain(), 5080263508Sdim LN0->getBasePtr(), N0.getValueType(), 5081263508Sdim LN0->getMemOperand()); 5082193323Sed CombineTo(N, ExtLoad); 5083263508Sdim SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), 5084193323Sed N0.getValueType(), ExtLoad); 5085193323Sed CombineTo(N0.getNode(), Trunc, ExtLoad.getValue(1)); 5086263508Sdim ExtendSetCCUses(SetCCs, Trunc, ExtLoad, SDLoc(N), 5087224145Sdim ISD::ANY_EXTEND); 5088193323Sed return SDValue(N, 0); // Return N so it doesn't get rechecked! 5089193323Sed } 5090193323Sed } 5091193323Sed 5092193323Sed // fold (aext (zextload x)) -> (aext (truncate (zextload x))) 5093193323Sed // fold (aext (sextload x)) -> (aext (truncate (sextload x))) 5094193323Sed // fold (aext ( extload x)) -> (aext (truncate (extload x))) 5095193323Sed if (N0.getOpcode() == ISD::LOAD && 5096193323Sed !ISD::isNON_EXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) && 5097193323Sed N0.hasOneUse()) { 5098193323Sed LoadSDNode *LN0 = cast<LoadSDNode>(N0); 5099198090Srdivacky EVT MemVT = LN0->getMemoryVT(); 5100263508Sdim SDValue ExtLoad = DAG.getExtLoad(LN0->getExtensionType(), SDLoc(N), 5101218893Sdim VT, LN0->getChain(), LN0->getBasePtr(), 5102263508Sdim MemVT, LN0->getMemOperand()); 5103193323Sed CombineTo(N, ExtLoad); 5104193323Sed CombineTo(N0.getNode(), 5105263508Sdim DAG.getNode(ISD::TRUNCATE, SDLoc(N0), 5106193323Sed N0.getValueType(), ExtLoad), 5107193323Sed ExtLoad.getValue(1)); 5108193323Sed return SDValue(N, 0); // Return N so it doesn't get rechecked! 5109193323Sed } 5110193323Sed 5111193323Sed if (N0.getOpcode() == ISD::SETCC) { 5112208599Srdivacky // aext(setcc) -> sext_in_reg(vsetcc) for vectors. 5113208599Srdivacky // Only do this before legalize for now. 5114208599Srdivacky if (VT.isVector() && !LegalOperations) { 5115208599Srdivacky EVT N0VT = N0.getOperand(0).getValueType(); 5116208599Srdivacky // We know that the # elements of the results is the same as the 5117208599Srdivacky // # elements of the compare (and the # elements of the compare result 5118208599Srdivacky // for that matter). Check to see that they are the same size. If so, 5119208599Srdivacky // we know that the element size of the sext'd result matches the 5120208599Srdivacky // element size of the compare operands. 5121208599Srdivacky if (VT.getSizeInBits() == N0VT.getSizeInBits()) 5122263508Sdim return DAG.getSetCC(SDLoc(N), VT, N0.getOperand(0), 5123210299Sed N0.getOperand(1), 5124210299Sed cast<CondCodeSDNode>(N0.getOperand(2))->get()); 5125208599Srdivacky // If the desired elements are smaller or larger than the source 5126208599Srdivacky // elements we can use a matching integer vector type and then 5127208599Srdivacky // truncate/sign extend 5128208599Srdivacky else { 5129210299Sed EVT MatchingElementType = 5130210299Sed EVT::getIntegerVT(*DAG.getContext(), 5131210299Sed N0VT.getScalarType().getSizeInBits()); 5132210299Sed EVT MatchingVectorType = 5133210299Sed EVT::getVectorVT(*DAG.getContext(), MatchingElementType, 5134210299Sed N0VT.getVectorNumElements()); 5135210299Sed SDValue VsetCC = 5136263508Sdim DAG.getSetCC(SDLoc(N), MatchingVectorType, N0.getOperand(0), 5137210299Sed N0.getOperand(1), 5138210299Sed cast<CondCodeSDNode>(N0.getOperand(2))->get()); 5139263508Sdim return DAG.getSExtOrTrunc(VsetCC, SDLoc(N), VT); 5140208599Srdivacky } 5141208599Srdivacky } 5142208599Srdivacky 5143208599Srdivacky // aext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc 5144193323Sed SDValue SCC = 5145263508Sdim SimplifySelectCC(SDLoc(N), N0.getOperand(0), N0.getOperand(1), 5146193323Sed DAG.getConstant(1, VT), DAG.getConstant(0, VT), 5147193323Sed cast<CondCodeSDNode>(N0.getOperand(2))->get(), true); 5148193323Sed if (SCC.getNode()) 5149193323Sed return SCC; 5150193323Sed } 5151193323Sed 5152193323Sed return SDValue(); 5153193323Sed} 5154193323Sed 5155193323Sed/// GetDemandedBits - See if the specified operand can be simplified with the 5156193323Sed/// knowledge that only the bits specified by Mask are used. If so, return the 5157193323Sed/// simpler operand, otherwise return a null SDValue. 5158193323SedSDValue DAGCombiner::GetDemandedBits(SDValue V, const APInt &Mask) { 5159193323Sed switch (V.getOpcode()) { 5160193323Sed default: break; 5161234353Sdim case ISD::Constant: { 5162234353Sdim const ConstantSDNode *CV = cast<ConstantSDNode>(V.getNode()); 5163234353Sdim assert(CV != 0 && "Const value should be ConstSDNode."); 5164234353Sdim const APInt &CVal = CV->getAPIntValue(); 5165234353Sdim APInt NewVal = CVal & Mask; 5166263508Sdim if (NewVal != CVal) 5167234353Sdim return DAG.getConstant(NewVal, V.getValueType()); 5168234353Sdim break; 5169234353Sdim } 5170193323Sed case ISD::OR: 5171193323Sed case ISD::XOR: 5172193323Sed // If the LHS or RHS don't contribute bits to the or, drop them. 5173193323Sed if (DAG.MaskedValueIsZero(V.getOperand(0), Mask)) 5174193323Sed return V.getOperand(1); 5175193323Sed if (DAG.MaskedValueIsZero(V.getOperand(1), Mask)) 5176193323Sed return V.getOperand(0); 5177193323Sed break; 5178193323Sed case ISD::SRL: 5179193323Sed // Only look at single-use SRLs. 5180193323Sed if (!V.getNode()->hasOneUse()) 5181193323Sed break; 5182193323Sed if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(V.getOperand(1))) { 5183193323Sed // See if we can recursively simplify the LHS. 5184193323Sed unsigned Amt = RHSC->getZExtValue(); 5185193323Sed 5186193323Sed // Watch out for shift count overflow though. 5187193323Sed if (Amt >= Mask.getBitWidth()) break; 5188193323Sed APInt NewMask = Mask << Amt; 5189193323Sed SDValue SimplifyLHS = GetDemandedBits(V.getOperand(0), NewMask); 5190193323Sed if (SimplifyLHS.getNode()) 5191263508Sdim return DAG.getNode(ISD::SRL, SDLoc(V), V.getValueType(), 5192193323Sed SimplifyLHS, V.getOperand(1)); 5193193323Sed } 5194193323Sed } 5195193323Sed return SDValue(); 5196193323Sed} 5197193323Sed 5198193323Sed/// ReduceLoadWidth - If the result of a wider load is shifted to right of N 5199193323Sed/// bits and then truncated to a narrower type and where N is a multiple 5200193323Sed/// of number of bits of the narrower type, transform it to a narrower load 5201193323Sed/// from address + N / num of bits of new type. If the result is to be 5202193323Sed/// extended, also fold the extension to form a extending load. 5203193323SedSDValue DAGCombiner::ReduceLoadWidth(SDNode *N) { 5204193323Sed unsigned Opc = N->getOpcode(); 5205210299Sed 5206193323Sed ISD::LoadExtType ExtType = ISD::NON_EXTLOAD; 5207193323Sed SDValue N0 = N->getOperand(0); 5208198090Srdivacky EVT VT = N->getValueType(0); 5209198090Srdivacky EVT ExtVT = VT; 5210193323Sed 5211193323Sed // This transformation isn't valid for vector loads. 5212193323Sed if (VT.isVector()) 5213193323Sed return SDValue(); 5214193323Sed 5215202375Srdivacky // Special case: SIGN_EXTEND_INREG is basically truncating to ExtVT then 5216193323Sed // extended to VT. 5217193323Sed if (Opc == ISD::SIGN_EXTEND_INREG) { 5218193323Sed ExtType = ISD::SEXTLOAD; 5219198090Srdivacky ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 5220210299Sed } else if (Opc == ISD::SRL) { 5221218893Sdim // Another special-case: SRL is basically zero-extending a narrower value. 5222210299Sed ExtType = ISD::ZEXTLOAD; 5223210299Sed N0 = SDValue(N, 0); 5224210299Sed ConstantSDNode *N01 = dyn_cast<ConstantSDNode>(N0.getOperand(1)); 5225210299Sed if (!N01) return SDValue(); 5226210299Sed ExtVT = EVT::getIntegerVT(*DAG.getContext(), 5227210299Sed VT.getSizeInBits() - N01->getZExtValue()); 5228193323Sed } 5229218893Sdim if (LegalOperations && !TLI.isLoadExtLegal(ExtType, ExtVT)) 5230218893Sdim return SDValue(); 5231193323Sed 5232198090Srdivacky unsigned EVTBits = ExtVT.getSizeInBits(); 5233219077Sdim 5234218893Sdim // Do not generate loads of non-round integer types since these can 5235218893Sdim // be expensive (and would be wrong if the type is not byte sized). 5236218893Sdim if (!ExtVT.isRound()) 5237218893Sdim return SDValue(); 5238219077Sdim 5239193323Sed unsigned ShAmt = 0; 5240218893Sdim if (N0.getOpcode() == ISD::SRL && N0.hasOneUse()) { 5241193323Sed if (ConstantSDNode *N01 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) { 5242193323Sed ShAmt = N01->getZExtValue(); 5243193323Sed // Is the shift amount a multiple of size of VT? 5244193323Sed if ((ShAmt & (EVTBits-1)) == 0) { 5245193323Sed N0 = N0.getOperand(0); 5246198090Srdivacky // Is the load width a multiple of size of VT? 5247198090Srdivacky if ((N0.getValueType().getSizeInBits() & (EVTBits-1)) != 0) 5248193323Sed return SDValue(); 5249193323Sed } 5250218893Sdim 5251218893Sdim // At this point, we must have a load or else we can't do the transform. 5252218893Sdim if (!isa<LoadSDNode>(N0)) return SDValue(); 5253219077Sdim 5254249423Sdim // Because a SRL must be assumed to *need* to zero-extend the high bits 5255249423Sdim // (as opposed to anyext the high bits), we can't combine the zextload 5256249423Sdim // lowering of SRL and an sextload. 5257249423Sdim if (cast<LoadSDNode>(N0)->getExtensionType() == ISD::SEXTLOAD) 5258249423Sdim return SDValue(); 5259249423Sdim 5260218893Sdim // If the shift amount is larger than the input type then we're not 5261218893Sdim // accessing any of the loaded bytes. If the load was a zextload/extload 5262218893Sdim // then the result of the shift+trunc is zero/undef (handled elsewhere). 5263218893Sdim if (ShAmt >= cast<LoadSDNode>(N0)->getMemoryVT().getSizeInBits()) 5264218893Sdim return SDValue(); 5265193323Sed } 5266193323Sed } 5267193323Sed 5268218893Sdim // If the load is shifted left (and the result isn't shifted back right), 5269218893Sdim // we can fold the truncate through the shift. 5270218893Sdim unsigned ShLeftAmt = 0; 5271218893Sdim if (ShAmt == 0 && N0.getOpcode() == ISD::SHL && N0.hasOneUse() && 5272218893Sdim ExtVT == VT && TLI.isNarrowingProfitable(N0.getValueType(), VT)) { 5273218893Sdim if (ConstantSDNode *N01 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) { 5274218893Sdim ShLeftAmt = N01->getZExtValue(); 5275218893Sdim N0 = N0.getOperand(0); 5276193323Sed } 5277218893Sdim } 5278219077Sdim 5279218893Sdim // If we haven't found a load, we can't narrow it. Don't transform one with 5280218893Sdim // multiple uses, this would require adding a new load. 5281249423Sdim if (!isa<LoadSDNode>(N0) || !N0.hasOneUse()) 5282218893Sdim return SDValue(); 5283219077Sdim 5284249423Sdim // Don't change the width of a volatile load. 5285249423Sdim LoadSDNode *LN0 = cast<LoadSDNode>(N0); 5286249423Sdim if (LN0->isVolatile()) 5287249423Sdim return SDValue(); 5288249423Sdim 5289218893Sdim // Verify that we are actually reducing a load width here. 5290249423Sdim if (LN0->getMemoryVT().getSizeInBits() < EVTBits) 5291218893Sdim return SDValue(); 5292219077Sdim 5293249423Sdim // For the transform to be legal, the load must produce only two values 5294249423Sdim // (the value loaded and the chain). Don't transform a pre-increment 5295263508Sdim // load, for example, which produces an extra value. Otherwise the 5296249423Sdim // transformation is not equivalent, and the downstream logic to replace 5297249423Sdim // uses gets things wrong. 5298249423Sdim if (LN0->getNumValues() > 2) 5299249423Sdim return SDValue(); 5300249423Sdim 5301263508Sdim // If the load that we're shrinking is an extload and we're not just 5302263508Sdim // discarding the extension we can't simply shrink the load. Bail. 5303263508Sdim // TODO: It would be possible to merge the extensions in some cases. 5304263508Sdim if (LN0->getExtensionType() != ISD::NON_EXTLOAD && 5305263508Sdim LN0->getMemoryVT().getSizeInBits() < ExtVT.getSizeInBits() + ShAmt) 5306263508Sdim return SDValue(); 5307263508Sdim 5308218893Sdim EVT PtrType = N0.getOperand(1).getValueType(); 5309193323Sed 5310239462Sdim if (PtrType == MVT::Untyped || PtrType.isExtended()) 5311239462Sdim // It's not possible to generate a constant of extended or untyped type. 5312239462Sdim return SDValue(); 5313239462Sdim 5314218893Sdim // For big endian targets, we need to adjust the offset to the pointer to 5315218893Sdim // load the correct bytes. 5316218893Sdim if (TLI.isBigEndian()) { 5317218893Sdim unsigned LVTStoreBits = LN0->getMemoryVT().getStoreSizeInBits(); 5318218893Sdim unsigned EVTStoreBits = ExtVT.getStoreSizeInBits(); 5319218893Sdim ShAmt = LVTStoreBits - EVTStoreBits - ShAmt; 5320218893Sdim } 5321193323Sed 5322218893Sdim uint64_t PtrOff = ShAmt / 8; 5323218893Sdim unsigned NewAlign = MinAlign(LN0->getAlignment(), PtrOff); 5324263508Sdim SDValue NewPtr = DAG.getNode(ISD::ADD, SDLoc(LN0), 5325218893Sdim PtrType, LN0->getBasePtr(), 5326218893Sdim DAG.getConstant(PtrOff, PtrType)); 5327218893Sdim AddToWorkList(NewPtr.getNode()); 5328193323Sed 5329218893Sdim SDValue Load; 5330218893Sdim if (ExtType == ISD::NON_EXTLOAD) 5331263508Sdim Load = DAG.getLoad(VT, SDLoc(N0), LN0->getChain(), NewPtr, 5332218893Sdim LN0->getPointerInfo().getWithOffset(PtrOff), 5333234353Sdim LN0->isVolatile(), LN0->isNonTemporal(), 5334263508Sdim LN0->isInvariant(), NewAlign, LN0->getTBAAInfo()); 5335218893Sdim else 5336263508Sdim Load = DAG.getExtLoad(ExtType, SDLoc(N0), VT, LN0->getChain(),NewPtr, 5337218893Sdim LN0->getPointerInfo().getWithOffset(PtrOff), 5338218893Sdim ExtVT, LN0->isVolatile(), LN0->isNonTemporal(), 5339263508Sdim NewAlign, LN0->getTBAAInfo()); 5340193323Sed 5341218893Sdim // Replace the old load's chain with the new load's chain. 5342218893Sdim WorkListRemover DeadNodes(*this); 5343239462Sdim DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1)); 5344218893Sdim 5345218893Sdim // Shift the result left, if we've swallowed a left shift. 5346218893Sdim SDValue Result = Load; 5347218893Sdim if (ShLeftAmt != 0) { 5348219077Sdim EVT ShImmTy = getShiftAmountTy(Result.getValueType()); 5349218893Sdim if (!isUIntN(ShImmTy.getSizeInBits(), ShLeftAmt)) 5350218893Sdim ShImmTy = VT; 5351249423Sdim // If the shift amount is as large as the result size (but, presumably, 5352249423Sdim // no larger than the source) then the useful bits of the result are 5353249423Sdim // zero; we can't simply return the shortened shift, because the result 5354249423Sdim // of that operation is undefined. 5355249423Sdim if (ShLeftAmt >= VT.getSizeInBits()) 5356249423Sdim Result = DAG.getConstant(0, VT); 5357249423Sdim else 5358263508Sdim Result = DAG.getNode(ISD::SHL, SDLoc(N0), VT, 5359249423Sdim Result, DAG.getConstant(ShLeftAmt, ShImmTy)); 5360193323Sed } 5361193323Sed 5362218893Sdim // Return the new loaded value. 5363218893Sdim return Result; 5364193323Sed} 5365193323Sed 5366193323SedSDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) { 5367193323Sed SDValue N0 = N->getOperand(0); 5368193323Sed SDValue N1 = N->getOperand(1); 5369198090Srdivacky EVT VT = N->getValueType(0); 5370198090Srdivacky EVT EVT = cast<VTSDNode>(N1)->getVT(); 5371200581Srdivacky unsigned VTBits = VT.getScalarType().getSizeInBits(); 5372202375Srdivacky unsigned EVTBits = EVT.getScalarType().getSizeInBits(); 5373193323Sed 5374193323Sed // fold (sext_in_reg c1) -> c1 5375193323Sed if (isa<ConstantSDNode>(N0) || N0.getOpcode() == ISD::UNDEF) 5376263508Sdim return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, N0, N1); 5377193323Sed 5378193323Sed // If the input is already sign extended, just drop the extension. 5379200581Srdivacky if (DAG.ComputeNumSignBits(N0) >= VTBits-EVTBits+1) 5380193323Sed return N0; 5381193323Sed 5382193323Sed // fold (sext_in_reg (sext_in_reg x, VT2), VT1) -> (sext_in_reg x, minVT) pt2 5383193323Sed if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG && 5384263508Sdim EVT.bitsLT(cast<VTSDNode>(N0.getOperand(1))->getVT())) 5385263508Sdim return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, 5386193323Sed N0.getOperand(0), N1); 5387193323Sed 5388193323Sed // fold (sext_in_reg (sext x)) -> (sext x) 5389193323Sed // fold (sext_in_reg (aext x)) -> (sext x) 5390193323Sed // if x is small enough. 5391193323Sed if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) { 5392193323Sed SDValue N00 = N0.getOperand(0); 5393207618Srdivacky if (N00.getValueType().getScalarType().getSizeInBits() <= EVTBits && 5394207618Srdivacky (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT))) 5395263508Sdim return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00, N1); 5396193323Sed } 5397193323Sed 5398193323Sed // fold (sext_in_reg x) -> (zext_in_reg x) if the sign bit is known zero. 5399193323Sed if (DAG.MaskedValueIsZero(N0, APInt::getBitsSet(VTBits, EVTBits-1, EVTBits))) 5400263508Sdim return DAG.getZeroExtendInReg(N0, SDLoc(N), EVT); 5401193323Sed 5402193323Sed // fold operands of sext_in_reg based on knowledge that the top bits are not 5403193323Sed // demanded. 5404193323Sed if (SimplifyDemandedBits(SDValue(N, 0))) 5405193323Sed return SDValue(N, 0); 5406193323Sed 5407193323Sed // fold (sext_in_reg (load x)) -> (smaller sextload x) 5408193323Sed // fold (sext_in_reg (srl (load x), c)) -> (smaller sextload (x+c/evtbits)) 5409193323Sed SDValue NarrowLoad = ReduceLoadWidth(N); 5410193323Sed if (NarrowLoad.getNode()) 5411193323Sed return NarrowLoad; 5412193323Sed 5413193323Sed // fold (sext_in_reg (srl X, 24), i8) -> (sra X, 24) 5414193323Sed // fold (sext_in_reg (srl X, 23), i8) -> (sra X, 23) iff possible. 5415193323Sed // We already fold "(sext_in_reg (srl X, 25), i8) -> srl X, 25" above. 5416193323Sed if (N0.getOpcode() == ISD::SRL) { 5417193323Sed if (ConstantSDNode *ShAmt = dyn_cast<ConstantSDNode>(N0.getOperand(1))) 5418200581Srdivacky if (ShAmt->getZExtValue()+EVTBits <= VTBits) { 5419193323Sed // We can turn this into an SRA iff the input to the SRL is already sign 5420193323Sed // extended enough. 5421193323Sed unsigned InSignBits = DAG.ComputeNumSignBits(N0.getOperand(0)); 5422200581Srdivacky if (VTBits-(ShAmt->getZExtValue()+EVTBits) < InSignBits) 5423263508Sdim return DAG.getNode(ISD::SRA, SDLoc(N), VT, 5424193323Sed N0.getOperand(0), N0.getOperand(1)); 5425193323Sed } 5426193323Sed } 5427193323Sed 5428193323Sed // fold (sext_inreg (extload x)) -> (sextload x) 5429193323Sed if (ISD::isEXTLoad(N0.getNode()) && 5430193323Sed ISD::isUNINDEXEDLoad(N0.getNode()) && 5431193323Sed EVT == cast<LoadSDNode>(N0)->getMemoryVT() && 5432193323Sed ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) || 5433193323Sed TLI.isLoadExtLegal(ISD::SEXTLOAD, EVT))) { 5434193323Sed LoadSDNode *LN0 = cast<LoadSDNode>(N0); 5435263508Sdim SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT, 5436193323Sed LN0->getChain(), 5437263508Sdim LN0->getBasePtr(), EVT, 5438263508Sdim LN0->getMemOperand()); 5439193323Sed CombineTo(N, ExtLoad); 5440193323Sed CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1)); 5441249423Sdim AddToWorkList(ExtLoad.getNode()); 5442193323Sed return SDValue(N, 0); // Return N so it doesn't get rechecked! 5443193323Sed } 5444193323Sed // fold (sext_inreg (zextload x)) -> (sextload x) iff load has one use 5445193323Sed if (ISD::isZEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) && 5446193323Sed N0.hasOneUse() && 5447193323Sed EVT == cast<LoadSDNode>(N0)->getMemoryVT() && 5448193323Sed ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) || 5449193323Sed TLI.isLoadExtLegal(ISD::SEXTLOAD, EVT))) { 5450193323Sed LoadSDNode *LN0 = cast<LoadSDNode>(N0); 5451263508Sdim SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT, 5452193323Sed LN0->getChain(), 5453263508Sdim LN0->getBasePtr(), EVT, 5454263508Sdim LN0->getMemOperand()); 5455193323Sed CombineTo(N, ExtLoad); 5456193323Sed CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1)); 5457193323Sed return SDValue(N, 0); // Return N so it doesn't get rechecked! 5458193323Sed } 5459224145Sdim 5460224145Sdim // Form (sext_inreg (bswap >> 16)) or (sext_inreg (rotl (bswap) 16)) 5461224145Sdim if (EVTBits <= 16 && N0.getOpcode() == ISD::OR) { 5462224145Sdim SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0), 5463224145Sdim N0.getOperand(1), false); 5464224145Sdim if (BSwap.getNode() != 0) 5465263508Sdim return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, 5466224145Sdim BSwap, N1); 5467224145Sdim } 5468224145Sdim 5469193323Sed return SDValue(); 5470193323Sed} 5471193323Sed 5472193323SedSDValue DAGCombiner::visitTRUNCATE(SDNode *N) { 5473193323Sed SDValue N0 = N->getOperand(0); 5474198090Srdivacky EVT VT = N->getValueType(0); 5475234353Sdim bool isLE = TLI.isLittleEndian(); 5476193323Sed 5477193323Sed // noop truncate 5478193323Sed if (N0.getValueType() == N->getValueType(0)) 5479193323Sed return N0; 5480193323Sed // fold (truncate c1) -> c1 5481193323Sed if (isa<ConstantSDNode>(N0)) 5482263508Sdim return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0); 5483193323Sed // fold (truncate (truncate x)) -> (truncate x) 5484193323Sed if (N0.getOpcode() == ISD::TRUNCATE) 5485263508Sdim return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0)); 5486193323Sed // fold (truncate (ext x)) -> (ext x) or (truncate x) or x 5487207618Srdivacky if (N0.getOpcode() == ISD::ZERO_EXTEND || 5488207618Srdivacky N0.getOpcode() == ISD::SIGN_EXTEND || 5489193323Sed N0.getOpcode() == ISD::ANY_EXTEND) { 5490193323Sed if (N0.getOperand(0).getValueType().bitsLT(VT)) 5491193323Sed // if the source is smaller than the dest, we still need an extend 5492263508Sdim return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, 5493193323Sed N0.getOperand(0)); 5494243830Sdim if (N0.getOperand(0).getValueType().bitsGT(VT)) 5495193323Sed // if the source is larger than the dest, than we just need the truncate 5496263508Sdim return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0)); 5497243830Sdim // if the source and dest are the same type, we can drop both the extend 5498243830Sdim // and the truncate. 5499243830Sdim return N0.getOperand(0); 5500193323Sed } 5501193323Sed 5502234353Sdim // Fold extract-and-trunc into a narrow extract. For example: 5503234353Sdim // i64 x = EXTRACT_VECTOR_ELT(v2i64 val, i32 1) 5504234353Sdim // i32 y = TRUNCATE(i64 x) 5505234353Sdim // -- becomes -- 5506234353Sdim // v16i8 b = BITCAST (v2i64 val) 5507234353Sdim // i8 x = EXTRACT_VECTOR_ELT(v16i8 b, i32 8) 5508234353Sdim // 5509234353Sdim // Note: We only run this optimization after type legalization (which often 5510234353Sdim // creates this pattern) and before operation legalization after which 5511234353Sdim // we need to be more careful about the vector instructions that we generate. 5512234353Sdim if (N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT && 5513234353Sdim LegalTypes && !LegalOperations && N0->hasOneUse()) { 5514234353Sdim 5515234353Sdim EVT VecTy = N0.getOperand(0).getValueType(); 5516234353Sdim EVT ExTy = N0.getValueType(); 5517234353Sdim EVT TrTy = N->getValueType(0); 5518234353Sdim 5519234353Sdim unsigned NumElem = VecTy.getVectorNumElements(); 5520234353Sdim unsigned SizeRatio = ExTy.getSizeInBits()/TrTy.getSizeInBits(); 5521234353Sdim 5522234353Sdim EVT NVT = EVT::getVectorVT(*DAG.getContext(), TrTy, SizeRatio * NumElem); 5523234353Sdim assert(NVT.getSizeInBits() == VecTy.getSizeInBits() && "Invalid Size"); 5524234353Sdim 5525234353Sdim SDValue EltNo = N0->getOperand(1); 5526234353Sdim if (isa<ConstantSDNode>(EltNo) && isTypeLegal(NVT)) { 5527234353Sdim int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue(); 5528263508Sdim EVT IndexTy = TLI.getVectorIdxTy(); 5529234353Sdim int Index = isLE ? (Elt*SizeRatio) : (Elt*SizeRatio + (SizeRatio-1)); 5530234353Sdim 5531263508Sdim SDValue V = DAG.getNode(ISD::BITCAST, SDLoc(N), 5532234353Sdim NVT, N0.getOperand(0)); 5533234353Sdim 5534234353Sdim return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, 5535263508Sdim SDLoc(N), TrTy, V, 5536239462Sdim DAG.getConstant(Index, IndexTy)); 5537234353Sdim } 5538234353Sdim } 5539234353Sdim 5540249423Sdim // Fold a series of buildvector, bitcast, and truncate if possible. 5541249423Sdim // For example fold 5542249423Sdim // (2xi32 trunc (bitcast ((4xi32)buildvector x, x, y, y) 2xi64)) to 5543249423Sdim // (2xi32 (buildvector x, y)). 5544249423Sdim if (Level == AfterLegalizeVectorOps && VT.isVector() && 5545249423Sdim N0.getOpcode() == ISD::BITCAST && N0.hasOneUse() && 5546249423Sdim N0.getOperand(0).getOpcode() == ISD::BUILD_VECTOR && 5547249423Sdim N0.getOperand(0).hasOneUse()) { 5548249423Sdim 5549249423Sdim SDValue BuildVect = N0.getOperand(0); 5550249423Sdim EVT BuildVectEltTy = BuildVect.getValueType().getVectorElementType(); 5551249423Sdim EVT TruncVecEltTy = VT.getVectorElementType(); 5552249423Sdim 5553249423Sdim // Check that the element types match. 5554249423Sdim if (BuildVectEltTy == TruncVecEltTy) { 5555249423Sdim // Now we only need to compute the offset of the truncated elements. 5556249423Sdim unsigned BuildVecNumElts = BuildVect.getNumOperands(); 5557249423Sdim unsigned TruncVecNumElts = VT.getVectorNumElements(); 5558249423Sdim unsigned TruncEltOffset = BuildVecNumElts / TruncVecNumElts; 5559249423Sdim 5560249423Sdim assert((BuildVecNumElts % TruncVecNumElts) == 0 && 5561249423Sdim "Invalid number of elements"); 5562249423Sdim 5563249423Sdim SmallVector<SDValue, 8> Opnds; 5564249423Sdim for (unsigned i = 0, e = BuildVecNumElts; i != e; i += TruncEltOffset) 5565249423Sdim Opnds.push_back(BuildVect.getOperand(i)); 5566249423Sdim 5567263508Sdim return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), VT, &Opnds[0], 5568249423Sdim Opnds.size()); 5569249423Sdim } 5570249423Sdim } 5571249423Sdim 5572193323Sed // See if we can simplify the input to this truncate through knowledge that 5573219077Sdim // only the low bits are being used. 5574219077Sdim // For example "trunc (or (shl x, 8), y)" // -> trunc y 5575221345Sdim // Currently we only perform this optimization on scalars because vectors 5576219077Sdim // may have different active low bits. 5577219077Sdim if (!VT.isVector()) { 5578219077Sdim SDValue Shorter = 5579219077Sdim GetDemandedBits(N0, APInt::getLowBitsSet(N0.getValueSizeInBits(), 5580219077Sdim VT.getSizeInBits())); 5581219077Sdim if (Shorter.getNode()) 5582263508Sdim return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Shorter); 5583219077Sdim } 5584193323Sed // fold (truncate (load x)) -> (smaller load x) 5585193323Sed // fold (truncate (srl (load x), c)) -> (smaller load (x+c/evtbits)) 5586210299Sed if (!LegalTypes || TLI.isTypeDesirableForOp(N0.getOpcode(), VT)) { 5587210299Sed SDValue Reduced = ReduceLoadWidth(N); 5588210299Sed if (Reduced.getNode()) 5589210299Sed return Reduced; 5590210299Sed } 5591243830Sdim // fold (trunc (concat ... x ...)) -> (concat ..., (trunc x), ...)), 5592243830Sdim // where ... are all 'undef'. 5593243830Sdim if (N0.getOpcode() == ISD::CONCAT_VECTORS && !LegalTypes) { 5594243830Sdim SmallVector<EVT, 8> VTs; 5595243830Sdim SDValue V; 5596243830Sdim unsigned Idx = 0; 5597243830Sdim unsigned NumDefs = 0; 5598210299Sed 5599243830Sdim for (unsigned i = 0, e = N0.getNumOperands(); i != e; ++i) { 5600243830Sdim SDValue X = N0.getOperand(i); 5601243830Sdim if (X.getOpcode() != ISD::UNDEF) { 5602243830Sdim V = X; 5603243830Sdim Idx = i; 5604243830Sdim NumDefs++; 5605243830Sdim } 5606243830Sdim // Stop if more than one members are non-undef. 5607243830Sdim if (NumDefs > 1) 5608243830Sdim break; 5609243830Sdim VTs.push_back(EVT::getVectorVT(*DAG.getContext(), 5610243830Sdim VT.getVectorElementType(), 5611243830Sdim X.getValueType().getVectorNumElements())); 5612243830Sdim } 5613243830Sdim 5614243830Sdim if (NumDefs == 0) 5615243830Sdim return DAG.getUNDEF(VT); 5616243830Sdim 5617243830Sdim if (NumDefs == 1) { 5618243830Sdim assert(V.getNode() && "The single defined operand is empty!"); 5619243830Sdim SmallVector<SDValue, 8> Opnds; 5620243830Sdim for (unsigned i = 0, e = VTs.size(); i != e; ++i) { 5621243830Sdim if (i != Idx) { 5622243830Sdim Opnds.push_back(DAG.getUNDEF(VTs[i])); 5623243830Sdim continue; 5624243830Sdim } 5625263508Sdim SDValue NV = DAG.getNode(ISD::TRUNCATE, SDLoc(V), VTs[i], V); 5626243830Sdim AddToWorkList(NV.getNode()); 5627243830Sdim Opnds.push_back(NV); 5628243830Sdim } 5629263508Sdim return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, 5630243830Sdim &Opnds[0], Opnds.size()); 5631243830Sdim } 5632243830Sdim } 5633243830Sdim 5634210299Sed // Simplify the operands using demanded-bits information. 5635210299Sed if (!VT.isVector() && 5636210299Sed SimplifyDemandedBits(SDValue(N, 0))) 5637210299Sed return SDValue(N, 0); 5638210299Sed 5639207618Srdivacky return SDValue(); 5640193323Sed} 5641193323Sed 5642193323Sedstatic SDNode *getBuildPairElt(SDNode *N, unsigned i) { 5643193323Sed SDValue Elt = N->getOperand(i); 5644193323Sed if (Elt.getOpcode() != ISD::MERGE_VALUES) 5645193323Sed return Elt.getNode(); 5646193323Sed return Elt.getOperand(Elt.getResNo()).getNode(); 5647193323Sed} 5648193323Sed 5649193323Sed/// CombineConsecutiveLoads - build_pair (load, load) -> load 5650193323Sed/// if load locations are consecutive. 5651198090SrdivackySDValue DAGCombiner::CombineConsecutiveLoads(SDNode *N, EVT VT) { 5652193323Sed assert(N->getOpcode() == ISD::BUILD_PAIR); 5653193323Sed 5654193574Sed LoadSDNode *LD1 = dyn_cast<LoadSDNode>(getBuildPairElt(N, 0)); 5655193574Sed LoadSDNode *LD2 = dyn_cast<LoadSDNode>(getBuildPairElt(N, 1)); 5656218893Sdim if (!LD1 || !LD2 || !ISD::isNON_EXTLoad(LD1) || !LD1->hasOneUse() || 5657218893Sdim LD1->getPointerInfo().getAddrSpace() != 5658218893Sdim LD2->getPointerInfo().getAddrSpace()) 5659193323Sed return SDValue(); 5660198090Srdivacky EVT LD1VT = LD1->getValueType(0); 5661193323Sed 5662193323Sed if (ISD::isNON_EXTLoad(LD2) && 5663193323Sed LD2->hasOneUse() && 5664193323Sed // If both are volatile this would reduce the number of volatile loads. 5665193323Sed // If one is volatile it might be ok, but play conservative and bail out. 5666193574Sed !LD1->isVolatile() && 5667193574Sed !LD2->isVolatile() && 5668200581Srdivacky DAG.isConsecutiveLoad(LD2, LD1, LD1VT.getSizeInBits()/8, 1)) { 5669193574Sed unsigned Align = LD1->getAlignment(); 5670243830Sdim unsigned NewAlign = TLI.getDataLayout()-> 5671198090Srdivacky getABITypeAlignment(VT.getTypeForEVT(*DAG.getContext())); 5672193323Sed 5673193323Sed if (NewAlign <= Align && 5674193323Sed (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT))) 5675263508Sdim return DAG.getLoad(VT, SDLoc(N), LD1->getChain(), 5676218893Sdim LD1->getBasePtr(), LD1->getPointerInfo(), 5677234353Sdim false, false, false, Align); 5678193323Sed } 5679193323Sed 5680193323Sed return SDValue(); 5681193323Sed} 5682193323Sed 5683218893SdimSDValue DAGCombiner::visitBITCAST(SDNode *N) { 5684193323Sed SDValue N0 = N->getOperand(0); 5685198090Srdivacky EVT VT = N->getValueType(0); 5686193323Sed 5687193323Sed // If the input is a BUILD_VECTOR with all constant elements, fold this now. 5688193323Sed // Only do this before legalize, since afterward the target may be depending 5689193323Sed // on the bitconvert. 5690193323Sed // First check to see if this is all constant. 5691193323Sed if (!LegalTypes && 5692193323Sed N0.getOpcode() == ISD::BUILD_VECTOR && N0.getNode()->hasOneUse() && 5693193323Sed VT.isVector()) { 5694193323Sed bool isSimple = true; 5695193323Sed for (unsigned i = 0, e = N0.getNumOperands(); i != e; ++i) 5696193323Sed if (N0.getOperand(i).getOpcode() != ISD::UNDEF && 5697193323Sed N0.getOperand(i).getOpcode() != ISD::Constant && 5698193323Sed N0.getOperand(i).getOpcode() != ISD::ConstantFP) { 5699193323Sed isSimple = false; 5700193323Sed break; 5701193323Sed } 5702193323Sed 5703198090Srdivacky EVT DestEltVT = N->getValueType(0).getVectorElementType(); 5704193323Sed assert(!DestEltVT.isVector() && 5705193323Sed "Element type of vector ValueType must not be vector!"); 5706193323Sed if (isSimple) 5707218893Sdim return ConstantFoldBITCASTofBUILD_VECTOR(N0.getNode(), DestEltVT); 5708193323Sed } 5709193323Sed 5710193323Sed // If the input is a constant, let getNode fold it. 5711193323Sed if (isa<ConstantSDNode>(N0) || isa<ConstantFPSDNode>(N0)) { 5712263508Sdim SDValue Res = DAG.getNode(ISD::BITCAST, SDLoc(N), VT, N0); 5713198090Srdivacky if (Res.getNode() != N) { 5714198090Srdivacky if (!LegalOperations || 5715198090Srdivacky TLI.isOperationLegal(Res.getNode()->getOpcode(), VT)) 5716198090Srdivacky return Res; 5717198090Srdivacky 5718198090Srdivacky // Folding it resulted in an illegal node, and it's too late to 5719198090Srdivacky // do that. Clean up the old node and forego the transformation. 5720198090Srdivacky // Ideally this won't happen very often, because instcombine 5721198090Srdivacky // and the earlier dagcombine runs (where illegal nodes are 5722198090Srdivacky // permitted) should have folded most of them already. 5723198090Srdivacky DAG.DeleteNode(Res.getNode()); 5724198090Srdivacky } 5725193323Sed } 5726193323Sed 5727193323Sed // (conv (conv x, t1), t2) -> (conv x, t2) 5728218893Sdim if (N0.getOpcode() == ISD::BITCAST) 5729263508Sdim return DAG.getNode(ISD::BITCAST, SDLoc(N), VT, 5730193323Sed N0.getOperand(0)); 5731193323Sed 5732193323Sed // fold (conv (load x)) -> (load (conv*)x) 5733193323Sed // If the resultant load doesn't need a higher alignment than the original! 5734193323Sed if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && 5735193323Sed // Do not change the width of a volatile load. 5736193323Sed !cast<LoadSDNode>(N0)->isVolatile() && 5737263508Sdim (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT)) && 5738263508Sdim TLI.isLoadBitCastBeneficial(N0.getValueType(), VT)) { 5739193323Sed LoadSDNode *LN0 = cast<LoadSDNode>(N0); 5740243830Sdim unsigned Align = TLI.getDataLayout()-> 5741198090Srdivacky getABITypeAlignment(VT.getTypeForEVT(*DAG.getContext())); 5742193323Sed unsigned OrigAlign = LN0->getAlignment(); 5743193323Sed 5744193323Sed if (Align <= OrigAlign) { 5745263508Sdim SDValue Load = DAG.getLoad(VT, SDLoc(N), LN0->getChain(), 5746218893Sdim LN0->getBasePtr(), LN0->getPointerInfo(), 5747203954Srdivacky LN0->isVolatile(), LN0->isNonTemporal(), 5748263508Sdim LN0->isInvariant(), OrigAlign, 5749263508Sdim LN0->getTBAAInfo()); 5750193323Sed AddToWorkList(N); 5751193323Sed CombineTo(N0.getNode(), 5752263508Sdim DAG.getNode(ISD::BITCAST, SDLoc(N0), 5753193323Sed N0.getValueType(), Load), 5754193323Sed Load.getValue(1)); 5755193323Sed return Load; 5756193323Sed } 5757193323Sed } 5758193323Sed 5759193323Sed // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit) 5760193323Sed // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit)) 5761193323Sed // This often reduces constant pool loads. 5762263508Sdim if (((N0.getOpcode() == ISD::FNEG && !TLI.isFNegFree(N0.getValueType())) || 5763263508Sdim (N0.getOpcode() == ISD::FABS && !TLI.isFAbsFree(N0.getValueType()))) && 5764243830Sdim N0.getNode()->hasOneUse() && VT.isInteger() && 5765243830Sdim !VT.isVector() && !N0.getValueType().isVector()) { 5766263508Sdim SDValue NewConv = DAG.getNode(ISD::BITCAST, SDLoc(N0), VT, 5767193323Sed N0.getOperand(0)); 5768193323Sed AddToWorkList(NewConv.getNode()); 5769193323Sed 5770193323Sed APInt SignBit = APInt::getSignBit(VT.getSizeInBits()); 5771193323Sed if (N0.getOpcode() == ISD::FNEG) 5772263508Sdim return DAG.getNode(ISD::XOR, SDLoc(N), VT, 5773193323Sed NewConv, DAG.getConstant(SignBit, VT)); 5774193323Sed assert(N0.getOpcode() == ISD::FABS); 5775263508Sdim return DAG.getNode(ISD::AND, SDLoc(N), VT, 5776193323Sed NewConv, DAG.getConstant(~SignBit, VT)); 5777193323Sed } 5778193323Sed 5779193323Sed // fold (bitconvert (fcopysign cst, x)) -> 5780193323Sed // (or (and (bitconvert x), sign), (and cst, (not sign))) 5781193323Sed // Note that we don't handle (copysign x, cst) because this can always be 5782193323Sed // folded to an fneg or fabs. 5783193323Sed if (N0.getOpcode() == ISD::FCOPYSIGN && N0.getNode()->hasOneUse() && 5784193323Sed isa<ConstantFPSDNode>(N0.getOperand(0)) && 5785193323Sed VT.isInteger() && !VT.isVector()) { 5786193323Sed unsigned OrigXWidth = N0.getOperand(1).getValueType().getSizeInBits(); 5787198090Srdivacky EVT IntXVT = EVT::getIntegerVT(*DAG.getContext(), OrigXWidth); 5788207618Srdivacky if (isTypeLegal(IntXVT)) { 5789263508Sdim SDValue X = DAG.getNode(ISD::BITCAST, SDLoc(N0), 5790193323Sed IntXVT, N0.getOperand(1)); 5791193323Sed AddToWorkList(X.getNode()); 5792193323Sed 5793193323Sed // If X has a different width than the result/lhs, sext it or truncate it. 5794193323Sed unsigned VTWidth = VT.getSizeInBits(); 5795193323Sed if (OrigXWidth < VTWidth) { 5796263508Sdim X = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, X); 5797193323Sed AddToWorkList(X.getNode()); 5798193323Sed } else if (OrigXWidth > VTWidth) { 5799193323Sed // To get the sign bit in the right place, we have to shift it right 5800193323Sed // before truncating. 5801263508Sdim X = DAG.getNode(ISD::SRL, SDLoc(X), 5802193323Sed X.getValueType(), X, 5803193323Sed DAG.getConstant(OrigXWidth-VTWidth, X.getValueType())); 5804193323Sed AddToWorkList(X.getNode()); 5805263508Sdim X = DAG.getNode(ISD::TRUNCATE, SDLoc(X), VT, X); 5806193323Sed AddToWorkList(X.getNode()); 5807193323Sed } 5808193323Sed 5809193323Sed APInt SignBit = APInt::getSignBit(VT.getSizeInBits()); 5810263508Sdim X = DAG.getNode(ISD::AND, SDLoc(X), VT, 5811193323Sed X, DAG.getConstant(SignBit, VT)); 5812193323Sed AddToWorkList(X.getNode()); 5813193323Sed 5814263508Sdim SDValue Cst = DAG.getNode(ISD::BITCAST, SDLoc(N0), 5815193323Sed VT, N0.getOperand(0)); 5816263508Sdim Cst = DAG.getNode(ISD::AND, SDLoc(Cst), VT, 5817193323Sed Cst, DAG.getConstant(~SignBit, VT)); 5818193323Sed AddToWorkList(Cst.getNode()); 5819193323Sed 5820263508Sdim return DAG.getNode(ISD::OR, SDLoc(N), VT, X, Cst); 5821193323Sed } 5822193323Sed } 5823193323Sed 5824193323Sed // bitconvert(build_pair(ld, ld)) -> ld iff load locations are consecutive. 5825193323Sed if (N0.getOpcode() == ISD::BUILD_PAIR) { 5826193323Sed SDValue CombineLD = CombineConsecutiveLoads(N0.getNode(), VT); 5827193323Sed if (CombineLD.getNode()) 5828193323Sed return CombineLD; 5829193323Sed } 5830193323Sed 5831193323Sed return SDValue(); 5832193323Sed} 5833193323Sed 5834193323SedSDValue DAGCombiner::visitBUILD_PAIR(SDNode *N) { 5835198090Srdivacky EVT VT = N->getValueType(0); 5836193323Sed return CombineConsecutiveLoads(N, VT); 5837193323Sed} 5838193323Sed 5839218893Sdim/// ConstantFoldBITCASTofBUILD_VECTOR - We know that BV is a build_vector 5840193323Sed/// node with Constant, ConstantFP or Undef operands. DstEltVT indicates the 5841193323Sed/// destination element value type. 5842193323SedSDValue DAGCombiner:: 5843218893SdimConstantFoldBITCASTofBUILD_VECTOR(SDNode *BV, EVT DstEltVT) { 5844198090Srdivacky EVT SrcEltVT = BV->getValueType(0).getVectorElementType(); 5845193323Sed 5846193323Sed // If this is already the right type, we're done. 5847193323Sed if (SrcEltVT == DstEltVT) return SDValue(BV, 0); 5848193323Sed 5849193323Sed unsigned SrcBitSize = SrcEltVT.getSizeInBits(); 5850193323Sed unsigned DstBitSize = DstEltVT.getSizeInBits(); 5851193323Sed 5852193323Sed // If this is a conversion of N elements of one type to N elements of another 5853193323Sed // type, convert each element. This handles FP<->INT cases. 5854193323Sed if (SrcBitSize == DstBitSize) { 5855212904Sdim EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT, 5856212904Sdim BV->getValueType(0).getVectorNumElements()); 5857212904Sdim 5858212904Sdim // Due to the FP element handling below calling this routine recursively, 5859212904Sdim // we can end up with a scalar-to-vector node here. 5860212904Sdim if (BV->getOpcode() == ISD::SCALAR_TO_VECTOR) 5861263508Sdim return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(BV), VT, 5862263508Sdim DAG.getNode(ISD::BITCAST, SDLoc(BV), 5863212904Sdim DstEltVT, BV->getOperand(0))); 5864218893Sdim 5865193323Sed SmallVector<SDValue, 8> Ops; 5866193323Sed for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) { 5867193323Sed SDValue Op = BV->getOperand(i); 5868193323Sed // If the vector element type is not legal, the BUILD_VECTOR operands 5869193323Sed // are promoted and implicitly truncated. Make that explicit here. 5870193323Sed if (Op.getValueType() != SrcEltVT) 5871263508Sdim Op = DAG.getNode(ISD::TRUNCATE, SDLoc(BV), SrcEltVT, Op); 5872263508Sdim Ops.push_back(DAG.getNode(ISD::BITCAST, SDLoc(BV), 5873193323Sed DstEltVT, Op)); 5874193323Sed AddToWorkList(Ops.back().getNode()); 5875193323Sed } 5876263508Sdim return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(BV), VT, 5877193323Sed &Ops[0], Ops.size()); 5878193323Sed } 5879193323Sed 5880193323Sed // Otherwise, we're growing or shrinking the elements. To avoid having to 5881193323Sed // handle annoying details of growing/shrinking FP values, we convert them to 5882193323Sed // int first. 5883193323Sed if (SrcEltVT.isFloatingPoint()) { 5884193323Sed // Convert the input float vector to a int vector where the elements are the 5885193323Sed // same sizes. 5886193323Sed assert((SrcEltVT == MVT::f32 || SrcEltVT == MVT::f64) && "Unknown FP VT!"); 5887198090Srdivacky EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), SrcEltVT.getSizeInBits()); 5888218893Sdim BV = ConstantFoldBITCASTofBUILD_VECTOR(BV, IntVT).getNode(); 5889193323Sed SrcEltVT = IntVT; 5890193323Sed } 5891193323Sed 5892193323Sed // Now we know the input is an integer vector. If the output is a FP type, 5893193323Sed // convert to integer first, then to FP of the right size. 5894193323Sed if (DstEltVT.isFloatingPoint()) { 5895193323Sed assert((DstEltVT == MVT::f32 || DstEltVT == MVT::f64) && "Unknown FP VT!"); 5896198090Srdivacky EVT TmpVT = EVT::getIntegerVT(*DAG.getContext(), DstEltVT.getSizeInBits()); 5897218893Sdim SDNode *Tmp = ConstantFoldBITCASTofBUILD_VECTOR(BV, TmpVT).getNode(); 5898193323Sed 5899193323Sed // Next, convert to FP elements of the same size. 5900218893Sdim return ConstantFoldBITCASTofBUILD_VECTOR(Tmp, DstEltVT); 5901193323Sed } 5902193323Sed 5903193323Sed // Okay, we know the src/dst types are both integers of differing types. 5904193323Sed // Handling growing first. 5905193323Sed assert(SrcEltVT.isInteger() && DstEltVT.isInteger()); 5906193323Sed if (SrcBitSize < DstBitSize) { 5907193323Sed unsigned NumInputsPerOutput = DstBitSize/SrcBitSize; 5908193323Sed 5909193323Sed SmallVector<SDValue, 8> Ops; 5910193323Sed for (unsigned i = 0, e = BV->getNumOperands(); i != e; 5911193323Sed i += NumInputsPerOutput) { 5912193323Sed bool isLE = TLI.isLittleEndian(); 5913193323Sed APInt NewBits = APInt(DstBitSize, 0); 5914193323Sed bool EltIsUndef = true; 5915193323Sed for (unsigned j = 0; j != NumInputsPerOutput; ++j) { 5916193323Sed // Shift the previously computed bits over. 5917193323Sed NewBits <<= SrcBitSize; 5918193323Sed SDValue Op = BV->getOperand(i+ (isLE ? (NumInputsPerOutput-j-1) : j)); 5919193323Sed if (Op.getOpcode() == ISD::UNDEF) continue; 5920193323Sed EltIsUndef = false; 5921193323Sed 5922218893Sdim NewBits |= cast<ConstantSDNode>(Op)->getAPIntValue(). 5923207618Srdivacky zextOrTrunc(SrcBitSize).zext(DstBitSize); 5924193323Sed } 5925193323Sed 5926193323Sed if (EltIsUndef) 5927193323Sed Ops.push_back(DAG.getUNDEF(DstEltVT)); 5928193323Sed else 5929193323Sed Ops.push_back(DAG.getConstant(NewBits, DstEltVT)); 5930193323Sed } 5931193323Sed 5932198090Srdivacky EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT, Ops.size()); 5933263508Sdim return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(BV), VT, 5934193323Sed &Ops[0], Ops.size()); 5935193323Sed } 5936193323Sed 5937193323Sed // Finally, this must be the case where we are shrinking elements: each input 5938193323Sed // turns into multiple outputs. 5939193323Sed bool isS2V = ISD::isScalarToVector(BV); 5940193323Sed unsigned NumOutputsPerInput = SrcBitSize/DstBitSize; 5941198090Srdivacky EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT, 5942198090Srdivacky NumOutputsPerInput*BV->getNumOperands()); 5943193323Sed SmallVector<SDValue, 8> Ops; 5944193323Sed 5945193323Sed for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) { 5946193323Sed if (BV->getOperand(i).getOpcode() == ISD::UNDEF) { 5947193323Sed for (unsigned j = 0; j != NumOutputsPerInput; ++j) 5948193323Sed Ops.push_back(DAG.getUNDEF(DstEltVT)); 5949193323Sed continue; 5950193323Sed } 5951193323Sed 5952218893Sdim APInt OpVal = cast<ConstantSDNode>(BV->getOperand(i))-> 5953218893Sdim getAPIntValue().zextOrTrunc(SrcBitSize); 5954193323Sed 5955193323Sed for (unsigned j = 0; j != NumOutputsPerInput; ++j) { 5956218893Sdim APInt ThisVal = OpVal.trunc(DstBitSize); 5957193323Sed Ops.push_back(DAG.getConstant(ThisVal, DstEltVT)); 5958218893Sdim if (isS2V && i == 0 && j == 0 && ThisVal.zext(SrcBitSize) == OpVal) 5959193323Sed // Simply turn this into a SCALAR_TO_VECTOR of the new type. 5960263508Sdim return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(BV), VT, 5961193323Sed Ops[0]); 5962193323Sed OpVal = OpVal.lshr(DstBitSize); 5963193323Sed } 5964193323Sed 5965193323Sed // For big endian targets, swap the order of the pieces of each element. 5966193323Sed if (TLI.isBigEndian()) 5967193323Sed std::reverse(Ops.end()-NumOutputsPerInput, Ops.end()); 5968193323Sed } 5969193323Sed 5970263508Sdim return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(BV), VT, 5971193323Sed &Ops[0], Ops.size()); 5972193323Sed} 5973193323Sed 5974193323SedSDValue DAGCombiner::visitFADD(SDNode *N) { 5975193323Sed SDValue N0 = N->getOperand(0); 5976193323Sed SDValue N1 = N->getOperand(1); 5977193323Sed ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0); 5978193323Sed ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); 5979198090Srdivacky EVT VT = N->getValueType(0); 5980193323Sed 5981193323Sed // fold vector ops 5982193323Sed if (VT.isVector()) { 5983193323Sed SDValue FoldedVOp = SimplifyVBinOp(N); 5984193323Sed if (FoldedVOp.getNode()) return FoldedVOp; 5985193323Sed } 5986193323Sed 5987239462Sdim // fold (fadd c1, c2) -> c1 + c2 5988243830Sdim if (N0CFP && N1CFP) 5989263508Sdim return DAG.getNode(ISD::FADD, SDLoc(N), VT, N0, N1); 5990193323Sed // canonicalize constant to RHS 5991193323Sed if (N0CFP && !N1CFP) 5992263508Sdim return DAG.getNode(ISD::FADD, SDLoc(N), VT, N1, N0); 5993193323Sed // fold (fadd A, 0) -> A 5994234353Sdim if (DAG.getTarget().Options.UnsafeFPMath && N1CFP && 5995234353Sdim N1CFP->getValueAPF().isZero()) 5996193323Sed return N0; 5997193323Sed // fold (fadd A, (fneg B)) -> (fsub A, B) 5998234353Sdim if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) && 5999243830Sdim isNegatibleForFree(N1, LegalOperations, TLI, &DAG.getTarget().Options) == 2) 6000263508Sdim return DAG.getNode(ISD::FSUB, SDLoc(N), VT, N0, 6001193323Sed GetNegatedExpression(N1, DAG, LegalOperations)); 6002193323Sed // fold (fadd (fneg A), B) -> (fsub B, A) 6003234353Sdim if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) && 6004243830Sdim isNegatibleForFree(N0, LegalOperations, TLI, &DAG.getTarget().Options) == 2) 6005263508Sdim return DAG.getNode(ISD::FSUB, SDLoc(N), VT, N1, 6006193323Sed GetNegatedExpression(N0, DAG, LegalOperations)); 6007193323Sed 6008193323Sed // If allowed, fold (fadd (fadd x, c1), c2) -> (fadd x, (fadd c1, c2)) 6009234353Sdim if (DAG.getTarget().Options.UnsafeFPMath && N1CFP && 6010234353Sdim N0.getOpcode() == ISD::FADD && N0.getNode()->hasOneUse() && 6011234353Sdim isa<ConstantFPSDNode>(N0.getOperand(1))) 6012263508Sdim return DAG.getNode(ISD::FADD, SDLoc(N), VT, N0.getOperand(0), 6013263508Sdim DAG.getNode(ISD::FADD, SDLoc(N), VT, 6014193323Sed N0.getOperand(1), N1)); 6015193323Sed 6016249423Sdim // No FP constant should be created after legalization as Instruction 6017249423Sdim // Selection pass has hard time in dealing with FP constant. 6018249423Sdim // 6019249423Sdim // We don't need test this condition for transformation like following, as 6020249423Sdim // the DAG being transformed implies it is legal to take FP constant as 6021249423Sdim // operand. 6022263508Sdim // 6023249423Sdim // (fadd (fmul c, x), x) -> (fmul c+1, x) 6024263508Sdim // 6025249423Sdim bool AllowNewFpConst = (Level < AfterLegalizeDAG); 6026249423Sdim 6027243830Sdim // If allow, fold (fadd (fneg x), x) -> 0.0 6028249423Sdim if (AllowNewFpConst && DAG.getTarget().Options.UnsafeFPMath && 6029263508Sdim N0.getOpcode() == ISD::FNEG && N0.getOperand(0) == N1) 6030243830Sdim return DAG.getConstantFP(0.0, VT); 6031243830Sdim 6032243830Sdim // If allow, fold (fadd x, (fneg x)) -> 0.0 6033249423Sdim if (AllowNewFpConst && DAG.getTarget().Options.UnsafeFPMath && 6034263508Sdim N1.getOpcode() == ISD::FNEG && N1.getOperand(0) == N0) 6035243830Sdim return DAG.getConstantFP(0.0, VT); 6036243830Sdim 6037243830Sdim // In unsafe math mode, we can fold chains of FADD's of the same value 6038243830Sdim // into multiplications. This transform is not safe in general because 6039243830Sdim // we are reducing the number of rounding steps. 6040243830Sdim if (DAG.getTarget().Options.UnsafeFPMath && 6041243830Sdim TLI.isOperationLegalOrCustom(ISD::FMUL, VT) && 6042243830Sdim !N0CFP && !N1CFP) { 6043243830Sdim if (N0.getOpcode() == ISD::FMUL) { 6044243830Sdim ConstantFPSDNode *CFP00 = dyn_cast<ConstantFPSDNode>(N0.getOperand(0)); 6045243830Sdim ConstantFPSDNode *CFP01 = dyn_cast<ConstantFPSDNode>(N0.getOperand(1)); 6046243830Sdim 6047263508Sdim // (fadd (fmul c, x), x) -> (fmul x, c+1) 6048243830Sdim if (CFP00 && !CFP01 && N0.getOperand(1) == N1) { 6049263508Sdim SDValue NewCFP = DAG.getNode(ISD::FADD, SDLoc(N), VT, 6050243830Sdim SDValue(CFP00, 0), 6051243830Sdim DAG.getConstantFP(1.0, VT)); 6052263508Sdim return DAG.getNode(ISD::FMUL, SDLoc(N), VT, 6053243830Sdim N1, NewCFP); 6054243830Sdim } 6055243830Sdim 6056263508Sdim // (fadd (fmul x, c), x) -> (fmul x, c+1) 6057243830Sdim if (CFP01 && !CFP00 && N0.getOperand(0) == N1) { 6058263508Sdim SDValue NewCFP = DAG.getNode(ISD::FADD, SDLoc(N), VT, 6059243830Sdim SDValue(CFP01, 0), 6060243830Sdim DAG.getConstantFP(1.0, VT)); 6061263508Sdim return DAG.getNode(ISD::FMUL, SDLoc(N), VT, 6062243830Sdim N1, NewCFP); 6063243830Sdim } 6064243830Sdim 6065263508Sdim // (fadd (fmul c, x), (fadd x, x)) -> (fmul x, c+2) 6066243830Sdim if (CFP00 && !CFP01 && N1.getOpcode() == ISD::FADD && 6067243830Sdim N1.getOperand(0) == N1.getOperand(1) && 6068243830Sdim N0.getOperand(1) == N1.getOperand(0)) { 6069263508Sdim SDValue NewCFP = DAG.getNode(ISD::FADD, SDLoc(N), VT, 6070243830Sdim SDValue(CFP00, 0), 6071243830Sdim DAG.getConstantFP(2.0, VT)); 6072263508Sdim return DAG.getNode(ISD::FMUL, SDLoc(N), VT, 6073243830Sdim N0.getOperand(1), NewCFP); 6074243830Sdim } 6075243830Sdim 6076263508Sdim // (fadd (fmul x, c), (fadd x, x)) -> (fmul x, c+2) 6077243830Sdim if (CFP01 && !CFP00 && N1.getOpcode() == ISD::FADD && 6078243830Sdim N1.getOperand(0) == N1.getOperand(1) && 6079243830Sdim N0.getOperand(0) == N1.getOperand(0)) { 6080263508Sdim SDValue NewCFP = DAG.getNode(ISD::FADD, SDLoc(N), VT, 6081243830Sdim SDValue(CFP01, 0), 6082243830Sdim DAG.getConstantFP(2.0, VT)); 6083263508Sdim return DAG.getNode(ISD::FMUL, SDLoc(N), VT, 6084243830Sdim N0.getOperand(0), NewCFP); 6085243830Sdim } 6086243830Sdim } 6087243830Sdim 6088243830Sdim if (N1.getOpcode() == ISD::FMUL) { 6089243830Sdim ConstantFPSDNode *CFP10 = dyn_cast<ConstantFPSDNode>(N1.getOperand(0)); 6090243830Sdim ConstantFPSDNode *CFP11 = dyn_cast<ConstantFPSDNode>(N1.getOperand(1)); 6091243830Sdim 6092263508Sdim // (fadd x, (fmul c, x)) -> (fmul x, c+1) 6093243830Sdim if (CFP10 && !CFP11 && N1.getOperand(1) == N0) { 6094263508Sdim SDValue NewCFP = DAG.getNode(ISD::FADD, SDLoc(N), VT, 6095243830Sdim SDValue(CFP10, 0), 6096243830Sdim DAG.getConstantFP(1.0, VT)); 6097263508Sdim return DAG.getNode(ISD::FMUL, SDLoc(N), VT, 6098243830Sdim N0, NewCFP); 6099243830Sdim } 6100243830Sdim 6101263508Sdim // (fadd x, (fmul x, c)) -> (fmul x, c+1) 6102243830Sdim if (CFP11 && !CFP10 && N1.getOperand(0) == N0) { 6103263508Sdim SDValue NewCFP = DAG.getNode(ISD::FADD, SDLoc(N), VT, 6104243830Sdim SDValue(CFP11, 0), 6105243830Sdim DAG.getConstantFP(1.0, VT)); 6106263508Sdim return DAG.getNode(ISD::FMUL, SDLoc(N), VT, 6107243830Sdim N0, NewCFP); 6108243830Sdim } 6109243830Sdim 6110243830Sdim 6111263508Sdim // (fadd (fadd x, x), (fmul c, x)) -> (fmul x, c+2) 6112263508Sdim if (CFP10 && !CFP11 && N0.getOpcode() == ISD::FADD && 6113263508Sdim N0.getOperand(0) == N0.getOperand(1) && 6114263508Sdim N1.getOperand(1) == N0.getOperand(0)) { 6115263508Sdim SDValue NewCFP = DAG.getNode(ISD::FADD, SDLoc(N), VT, 6116243830Sdim SDValue(CFP10, 0), 6117243830Sdim DAG.getConstantFP(2.0, VT)); 6118263508Sdim return DAG.getNode(ISD::FMUL, SDLoc(N), VT, 6119263508Sdim N1.getOperand(1), NewCFP); 6120243830Sdim } 6121243830Sdim 6122263508Sdim // (fadd (fadd x, x), (fmul x, c)) -> (fmul x, c+2) 6123263508Sdim if (CFP11 && !CFP10 && N0.getOpcode() == ISD::FADD && 6124263508Sdim N0.getOperand(0) == N0.getOperand(1) && 6125263508Sdim N1.getOperand(0) == N0.getOperand(0)) { 6126263508Sdim SDValue NewCFP = DAG.getNode(ISD::FADD, SDLoc(N), VT, 6127243830Sdim SDValue(CFP11, 0), 6128243830Sdim DAG.getConstantFP(2.0, VT)); 6129263508Sdim return DAG.getNode(ISD::FMUL, SDLoc(N), VT, 6130263508Sdim N1.getOperand(0), NewCFP); 6131243830Sdim } 6132243830Sdim } 6133243830Sdim 6134249423Sdim if (N0.getOpcode() == ISD::FADD && AllowNewFpConst) { 6135249423Sdim ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N0.getOperand(0)); 6136263508Sdim // (fadd (fadd x, x), x) -> (fmul x, 3.0) 6137249423Sdim if (!CFP && N0.getOperand(0) == N0.getOperand(1) && 6138263508Sdim (N0.getOperand(0) == N1)) 6139263508Sdim return DAG.getNode(ISD::FMUL, SDLoc(N), VT, 6140249423Sdim N1, DAG.getConstantFP(3.0, VT)); 6141249423Sdim } 6142249423Sdim 6143249423Sdim if (N1.getOpcode() == ISD::FADD && AllowNewFpConst) { 6144249423Sdim ConstantFPSDNode *CFP10 = dyn_cast<ConstantFPSDNode>(N1.getOperand(0)); 6145263508Sdim // (fadd x, (fadd x, x)) -> (fmul x, 3.0) 6146249423Sdim if (!CFP10 && N1.getOperand(0) == N1.getOperand(1) && 6147263508Sdim N1.getOperand(0) == N0) 6148263508Sdim return DAG.getNode(ISD::FMUL, SDLoc(N), VT, 6149249423Sdim N0, DAG.getConstantFP(3.0, VT)); 6150249423Sdim } 6151249423Sdim 6152263508Sdim // (fadd (fadd x, x), (fadd x, x)) -> (fmul x, 4.0) 6153249423Sdim if (AllowNewFpConst && 6154249423Sdim N0.getOpcode() == ISD::FADD && N1.getOpcode() == ISD::FADD && 6155243830Sdim N0.getOperand(0) == N0.getOperand(1) && 6156243830Sdim N1.getOperand(0) == N1.getOperand(1) && 6157263508Sdim N0.getOperand(0) == N1.getOperand(0)) 6158263508Sdim return DAG.getNode(ISD::FMUL, SDLoc(N), VT, 6159243830Sdim N0.getOperand(0), 6160243830Sdim DAG.getConstantFP(4.0, VT)); 6161243830Sdim } 6162243830Sdim 6163239462Sdim // FADD -> FMA combines: 6164239462Sdim if ((DAG.getTarget().Options.AllowFPOpFusion == FPOpFusion::Fast || 6165239462Sdim DAG.getTarget().Options.UnsafeFPMath) && 6166263508Sdim DAG.getTarget().getTargetLowering()->isFMAFasterThanFMulAndFAdd(VT) && 6167263508Sdim (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT))) { 6168239462Sdim 6169239462Sdim // fold (fadd (fmul x, y), z) -> (fma x, y, z) 6170263508Sdim if (N0.getOpcode() == ISD::FMUL && N0->hasOneUse()) 6171263508Sdim return DAG.getNode(ISD::FMA, SDLoc(N), VT, 6172239462Sdim N0.getOperand(0), N0.getOperand(1), N1); 6173243830Sdim 6174243830Sdim // fold (fadd x, (fmul y, z)) -> (fma y, z, x) 6175239462Sdim // Note: Commutes FADD operands. 6176263508Sdim if (N1.getOpcode() == ISD::FMUL && N1->hasOneUse()) 6177263508Sdim return DAG.getNode(ISD::FMA, SDLoc(N), VT, 6178239462Sdim N1.getOperand(0), N1.getOperand(1), N0); 6179239462Sdim } 6180239462Sdim 6181193323Sed return SDValue(); 6182193323Sed} 6183193323Sed 6184193323SedSDValue DAGCombiner::visitFSUB(SDNode *N) { 6185193323Sed SDValue N0 = N->getOperand(0); 6186193323Sed SDValue N1 = N->getOperand(1); 6187193323Sed ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0); 6188193323Sed ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); 6189198090Srdivacky EVT VT = N->getValueType(0); 6190263508Sdim SDLoc dl(N); 6191193323Sed 6192193323Sed // fold vector ops 6193193323Sed if (VT.isVector()) { 6194193323Sed SDValue FoldedVOp = SimplifyVBinOp(N); 6195193323Sed if (FoldedVOp.getNode()) return FoldedVOp; 6196193323Sed } 6197193323Sed 6198193323Sed // fold (fsub c1, c2) -> c1-c2 6199243830Sdim if (N0CFP && N1CFP) 6200263508Sdim return DAG.getNode(ISD::FSUB, SDLoc(N), VT, N0, N1); 6201193323Sed // fold (fsub A, 0) -> A 6202234353Sdim if (DAG.getTarget().Options.UnsafeFPMath && 6203234353Sdim N1CFP && N1CFP->getValueAPF().isZero()) 6204193323Sed return N0; 6205193323Sed // fold (fsub 0, B) -> -B 6206234353Sdim if (DAG.getTarget().Options.UnsafeFPMath && 6207234353Sdim N0CFP && N0CFP->getValueAPF().isZero()) { 6208234353Sdim if (isNegatibleForFree(N1, LegalOperations, TLI, &DAG.getTarget().Options)) 6209193323Sed return GetNegatedExpression(N1, DAG, LegalOperations); 6210193323Sed if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) 6211239462Sdim return DAG.getNode(ISD::FNEG, dl, VT, N1); 6212193323Sed } 6213193323Sed // fold (fsub A, (fneg B)) -> (fadd A, B) 6214234353Sdim if (isNegatibleForFree(N1, LegalOperations, TLI, &DAG.getTarget().Options)) 6215239462Sdim return DAG.getNode(ISD::FADD, dl, VT, N0, 6216193323Sed GetNegatedExpression(N1, DAG, LegalOperations)); 6217193323Sed 6218234353Sdim // If 'unsafe math' is enabled, fold 6219239462Sdim // (fsub x, x) -> 0.0 & 6220234353Sdim // (fsub x, (fadd x, y)) -> (fneg y) & 6221234353Sdim // (fsub x, (fadd y, x)) -> (fneg y) 6222234353Sdim if (DAG.getTarget().Options.UnsafeFPMath) { 6223239462Sdim if (N0 == N1) 6224239462Sdim return DAG.getConstantFP(0.0f, VT); 6225239462Sdim 6226234353Sdim if (N1.getOpcode() == ISD::FADD) { 6227234353Sdim SDValue N10 = N1->getOperand(0); 6228234353Sdim SDValue N11 = N1->getOperand(1); 6229234353Sdim 6230234353Sdim if (N10 == N0 && isNegatibleForFree(N11, LegalOperations, TLI, 6231234353Sdim &DAG.getTarget().Options)) 6232234353Sdim return GetNegatedExpression(N11, DAG, LegalOperations); 6233263508Sdim 6234263508Sdim if (N11 == N0 && isNegatibleForFree(N10, LegalOperations, TLI, 6235263508Sdim &DAG.getTarget().Options)) 6236234353Sdim return GetNegatedExpression(N10, DAG, LegalOperations); 6237234353Sdim } 6238234353Sdim } 6239234353Sdim 6240239462Sdim // FSUB -> FMA combines: 6241239462Sdim if ((DAG.getTarget().Options.AllowFPOpFusion == FPOpFusion::Fast || 6242239462Sdim DAG.getTarget().Options.UnsafeFPMath) && 6243263508Sdim DAG.getTarget().getTargetLowering()->isFMAFasterThanFMulAndFAdd(VT) && 6244263508Sdim (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT))) { 6245239462Sdim 6246239462Sdim // fold (fsub (fmul x, y), z) -> (fma x, y, (fneg z)) 6247263508Sdim if (N0.getOpcode() == ISD::FMUL && N0->hasOneUse()) 6248239462Sdim return DAG.getNode(ISD::FMA, dl, VT, 6249239462Sdim N0.getOperand(0), N0.getOperand(1), 6250239462Sdim DAG.getNode(ISD::FNEG, dl, VT, N1)); 6251239462Sdim 6252239462Sdim // fold (fsub x, (fmul y, z)) -> (fma (fneg y), z, x) 6253239462Sdim // Note: Commutes FSUB operands. 6254263508Sdim if (N1.getOpcode() == ISD::FMUL && N1->hasOneUse()) 6255239462Sdim return DAG.getNode(ISD::FMA, dl, VT, 6256239462Sdim DAG.getNode(ISD::FNEG, dl, VT, 6257239462Sdim N1.getOperand(0)), 6258239462Sdim N1.getOperand(1), N0); 6259239462Sdim 6260263508Sdim // fold (fsub (fneg (fmul, x, y)), z) -> (fma (fneg x), y, (fneg z)) 6261263508Sdim if (N0.getOpcode() == ISD::FNEG && 6262239462Sdim N0.getOperand(0).getOpcode() == ISD::FMUL && 6263239462Sdim N0->hasOneUse() && N0.getOperand(0).hasOneUse()) { 6264239462Sdim SDValue N00 = N0.getOperand(0).getOperand(0); 6265239462Sdim SDValue N01 = N0.getOperand(0).getOperand(1); 6266239462Sdim return DAG.getNode(ISD::FMA, dl, VT, 6267239462Sdim DAG.getNode(ISD::FNEG, dl, VT, N00), N01, 6268239462Sdim DAG.getNode(ISD::FNEG, dl, VT, N1)); 6269239462Sdim } 6270239462Sdim } 6271239462Sdim 6272193323Sed return SDValue(); 6273193323Sed} 6274193323Sed 6275193323SedSDValue DAGCombiner::visitFMUL(SDNode *N) { 6276193323Sed SDValue N0 = N->getOperand(0); 6277193323Sed SDValue N1 = N->getOperand(1); 6278193323Sed ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0); 6279193323Sed ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); 6280198090Srdivacky EVT VT = N->getValueType(0); 6281234353Sdim const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 6282193323Sed 6283193323Sed // fold vector ops 6284193323Sed if (VT.isVector()) { 6285193323Sed SDValue FoldedVOp = SimplifyVBinOp(N); 6286193323Sed if (FoldedVOp.getNode()) return FoldedVOp; 6287193323Sed } 6288193323Sed 6289193323Sed // fold (fmul c1, c2) -> c1*c2 6290243830Sdim if (N0CFP && N1CFP) 6291263508Sdim return DAG.getNode(ISD::FMUL, SDLoc(N), VT, N0, N1); 6292193323Sed // canonicalize constant to RHS 6293193323Sed if (N0CFP && !N1CFP) 6294263508Sdim return DAG.getNode(ISD::FMUL, SDLoc(N), VT, N1, N0); 6295193323Sed // fold (fmul A, 0) -> 0 6296234353Sdim if (DAG.getTarget().Options.UnsafeFPMath && 6297234353Sdim N1CFP && N1CFP->getValueAPF().isZero()) 6298193323Sed return N1; 6299193574Sed // fold (fmul A, 0) -> 0, vector edition. 6300234353Sdim if (DAG.getTarget().Options.UnsafeFPMath && 6301234353Sdim ISD::isBuildVectorAllZeros(N1.getNode())) 6302193574Sed return N1; 6303239462Sdim // fold (fmul A, 1.0) -> A 6304239462Sdim if (N1CFP && N1CFP->isExactlyValue(1.0)) 6305239462Sdim return N0; 6306193323Sed // fold (fmul X, 2.0) -> (fadd X, X) 6307193323Sed if (N1CFP && N1CFP->isExactlyValue(+2.0)) 6308263508Sdim return DAG.getNode(ISD::FADD, SDLoc(N), VT, N0, N0); 6309198090Srdivacky // fold (fmul X, -1.0) -> (fneg X) 6310193323Sed if (N1CFP && N1CFP->isExactlyValue(-1.0)) 6311193323Sed if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) 6312263508Sdim return DAG.getNode(ISD::FNEG, SDLoc(N), VT, N0); 6313193323Sed 6314193323Sed // fold (fmul (fneg X), (fneg Y)) -> (fmul X, Y) 6315234353Sdim if (char LHSNeg = isNegatibleForFree(N0, LegalOperations, TLI, 6316234353Sdim &DAG.getTarget().Options)) { 6317263508Sdim if (char RHSNeg = isNegatibleForFree(N1, LegalOperations, TLI, 6318234353Sdim &DAG.getTarget().Options)) { 6319193323Sed // Both can be negated for free, check to see if at least one is cheaper 6320193323Sed // negated. 6321193323Sed if (LHSNeg == 2 || RHSNeg == 2) 6322263508Sdim return DAG.getNode(ISD::FMUL, SDLoc(N), VT, 6323193323Sed GetNegatedExpression(N0, DAG, LegalOperations), 6324193323Sed GetNegatedExpression(N1, DAG, LegalOperations)); 6325193323Sed } 6326193323Sed } 6327193323Sed 6328193323Sed // If allowed, fold (fmul (fmul x, c1), c2) -> (fmul x, (fmul c1, c2)) 6329234353Sdim if (DAG.getTarget().Options.UnsafeFPMath && 6330234353Sdim N1CFP && N0.getOpcode() == ISD::FMUL && 6331193323Sed N0.getNode()->hasOneUse() && isa<ConstantFPSDNode>(N0.getOperand(1))) 6332263508Sdim return DAG.getNode(ISD::FMUL, SDLoc(N), VT, N0.getOperand(0), 6333263508Sdim DAG.getNode(ISD::FMUL, SDLoc(N), VT, 6334193323Sed N0.getOperand(1), N1)); 6335193323Sed 6336193323Sed return SDValue(); 6337193323Sed} 6338193323Sed 6339239462SdimSDValue DAGCombiner::visitFMA(SDNode *N) { 6340239462Sdim SDValue N0 = N->getOperand(0); 6341239462Sdim SDValue N1 = N->getOperand(1); 6342239462Sdim SDValue N2 = N->getOperand(2); 6343239462Sdim ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0); 6344239462Sdim ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); 6345239462Sdim EVT VT = N->getValueType(0); 6346263508Sdim SDLoc dl(N); 6347239462Sdim 6348243830Sdim if (DAG.getTarget().Options.UnsafeFPMath) { 6349243830Sdim if (N0CFP && N0CFP->isZero()) 6350243830Sdim return N2; 6351243830Sdim if (N1CFP && N1CFP->isZero()) 6352243830Sdim return N2; 6353243830Sdim } 6354239462Sdim if (N0CFP && N0CFP->isExactlyValue(1.0)) 6355263508Sdim return DAG.getNode(ISD::FADD, SDLoc(N), VT, N1, N2); 6356239462Sdim if (N1CFP && N1CFP->isExactlyValue(1.0)) 6357263508Sdim return DAG.getNode(ISD::FADD, SDLoc(N), VT, N0, N2); 6358239462Sdim 6359239462Sdim // Canonicalize (fma c, x, y) -> (fma x, c, y) 6360239462Sdim if (N0CFP && !N1CFP) 6361263508Sdim return DAG.getNode(ISD::FMA, SDLoc(N), VT, N1, N0, N2); 6362239462Sdim 6363243830Sdim // (fma x, c1, (fmul x, c2)) -> (fmul x, c1+c2) 6364243830Sdim if (DAG.getTarget().Options.UnsafeFPMath && N1CFP && 6365243830Sdim N2.getOpcode() == ISD::FMUL && 6366243830Sdim N0 == N2.getOperand(0) && 6367243830Sdim N2.getOperand(1).getOpcode() == ISD::ConstantFP) { 6368243830Sdim return DAG.getNode(ISD::FMUL, dl, VT, N0, 6369243830Sdim DAG.getNode(ISD::FADD, dl, VT, N1, N2.getOperand(1))); 6370243830Sdim } 6371243830Sdim 6372243830Sdim 6373243830Sdim // (fma (fmul x, c1), c2, y) -> (fma x, c1*c2, y) 6374243830Sdim if (DAG.getTarget().Options.UnsafeFPMath && 6375243830Sdim N0.getOpcode() == ISD::FMUL && N1CFP && 6376243830Sdim N0.getOperand(1).getOpcode() == ISD::ConstantFP) { 6377243830Sdim return DAG.getNode(ISD::FMA, dl, VT, 6378243830Sdim N0.getOperand(0), 6379243830Sdim DAG.getNode(ISD::FMUL, dl, VT, N1, N0.getOperand(1)), 6380243830Sdim N2); 6381243830Sdim } 6382243830Sdim 6383243830Sdim // (fma x, 1, y) -> (fadd x, y) 6384243830Sdim // (fma x, -1, y) -> (fadd (fneg x), y) 6385243830Sdim if (N1CFP) { 6386243830Sdim if (N1CFP->isExactlyValue(1.0)) 6387243830Sdim return DAG.getNode(ISD::FADD, dl, VT, N0, N2); 6388243830Sdim 6389243830Sdim if (N1CFP->isExactlyValue(-1.0) && 6390243830Sdim (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))) { 6391243830Sdim SDValue RHSNeg = DAG.getNode(ISD::FNEG, dl, VT, N0); 6392243830Sdim AddToWorkList(RHSNeg.getNode()); 6393243830Sdim return DAG.getNode(ISD::FADD, dl, VT, N2, RHSNeg); 6394243830Sdim } 6395243830Sdim } 6396243830Sdim 6397243830Sdim // (fma x, c, x) -> (fmul x, (c+1)) 6398263508Sdim if (DAG.getTarget().Options.UnsafeFPMath && N1CFP && N0 == N2) 6399263508Sdim return DAG.getNode(ISD::FMUL, dl, VT, N0, 6400243830Sdim DAG.getNode(ISD::FADD, dl, VT, 6401243830Sdim N1, DAG.getConstantFP(1.0, VT))); 6402243830Sdim 6403243830Sdim // (fma x, c, (fneg x)) -> (fmul x, (c-1)) 6404243830Sdim if (DAG.getTarget().Options.UnsafeFPMath && N1CFP && 6405263508Sdim N2.getOpcode() == ISD::FNEG && N2.getOperand(0) == N0) 6406263508Sdim return DAG.getNode(ISD::FMUL, dl, VT, N0, 6407243830Sdim DAG.getNode(ISD::FADD, dl, VT, 6408243830Sdim N1, DAG.getConstantFP(-1.0, VT))); 6409243830Sdim 6410243830Sdim 6411239462Sdim return SDValue(); 6412239462Sdim} 6413239462Sdim 6414193323SedSDValue DAGCombiner::visitFDIV(SDNode *N) { 6415193323Sed SDValue N0 = N->getOperand(0); 6416193323Sed SDValue N1 = N->getOperand(1); 6417193323Sed ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0); 6418193323Sed ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); 6419198090Srdivacky EVT VT = N->getValueType(0); 6420234353Sdim const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 6421193323Sed 6422193323Sed // fold vector ops 6423193323Sed if (VT.isVector()) { 6424193323Sed SDValue FoldedVOp = SimplifyVBinOp(N); 6425193323Sed if (FoldedVOp.getNode()) return FoldedVOp; 6426193323Sed } 6427193323Sed 6428193323Sed // fold (fdiv c1, c2) -> c1/c2 6429243830Sdim if (N0CFP && N1CFP) 6430263508Sdim return DAG.getNode(ISD::FDIV, SDLoc(N), VT, N0, N1); 6431193323Sed 6432234353Sdim // fold (fdiv X, c2) -> fmul X, 1/c2 if losing precision is acceptable. 6433243830Sdim if (N1CFP && DAG.getTarget().Options.UnsafeFPMath) { 6434234353Sdim // Compute the reciprocal 1.0 / c2. 6435234353Sdim APFloat N1APF = N1CFP->getValueAPF(); 6436234353Sdim APFloat Recip(N1APF.getSemantics(), 1); // 1.0 6437234353Sdim APFloat::opStatus st = Recip.divide(N1APF, APFloat::rmNearestTiesToEven); 6438234353Sdim // Only do the transform if the reciprocal is a legal fp immediate that 6439234353Sdim // isn't too nasty (eg NaN, denormal, ...). 6440234353Sdim if ((st == APFloat::opOK || st == APFloat::opInexact) && // Not too nasty 6441234353Sdim (!LegalOperations || 6442234353Sdim // FIXME: custom lowering of ConstantFP might fail (see e.g. ARM 6443234353Sdim // backend)... we should handle this gracefully after Legalize. 6444234353Sdim // TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT) || 6445234353Sdim TLI.isOperationLegal(llvm::ISD::ConstantFP, VT) || 6446234353Sdim TLI.isFPImmLegal(Recip, VT))) 6447263508Sdim return DAG.getNode(ISD::FMUL, SDLoc(N), VT, N0, 6448234353Sdim DAG.getConstantFP(Recip, VT)); 6449234353Sdim } 6450193323Sed 6451193323Sed // (fdiv (fneg X), (fneg Y)) -> (fdiv X, Y) 6452234353Sdim if (char LHSNeg = isNegatibleForFree(N0, LegalOperations, TLI, 6453234353Sdim &DAG.getTarget().Options)) { 6454234353Sdim if (char RHSNeg = isNegatibleForFree(N1, LegalOperations, TLI, 6455234353Sdim &DAG.getTarget().Options)) { 6456193323Sed // Both can be negated for free, check to see if at least one is cheaper 6457193323Sed // negated. 6458193323Sed if (LHSNeg == 2 || RHSNeg == 2) 6459263508Sdim return DAG.getNode(ISD::FDIV, SDLoc(N), VT, 6460193323Sed GetNegatedExpression(N0, DAG, LegalOperations), 6461193323Sed GetNegatedExpression(N1, DAG, LegalOperations)); 6462193323Sed } 6463193323Sed } 6464193323Sed 6465193323Sed return SDValue(); 6466193323Sed} 6467193323Sed 6468193323SedSDValue DAGCombiner::visitFREM(SDNode *N) { 6469193323Sed SDValue N0 = N->getOperand(0); 6470193323Sed SDValue N1 = N->getOperand(1); 6471193323Sed ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0); 6472193323Sed ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); 6473198090Srdivacky EVT VT = N->getValueType(0); 6474193323Sed 6475193323Sed // fold (frem c1, c2) -> fmod(c1,c2) 6476243830Sdim if (N0CFP && N1CFP) 6477263508Sdim return DAG.getNode(ISD::FREM, SDLoc(N), VT, N0, N1); 6478193323Sed 6479193323Sed return SDValue(); 6480193323Sed} 6481193323Sed 6482193323SedSDValue DAGCombiner::visitFCOPYSIGN(SDNode *N) { 6483193323Sed SDValue N0 = N->getOperand(0); 6484193323Sed SDValue N1 = N->getOperand(1); 6485193323Sed ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0); 6486193323Sed ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); 6487198090Srdivacky EVT VT = N->getValueType(0); 6488193323Sed 6489243830Sdim if (N0CFP && N1CFP) // Constant fold 6490263508Sdim return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0, N1); 6491193323Sed 6492193323Sed if (N1CFP) { 6493193323Sed const APFloat& V = N1CFP->getValueAPF(); 6494193323Sed // copysign(x, c1) -> fabs(x) iff ispos(c1) 6495193323Sed // copysign(x, c1) -> fneg(fabs(x)) iff isneg(c1) 6496193323Sed if (!V.isNegative()) { 6497193323Sed if (!LegalOperations || TLI.isOperationLegal(ISD::FABS, VT)) 6498263508Sdim return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); 6499193323Sed } else { 6500193323Sed if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) 6501263508Sdim return DAG.getNode(ISD::FNEG, SDLoc(N), VT, 6502263508Sdim DAG.getNode(ISD::FABS, SDLoc(N0), VT, N0)); 6503193323Sed } 6504193323Sed } 6505193323Sed 6506193323Sed // copysign(fabs(x), y) -> copysign(x, y) 6507193323Sed // copysign(fneg(x), y) -> copysign(x, y) 6508193323Sed // copysign(copysign(x,z), y) -> copysign(x, y) 6509193323Sed if (N0.getOpcode() == ISD::FABS || N0.getOpcode() == ISD::FNEG || 6510193323Sed N0.getOpcode() == ISD::FCOPYSIGN) 6511263508Sdim return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, 6512193323Sed N0.getOperand(0), N1); 6513193323Sed 6514193323Sed // copysign(x, abs(y)) -> abs(x) 6515193323Sed if (N1.getOpcode() == ISD::FABS) 6516263508Sdim return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); 6517193323Sed 6518193323Sed // copysign(x, copysign(y,z)) -> copysign(x, z) 6519193323Sed if (N1.getOpcode() == ISD::FCOPYSIGN) 6520263508Sdim return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, 6521193323Sed N0, N1.getOperand(1)); 6522193323Sed 6523193323Sed // copysign(x, fp_extend(y)) -> copysign(x, y) 6524193323Sed // copysign(x, fp_round(y)) -> copysign(x, y) 6525193323Sed if (N1.getOpcode() == ISD::FP_EXTEND || N1.getOpcode() == ISD::FP_ROUND) 6526263508Sdim return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, 6527193323Sed N0, N1.getOperand(0)); 6528193323Sed 6529193323Sed return SDValue(); 6530193323Sed} 6531193323Sed 6532193323SedSDValue DAGCombiner::visitSINT_TO_FP(SDNode *N) { 6533193323Sed SDValue N0 = N->getOperand(0); 6534193323Sed ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0); 6535198090Srdivacky EVT VT = N->getValueType(0); 6536198090Srdivacky EVT OpVT = N0.getValueType(); 6537193323Sed 6538193323Sed // fold (sint_to_fp c1) -> c1fp 6539243830Sdim if (N0C && 6540221345Sdim // ...but only if the target supports immediate floating-point values 6541234353Sdim (!LegalOperations || 6542224145Sdim TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT))) 6543263508Sdim return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0); 6544193323Sed 6545193323Sed // If the input is a legal type, and SINT_TO_FP is not legal on this target, 6546193323Sed // but UINT_TO_FP is legal on this target, try to convert. 6547193323Sed if (!TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, OpVT) && 6548193323Sed TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, OpVT)) { 6549193323Sed // If the sign bit is known to be zero, we can change this to UINT_TO_FP. 6550193323Sed if (DAG.SignBitIsZero(N0)) 6551263508Sdim return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0); 6552193323Sed } 6553193323Sed 6554239462Sdim // The next optimizations are desireable only if SELECT_CC can be lowered. 6555239462Sdim // Check against MVT::Other for SELECT_CC, which is a workaround for targets 6556239462Sdim // having to say they don't support SELECT_CC on every type the DAG knows 6557239462Sdim // about, since there is no way to mark an opcode illegal at all value types 6558239462Sdim // (See also visitSELECT) 6559239462Sdim if (TLI.isOperationLegalOrCustom(ISD::SELECT_CC, MVT::Other)) { 6560239462Sdim // fold (sint_to_fp (setcc x, y, cc)) -> (select_cc x, y, -1.0, 0.0,, cc) 6561239462Sdim if (N0.getOpcode() == ISD::SETCC && N0.getValueType() == MVT::i1 && 6562239462Sdim !VT.isVector() && 6563239462Sdim (!LegalOperations || 6564239462Sdim TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT))) { 6565239462Sdim SDValue Ops[] = 6566239462Sdim { N0.getOperand(0), N0.getOperand(1), 6567239462Sdim DAG.getConstantFP(-1.0, VT) , DAG.getConstantFP(0.0, VT), 6568239462Sdim N0.getOperand(2) }; 6569263508Sdim return DAG.getNode(ISD::SELECT_CC, SDLoc(N), VT, Ops, 5); 6570239462Sdim } 6571239462Sdim 6572239462Sdim // fold (sint_to_fp (zext (setcc x, y, cc))) -> 6573239462Sdim // (select_cc x, y, 1.0, 0.0,, cc) 6574239462Sdim if (N0.getOpcode() == ISD::ZERO_EXTEND && 6575239462Sdim N0.getOperand(0).getOpcode() == ISD::SETCC &&!VT.isVector() && 6576239462Sdim (!LegalOperations || 6577239462Sdim TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT))) { 6578239462Sdim SDValue Ops[] = 6579239462Sdim { N0.getOperand(0).getOperand(0), N0.getOperand(0).getOperand(1), 6580239462Sdim DAG.getConstantFP(1.0, VT) , DAG.getConstantFP(0.0, VT), 6581239462Sdim N0.getOperand(0).getOperand(2) }; 6582263508Sdim return DAG.getNode(ISD::SELECT_CC, SDLoc(N), VT, Ops, 5); 6583239462Sdim } 6584239462Sdim } 6585239462Sdim 6586193323Sed return SDValue(); 6587193323Sed} 6588193323Sed 6589193323SedSDValue DAGCombiner::visitUINT_TO_FP(SDNode *N) { 6590193323Sed SDValue N0 = N->getOperand(0); 6591193323Sed ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0); 6592198090Srdivacky EVT VT = N->getValueType(0); 6593198090Srdivacky EVT OpVT = N0.getValueType(); 6594193323Sed 6595193323Sed // fold (uint_to_fp c1) -> c1fp 6596243830Sdim if (N0C && 6597221345Sdim // ...but only if the target supports immediate floating-point values 6598234353Sdim (!LegalOperations || 6599224145Sdim TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT))) 6600263508Sdim return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0); 6601193323Sed 6602193323Sed // If the input is a legal type, and UINT_TO_FP is not legal on this target, 6603193323Sed // but SINT_TO_FP is legal on this target, try to convert. 6604193323Sed if (!TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, OpVT) && 6605193323Sed TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, OpVT)) { 6606193323Sed // If the sign bit is known to be zero, we can change this to SINT_TO_FP. 6607193323Sed if (DAG.SignBitIsZero(N0)) 6608263508Sdim return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0); 6609193323Sed } 6610193323Sed 6611239462Sdim // The next optimizations are desireable only if SELECT_CC can be lowered. 6612239462Sdim // Check against MVT::Other for SELECT_CC, which is a workaround for targets 6613239462Sdim // having to say they don't support SELECT_CC on every type the DAG knows 6614239462Sdim // about, since there is no way to mark an opcode illegal at all value types 6615239462Sdim // (See also visitSELECT) 6616239462Sdim if (TLI.isOperationLegalOrCustom(ISD::SELECT_CC, MVT::Other)) { 6617239462Sdim // fold (uint_to_fp (setcc x, y, cc)) -> (select_cc x, y, -1.0, 0.0,, cc) 6618239462Sdim 6619239462Sdim if (N0.getOpcode() == ISD::SETCC && !VT.isVector() && 6620239462Sdim (!LegalOperations || 6621239462Sdim TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT))) { 6622239462Sdim SDValue Ops[] = 6623239462Sdim { N0.getOperand(0), N0.getOperand(1), 6624239462Sdim DAG.getConstantFP(1.0, VT), DAG.getConstantFP(0.0, VT), 6625239462Sdim N0.getOperand(2) }; 6626263508Sdim return DAG.getNode(ISD::SELECT_CC, SDLoc(N), VT, Ops, 5); 6627239462Sdim } 6628239462Sdim } 6629239462Sdim 6630193323Sed return SDValue(); 6631193323Sed} 6632193323Sed 6633193323SedSDValue DAGCombiner::visitFP_TO_SINT(SDNode *N) { 6634193323Sed SDValue N0 = N->getOperand(0); 6635193323Sed ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0); 6636198090Srdivacky EVT VT = N->getValueType(0); 6637193323Sed 6638193323Sed // fold (fp_to_sint c1fp) -> c1 6639193323Sed if (N0CFP) 6640263508Sdim return DAG.getNode(ISD::FP_TO_SINT, SDLoc(N), VT, N0); 6641193323Sed 6642193323Sed return SDValue(); 6643193323Sed} 6644193323Sed 6645193323SedSDValue DAGCombiner::visitFP_TO_UINT(SDNode *N) { 6646193323Sed SDValue N0 = N->getOperand(0); 6647193323Sed ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0); 6648198090Srdivacky EVT VT = N->getValueType(0); 6649193323Sed 6650193323Sed // fold (fp_to_uint c1fp) -> c1 6651243830Sdim if (N0CFP) 6652263508Sdim return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), VT, N0); 6653193323Sed 6654193323Sed return SDValue(); 6655193323Sed} 6656193323Sed 6657193323SedSDValue DAGCombiner::visitFP_ROUND(SDNode *N) { 6658193323Sed SDValue N0 = N->getOperand(0); 6659193323Sed SDValue N1 = N->getOperand(1); 6660193323Sed ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0); 6661198090Srdivacky EVT VT = N->getValueType(0); 6662193323Sed 6663193323Sed // fold (fp_round c1fp) -> c1fp 6664243830Sdim if (N0CFP) 6665263508Sdim return DAG.getNode(ISD::FP_ROUND, SDLoc(N), VT, N0, N1); 6666193323Sed 6667193323Sed // fold (fp_round (fp_extend x)) -> x 6668193323Sed if (N0.getOpcode() == ISD::FP_EXTEND && VT == N0.getOperand(0).getValueType()) 6669193323Sed return N0.getOperand(0); 6670193323Sed 6671193323Sed // fold (fp_round (fp_round x)) -> (fp_round x) 6672193323Sed if (N0.getOpcode() == ISD::FP_ROUND) { 6673193323Sed // This is a value preserving truncation if both round's are. 6674193323Sed bool IsTrunc = N->getConstantOperandVal(1) == 1 && 6675193323Sed N0.getNode()->getConstantOperandVal(1) == 1; 6676263508Sdim return DAG.getNode(ISD::FP_ROUND, SDLoc(N), VT, N0.getOperand(0), 6677193323Sed DAG.getIntPtrConstant(IsTrunc)); 6678193323Sed } 6679193323Sed 6680193323Sed // fold (fp_round (copysign X, Y)) -> (copysign (fp_round X), Y) 6681193323Sed if (N0.getOpcode() == ISD::FCOPYSIGN && N0.getNode()->hasOneUse()) { 6682263508Sdim SDValue Tmp = DAG.getNode(ISD::FP_ROUND, SDLoc(N0), VT, 6683193323Sed N0.getOperand(0), N1); 6684193323Sed AddToWorkList(Tmp.getNode()); 6685263508Sdim return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, 6686193323Sed Tmp, N0.getOperand(1)); 6687193323Sed } 6688193323Sed 6689193323Sed return SDValue(); 6690193323Sed} 6691193323Sed 6692193323SedSDValue DAGCombiner::visitFP_ROUND_INREG(SDNode *N) { 6693193323Sed SDValue N0 = N->getOperand(0); 6694198090Srdivacky EVT VT = N->getValueType(0); 6695198090Srdivacky EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); 6696193323Sed ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0); 6697193323Sed 6698193323Sed // fold (fp_round_inreg c1fp) -> c1fp 6699207618Srdivacky if (N0CFP && isTypeLegal(EVT)) { 6700193323Sed SDValue Round = DAG.getConstantFP(*N0CFP->getConstantFPValue(), EVT); 6701263508Sdim return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Round); 6702193323Sed } 6703193323Sed 6704193323Sed return SDValue(); 6705193323Sed} 6706193323Sed 6707193323SedSDValue DAGCombiner::visitFP_EXTEND(SDNode *N) { 6708193323Sed SDValue N0 = N->getOperand(0); 6709193323Sed ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0); 6710198090Srdivacky EVT VT = N->getValueType(0); 6711193323Sed 6712193323Sed // If this is fp_round(fpextend), don't fold it, allow ourselves to be folded. 6713193323Sed if (N->hasOneUse() && 6714193323Sed N->use_begin()->getOpcode() == ISD::FP_ROUND) 6715193323Sed return SDValue(); 6716193323Sed 6717193323Sed // fold (fp_extend c1fp) -> c1fp 6718243830Sdim if (N0CFP) 6719263508Sdim return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, N0); 6720193323Sed 6721193323Sed // Turn fp_extend(fp_round(X, 1)) -> x since the fp_round doesn't affect the 6722193323Sed // value of X. 6723193323Sed if (N0.getOpcode() == ISD::FP_ROUND 6724193323Sed && N0.getNode()->getConstantOperandVal(1) == 1) { 6725193323Sed SDValue In = N0.getOperand(0); 6726193323Sed if (In.getValueType() == VT) return In; 6727193323Sed if (VT.bitsLT(In.getValueType())) 6728263508Sdim return DAG.getNode(ISD::FP_ROUND, SDLoc(N), VT, 6729193323Sed In, N0.getOperand(1)); 6730263508Sdim return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, In); 6731193323Sed } 6732193323Sed 6733193323Sed // fold (fpext (load x)) -> (fpext (fptrunc (extload x))) 6734263508Sdim if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && 6735193323Sed ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) || 6736193323Sed TLI.isLoadExtLegal(ISD::EXTLOAD, N0.getValueType()))) { 6737193323Sed LoadSDNode *LN0 = cast<LoadSDNode>(N0); 6738263508Sdim SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT, 6739193323Sed LN0->getChain(), 6740263508Sdim LN0->getBasePtr(), N0.getValueType(), 6741263508Sdim LN0->getMemOperand()); 6742193323Sed CombineTo(N, ExtLoad); 6743193323Sed CombineTo(N0.getNode(), 6744263508Sdim DAG.getNode(ISD::FP_ROUND, SDLoc(N0), 6745193323Sed N0.getValueType(), ExtLoad, DAG.getIntPtrConstant(1)), 6746193323Sed ExtLoad.getValue(1)); 6747193323Sed return SDValue(N, 0); // Return N so it doesn't get rechecked! 6748193323Sed } 6749193323Sed 6750193323Sed return SDValue(); 6751193323Sed} 6752193323Sed 6753193323SedSDValue DAGCombiner::visitFNEG(SDNode *N) { 6754193323Sed SDValue N0 = N->getOperand(0); 6755198396Srdivacky EVT VT = N->getValueType(0); 6756193323Sed 6757243830Sdim if (VT.isVector()) { 6758243830Sdim SDValue FoldedVOp = SimplifyVUnaryOp(N); 6759243830Sdim if (FoldedVOp.getNode()) return FoldedVOp; 6760243830Sdim } 6761243830Sdim 6762234353Sdim if (isNegatibleForFree(N0, LegalOperations, DAG.getTargetLoweringInfo(), 6763234353Sdim &DAG.getTarget().Options)) 6764193323Sed return GetNegatedExpression(N0, DAG, LegalOperations); 6765193323Sed 6766193323Sed // Transform fneg(bitconvert(x)) -> bitconvert(x^sign) to avoid loading 6767193323Sed // constant pool values. 6768234353Sdim if (!TLI.isFNegFree(VT) && N0.getOpcode() == ISD::BITCAST && 6769198396Srdivacky !VT.isVector() && 6770198396Srdivacky N0.getNode()->hasOneUse() && 6771198396Srdivacky N0.getOperand(0).getValueType().isInteger()) { 6772193323Sed SDValue Int = N0.getOperand(0); 6773198090Srdivacky EVT IntVT = Int.getValueType(); 6774193323Sed if (IntVT.isInteger() && !IntVT.isVector()) { 6775263508Sdim Int = DAG.getNode(ISD::XOR, SDLoc(N0), IntVT, Int, 6776193323Sed DAG.getConstant(APInt::getSignBit(IntVT.getSizeInBits()), IntVT)); 6777193323Sed AddToWorkList(Int.getNode()); 6778263508Sdim return DAG.getNode(ISD::BITCAST, SDLoc(N), 6779198396Srdivacky VT, Int); 6780193323Sed } 6781193323Sed } 6782193323Sed 6783243830Sdim // (fneg (fmul c, x)) -> (fmul -c, x) 6784243830Sdim if (N0.getOpcode() == ISD::FMUL) { 6785243830Sdim ConstantFPSDNode *CFP1 = dyn_cast<ConstantFPSDNode>(N0.getOperand(1)); 6786263508Sdim if (CFP1) 6787263508Sdim return DAG.getNode(ISD::FMUL, SDLoc(N), VT, 6788243830Sdim N0.getOperand(0), 6789263508Sdim DAG.getNode(ISD::FNEG, SDLoc(N), VT, 6790243830Sdim N0.getOperand(1))); 6791243830Sdim } 6792243830Sdim 6793193323Sed return SDValue(); 6794193323Sed} 6795193323Sed 6796239462SdimSDValue DAGCombiner::visitFCEIL(SDNode *N) { 6797239462Sdim SDValue N0 = N->getOperand(0); 6798239462Sdim ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0); 6799239462Sdim EVT VT = N->getValueType(0); 6800239462Sdim 6801239462Sdim // fold (fceil c1) -> fceil(c1) 6802243830Sdim if (N0CFP) 6803263508Sdim return DAG.getNode(ISD::FCEIL, SDLoc(N), VT, N0); 6804239462Sdim 6805239462Sdim return SDValue(); 6806239462Sdim} 6807239462Sdim 6808239462SdimSDValue DAGCombiner::visitFTRUNC(SDNode *N) { 6809239462Sdim SDValue N0 = N->getOperand(0); 6810239462Sdim ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0); 6811239462Sdim EVT VT = N->getValueType(0); 6812239462Sdim 6813239462Sdim // fold (ftrunc c1) -> ftrunc(c1) 6814243830Sdim if (N0CFP) 6815263508Sdim return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0); 6816239462Sdim 6817239462Sdim return SDValue(); 6818239462Sdim} 6819239462Sdim 6820239462SdimSDValue DAGCombiner::visitFFLOOR(SDNode *N) { 6821239462Sdim SDValue N0 = N->getOperand(0); 6822239462Sdim ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0); 6823239462Sdim EVT VT = N->getValueType(0); 6824239462Sdim 6825239462Sdim // fold (ffloor c1) -> ffloor(c1) 6826243830Sdim if (N0CFP) 6827263508Sdim return DAG.getNode(ISD::FFLOOR, SDLoc(N), VT, N0); 6828239462Sdim 6829239462Sdim return SDValue(); 6830239462Sdim} 6831239462Sdim 6832193323SedSDValue DAGCombiner::visitFABS(SDNode *N) { 6833193323Sed SDValue N0 = N->getOperand(0); 6834193323Sed ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0); 6835198090Srdivacky EVT VT = N->getValueType(0); 6836193323Sed 6837243830Sdim if (VT.isVector()) { 6838243830Sdim SDValue FoldedVOp = SimplifyVUnaryOp(N); 6839243830Sdim if (FoldedVOp.getNode()) return FoldedVOp; 6840243830Sdim } 6841243830Sdim 6842193323Sed // fold (fabs c1) -> fabs(c1) 6843243830Sdim if (N0CFP) 6844263508Sdim return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); 6845193323Sed // fold (fabs (fabs x)) -> (fabs x) 6846193323Sed if (N0.getOpcode() == ISD::FABS) 6847193323Sed return N->getOperand(0); 6848193323Sed // fold (fabs (fneg x)) -> (fabs x) 6849193323Sed // fold (fabs (fcopysign x, y)) -> (fabs x) 6850193323Sed if (N0.getOpcode() == ISD::FNEG || N0.getOpcode() == ISD::FCOPYSIGN) 6851263508Sdim return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0.getOperand(0)); 6852193323Sed 6853193323Sed // Transform fabs(bitconvert(x)) -> bitconvert(x&~sign) to avoid loading 6854193323Sed // constant pool values. 6855263508Sdim if (!TLI.isFAbsFree(VT) && 6856234353Sdim N0.getOpcode() == ISD::BITCAST && N0.getNode()->hasOneUse() && 6857193323Sed N0.getOperand(0).getValueType().isInteger() && 6858193323Sed !N0.getOperand(0).getValueType().isVector()) { 6859193323Sed SDValue Int = N0.getOperand(0); 6860198090Srdivacky EVT IntVT = Int.getValueType(); 6861193323Sed if (IntVT.isInteger() && !IntVT.isVector()) { 6862263508Sdim Int = DAG.getNode(ISD::AND, SDLoc(N0), IntVT, Int, 6863193323Sed DAG.getConstant(~APInt::getSignBit(IntVT.getSizeInBits()), IntVT)); 6864193323Sed AddToWorkList(Int.getNode()); 6865263508Sdim return DAG.getNode(ISD::BITCAST, SDLoc(N), 6866193323Sed N->getValueType(0), Int); 6867193323Sed } 6868193323Sed } 6869193323Sed 6870193323Sed return SDValue(); 6871193323Sed} 6872193323Sed 6873193323SedSDValue DAGCombiner::visitBRCOND(SDNode *N) { 6874193323Sed SDValue Chain = N->getOperand(0); 6875193323Sed SDValue N1 = N->getOperand(1); 6876193323Sed SDValue N2 = N->getOperand(2); 6877193323Sed 6878199481Srdivacky // If N is a constant we could fold this into a fallthrough or unconditional 6879199481Srdivacky // branch. However that doesn't happen very often in normal code, because 6880199481Srdivacky // Instcombine/SimplifyCFG should have handled the available opportunities. 6881199481Srdivacky // If we did this folding here, it would be necessary to update the 6882199481Srdivacky // MachineBasicBlock CFG, which is awkward. 6883199481Srdivacky 6884193323Sed // fold a brcond with a setcc condition into a BR_CC node if BR_CC is legal 6885193323Sed // on the target. 6886193323Sed if (N1.getOpcode() == ISD::SETCC && 6887249423Sdim TLI.isOperationLegalOrCustom(ISD::BR_CC, 6888249423Sdim N1.getOperand(0).getValueType())) { 6889263508Sdim return DAG.getNode(ISD::BR_CC, SDLoc(N), MVT::Other, 6890193323Sed Chain, N1.getOperand(2), 6891193323Sed N1.getOperand(0), N1.getOperand(1), N2); 6892193323Sed } 6893193323Sed 6894218893Sdim if ((N1.hasOneUse() && N1.getOpcode() == ISD::SRL) || 6895218893Sdim ((N1.getOpcode() == ISD::TRUNCATE && N1.hasOneUse()) && 6896218893Sdim (N1.getOperand(0).hasOneUse() && 6897218893Sdim N1.getOperand(0).getOpcode() == ISD::SRL))) { 6898218893Sdim SDNode *Trunc = 0; 6899218893Sdim if (N1.getOpcode() == ISD::TRUNCATE) { 6900218893Sdim // Look pass the truncate. 6901218893Sdim Trunc = N1.getNode(); 6902218893Sdim N1 = N1.getOperand(0); 6903218893Sdim } 6904202375Srdivacky 6905193323Sed // Match this pattern so that we can generate simpler code: 6906193323Sed // 6907193323Sed // %a = ... 6908193323Sed // %b = and i32 %a, 2 6909193323Sed // %c = srl i32 %b, 1 6910193323Sed // brcond i32 %c ... 6911193323Sed // 6912193323Sed // into 6913218893Sdim // 6914193323Sed // %a = ... 6915202375Srdivacky // %b = and i32 %a, 2 6916193323Sed // %c = setcc eq %b, 0 6917193323Sed // brcond %c ... 6918193323Sed // 6919193323Sed // This applies only when the AND constant value has one bit set and the 6920193323Sed // SRL constant is equal to the log2 of the AND constant. The back-end is 6921193323Sed // smart enough to convert the result into a TEST/JMP sequence. 6922193323Sed SDValue Op0 = N1.getOperand(0); 6923193323Sed SDValue Op1 = N1.getOperand(1); 6924193323Sed 6925193323Sed if (Op0.getOpcode() == ISD::AND && 6926193323Sed Op1.getOpcode() == ISD::Constant) { 6927193323Sed SDValue AndOp1 = Op0.getOperand(1); 6928193323Sed 6929193323Sed if (AndOp1.getOpcode() == ISD::Constant) { 6930193323Sed const APInt &AndConst = cast<ConstantSDNode>(AndOp1)->getAPIntValue(); 6931193323Sed 6932193323Sed if (AndConst.isPowerOf2() && 6933193323Sed cast<ConstantSDNode>(Op1)->getAPIntValue()==AndConst.logBase2()) { 6934193323Sed SDValue SetCC = 6935263508Sdim DAG.getSetCC(SDLoc(N), 6936263508Sdim getSetCCResultType(Op0.getValueType()), 6937193323Sed Op0, DAG.getConstant(0, Op0.getValueType()), 6938193323Sed ISD::SETNE); 6939193323Sed 6940263508Sdim SDValue NewBRCond = DAG.getNode(ISD::BRCOND, SDLoc(N), 6941202375Srdivacky MVT::Other, Chain, SetCC, N2); 6942202375Srdivacky // Don't add the new BRCond into the worklist or else SimplifySelectCC 6943202375Srdivacky // will convert it back to (X & C1) >> C2. 6944202375Srdivacky CombineTo(N, NewBRCond, false); 6945202375Srdivacky // Truncate is dead. 6946202375Srdivacky if (Trunc) { 6947202375Srdivacky removeFromWorkList(Trunc); 6948202375Srdivacky DAG.DeleteNode(Trunc); 6949202375Srdivacky } 6950193323Sed // Replace the uses of SRL with SETCC 6951204642Srdivacky WorkListRemover DeadNodes(*this); 6952239462Sdim DAG.ReplaceAllUsesOfValueWith(N1, SetCC); 6953193323Sed removeFromWorkList(N1.getNode()); 6954193323Sed DAG.DeleteNode(N1.getNode()); 6955202375Srdivacky return SDValue(N, 0); // Return N so it doesn't get rechecked! 6956193323Sed } 6957193323Sed } 6958193323Sed } 6959218893Sdim 6960218893Sdim if (Trunc) 6961218893Sdim // Restore N1 if the above transformation doesn't match. 6962218893Sdim N1 = N->getOperand(1); 6963193323Sed } 6964218893Sdim 6965204642Srdivacky // Transform br(xor(x, y)) -> br(x != y) 6966204642Srdivacky // Transform br(xor(xor(x,y), 1)) -> br (x == y) 6967204642Srdivacky if (N1.hasOneUse() && N1.getOpcode() == ISD::XOR) { 6968204642Srdivacky SDNode *TheXor = N1.getNode(); 6969204642Srdivacky SDValue Op0 = TheXor->getOperand(0); 6970204642Srdivacky SDValue Op1 = TheXor->getOperand(1); 6971204642Srdivacky if (Op0.getOpcode() == Op1.getOpcode()) { 6972204642Srdivacky // Avoid missing important xor optimizations. 6973204642Srdivacky SDValue Tmp = visitXOR(TheXor); 6974249423Sdim if (Tmp.getNode()) { 6975249423Sdim if (Tmp.getNode() != TheXor) { 6976249423Sdim DEBUG(dbgs() << "\nReplacing.8 "; 6977249423Sdim TheXor->dump(&DAG); 6978249423Sdim dbgs() << "\nWith: "; 6979249423Sdim Tmp.getNode()->dump(&DAG); 6980249423Sdim dbgs() << '\n'); 6981249423Sdim WorkListRemover DeadNodes(*this); 6982249423Sdim DAG.ReplaceAllUsesOfValueWith(N1, Tmp); 6983249423Sdim removeFromWorkList(TheXor); 6984249423Sdim DAG.DeleteNode(TheXor); 6985263508Sdim return DAG.getNode(ISD::BRCOND, SDLoc(N), 6986249423Sdim MVT::Other, Chain, Tmp, N2); 6987249423Sdim } 6988249423Sdim 6989249423Sdim // visitXOR has changed XOR's operands or replaced the XOR completely, 6990249423Sdim // bail out. 6991249423Sdim return SDValue(N, 0); 6992204642Srdivacky } 6993204642Srdivacky } 6994193323Sed 6995204642Srdivacky if (Op0.getOpcode() != ISD::SETCC && Op1.getOpcode() != ISD::SETCC) { 6996204642Srdivacky bool Equal = false; 6997204642Srdivacky if (ConstantSDNode *RHSCI = dyn_cast<ConstantSDNode>(Op0)) 6998204642Srdivacky if (RHSCI->getAPIntValue() == 1 && Op0.hasOneUse() && 6999204642Srdivacky Op0.getOpcode() == ISD::XOR) { 7000204642Srdivacky TheXor = Op0.getNode(); 7001204642Srdivacky Equal = true; 7002204642Srdivacky } 7003204642Srdivacky 7004218893Sdim EVT SetCCVT = N1.getValueType(); 7005204642Srdivacky if (LegalTypes) 7006263508Sdim SetCCVT = getSetCCResultType(SetCCVT); 7007263508Sdim SDValue SetCC = DAG.getSetCC(SDLoc(TheXor), 7008204642Srdivacky SetCCVT, 7009204642Srdivacky Op0, Op1, 7010204642Srdivacky Equal ? ISD::SETEQ : ISD::SETNE); 7011204642Srdivacky // Replace the uses of XOR with SETCC 7012204642Srdivacky WorkListRemover DeadNodes(*this); 7013239462Sdim DAG.ReplaceAllUsesOfValueWith(N1, SetCC); 7014218893Sdim removeFromWorkList(N1.getNode()); 7015218893Sdim DAG.DeleteNode(N1.getNode()); 7016263508Sdim return DAG.getNode(ISD::BRCOND, SDLoc(N), 7017204642Srdivacky MVT::Other, Chain, SetCC, N2); 7018204642Srdivacky } 7019204642Srdivacky } 7020204642Srdivacky 7021193323Sed return SDValue(); 7022193323Sed} 7023193323Sed 7024193323Sed// Operand List for BR_CC: Chain, CondCC, CondLHS, CondRHS, DestBB. 7025193323Sed// 7026193323SedSDValue DAGCombiner::visitBR_CC(SDNode *N) { 7027193323Sed CondCodeSDNode *CC = cast<CondCodeSDNode>(N->getOperand(1)); 7028193323Sed SDValue CondLHS = N->getOperand(2), CondRHS = N->getOperand(3); 7029193323Sed 7030199481Srdivacky // If N is a constant we could fold this into a fallthrough or unconditional 7031199481Srdivacky // branch. However that doesn't happen very often in normal code, because 7032199481Srdivacky // Instcombine/SimplifyCFG should have handled the available opportunities. 7033199481Srdivacky // If we did this folding here, it would be necessary to update the 7034199481Srdivacky // MachineBasicBlock CFG, which is awkward. 7035199481Srdivacky 7036193323Sed // Use SimplifySetCC to simplify SETCC's. 7037263508Sdim SDValue Simp = SimplifySetCC(getSetCCResultType(CondLHS.getValueType()), 7038263508Sdim CondLHS, CondRHS, CC->get(), SDLoc(N), 7039193323Sed false); 7040193323Sed if (Simp.getNode()) AddToWorkList(Simp.getNode()); 7041193323Sed 7042193323Sed // fold to a simpler setcc 7043193323Sed if (Simp.getNode() && Simp.getOpcode() == ISD::SETCC) 7044263508Sdim return DAG.getNode(ISD::BR_CC, SDLoc(N), MVT::Other, 7045193323Sed N->getOperand(0), Simp.getOperand(2), 7046193323Sed Simp.getOperand(0), Simp.getOperand(1), 7047193323Sed N->getOperand(4)); 7048193323Sed 7049193323Sed return SDValue(); 7050193323Sed} 7051193323Sed 7052234353Sdim/// canFoldInAddressingMode - Return true if 'Use' is a load or a store that 7053234353Sdim/// uses N as its base pointer and that N may be folded in the load / store 7054234353Sdim/// addressing mode. 7055234353Sdimstatic bool canFoldInAddressingMode(SDNode *N, SDNode *Use, 7056234353Sdim SelectionDAG &DAG, 7057234353Sdim const TargetLowering &TLI) { 7058234353Sdim EVT VT; 7059234353Sdim if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Use)) { 7060234353Sdim if (LD->isIndexed() || LD->getBasePtr().getNode() != N) 7061234353Sdim return false; 7062234353Sdim VT = Use->getValueType(0); 7063234353Sdim } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Use)) { 7064234353Sdim if (ST->isIndexed() || ST->getBasePtr().getNode() != N) 7065234353Sdim return false; 7066234353Sdim VT = ST->getValue().getValueType(); 7067234353Sdim } else 7068234353Sdim return false; 7069234353Sdim 7070249423Sdim TargetLowering::AddrMode AM; 7071234353Sdim if (N->getOpcode() == ISD::ADD) { 7072234353Sdim ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1)); 7073234353Sdim if (Offset) 7074234353Sdim // [reg +/- imm] 7075234353Sdim AM.BaseOffs = Offset->getSExtValue(); 7076234353Sdim else 7077234353Sdim // [reg +/- reg] 7078234353Sdim AM.Scale = 1; 7079234353Sdim } else if (N->getOpcode() == ISD::SUB) { 7080234353Sdim ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1)); 7081234353Sdim if (Offset) 7082234353Sdim // [reg +/- imm] 7083234353Sdim AM.BaseOffs = -Offset->getSExtValue(); 7084234353Sdim else 7085234353Sdim // [reg +/- reg] 7086234353Sdim AM.Scale = 1; 7087234353Sdim } else 7088234353Sdim return false; 7089234353Sdim 7090234353Sdim return TLI.isLegalAddressingMode(AM, VT.getTypeForEVT(*DAG.getContext())); 7091234353Sdim} 7092234353Sdim 7093193323Sed/// CombineToPreIndexedLoadStore - Try turning a load / store into a 7094193323Sed/// pre-indexed load / store when the base pointer is an add or subtract 7095193323Sed/// and it has other uses besides the load / store. After the 7096193323Sed/// transformation, the new indexed load / store has effectively folded 7097193323Sed/// the add / subtract in and all of its other uses are redirected to the 7098193323Sed/// new load / store. 7099193323Sedbool DAGCombiner::CombineToPreIndexedLoadStore(SDNode *N) { 7100234353Sdim if (Level < AfterLegalizeDAG) 7101193323Sed return false; 7102193323Sed 7103193323Sed bool isLoad = true; 7104193323Sed SDValue Ptr; 7105198090Srdivacky EVT VT; 7106193323Sed if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) { 7107193323Sed if (LD->isIndexed()) 7108193323Sed return false; 7109193323Sed VT = LD->getMemoryVT(); 7110193323Sed if (!TLI.isIndexedLoadLegal(ISD::PRE_INC, VT) && 7111193323Sed !TLI.isIndexedLoadLegal(ISD::PRE_DEC, VT)) 7112193323Sed return false; 7113193323Sed Ptr = LD->getBasePtr(); 7114193323Sed } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) { 7115193323Sed if (ST->isIndexed()) 7116193323Sed return false; 7117193323Sed VT = ST->getMemoryVT(); 7118193323Sed if (!TLI.isIndexedStoreLegal(ISD::PRE_INC, VT) && 7119193323Sed !TLI.isIndexedStoreLegal(ISD::PRE_DEC, VT)) 7120193323Sed return false; 7121193323Sed Ptr = ST->getBasePtr(); 7122193323Sed isLoad = false; 7123193323Sed } else { 7124193323Sed return false; 7125193323Sed } 7126193323Sed 7127193323Sed // If the pointer is not an add/sub, or if it doesn't have multiple uses, bail 7128193323Sed // out. There is no reason to make this a preinc/predec. 7129193323Sed if ((Ptr.getOpcode() != ISD::ADD && Ptr.getOpcode() != ISD::SUB) || 7130193323Sed Ptr.getNode()->hasOneUse()) 7131193323Sed return false; 7132193323Sed 7133193323Sed // Ask the target to do addressing mode selection. 7134193323Sed SDValue BasePtr; 7135193323Sed SDValue Offset; 7136193323Sed ISD::MemIndexedMode AM = ISD::UNINDEXED; 7137193323Sed if (!TLI.getPreIndexedAddressParts(N, BasePtr, Offset, AM, DAG)) 7138193323Sed return false; 7139249423Sdim 7140249423Sdim // Backends without true r+i pre-indexed forms may need to pass a 7141249423Sdim // constant base with a variable offset so that constant coercion 7142249423Sdim // will work with the patterns in canonical form. 7143249423Sdim bool Swapped = false; 7144249423Sdim if (isa<ConstantSDNode>(BasePtr)) { 7145249423Sdim std::swap(BasePtr, Offset); 7146249423Sdim Swapped = true; 7147249423Sdim } 7148249423Sdim 7149193323Sed // Don't create a indexed load / store with zero offset. 7150193323Sed if (isa<ConstantSDNode>(Offset) && 7151193323Sed cast<ConstantSDNode>(Offset)->isNullValue()) 7152193323Sed return false; 7153193323Sed 7154193323Sed // Try turning it into a pre-indexed load / store except when: 7155193323Sed // 1) The new base ptr is a frame index. 7156193323Sed // 2) If N is a store and the new base ptr is either the same as or is a 7157193323Sed // predecessor of the value being stored. 7158193323Sed // 3) Another use of old base ptr is a predecessor of N. If ptr is folded 7159193323Sed // that would create a cycle. 7160193323Sed // 4) All uses are load / store ops that use it as old base ptr. 7161193323Sed 7162193323Sed // Check #1. Preinc'ing a frame index would require copying the stack pointer 7163193323Sed // (plus the implicit offset) to a register to preinc anyway. 7164193323Sed if (isa<FrameIndexSDNode>(BasePtr) || isa<RegisterSDNode>(BasePtr)) 7165193323Sed return false; 7166193323Sed 7167193323Sed // Check #2. 7168193323Sed if (!isLoad) { 7169193323Sed SDValue Val = cast<StoreSDNode>(N)->getValue(); 7170193323Sed if (Val == BasePtr || BasePtr.getNode()->isPredecessorOf(Val.getNode())) 7171193323Sed return false; 7172193323Sed } 7173193323Sed 7174249423Sdim // If the offset is a constant, there may be other adds of constants that 7175249423Sdim // can be folded with this one. We should do this to avoid having to keep 7176249423Sdim // a copy of the original base pointer. 7177249423Sdim SmallVector<SDNode *, 16> OtherUses; 7178249423Sdim if (isa<ConstantSDNode>(Offset)) 7179249423Sdim for (SDNode::use_iterator I = BasePtr.getNode()->use_begin(), 7180249423Sdim E = BasePtr.getNode()->use_end(); I != E; ++I) { 7181249423Sdim SDNode *Use = *I; 7182249423Sdim if (Use == Ptr.getNode()) 7183249423Sdim continue; 7184249423Sdim 7185249423Sdim if (Use->isPredecessorOf(N)) 7186249423Sdim continue; 7187249423Sdim 7188249423Sdim if (Use->getOpcode() != ISD::ADD && Use->getOpcode() != ISD::SUB) { 7189249423Sdim OtherUses.clear(); 7190249423Sdim break; 7191249423Sdim } 7192249423Sdim 7193249423Sdim SDValue Op0 = Use->getOperand(0), Op1 = Use->getOperand(1); 7194249423Sdim if (Op1.getNode() == BasePtr.getNode()) 7195249423Sdim std::swap(Op0, Op1); 7196249423Sdim assert(Op0.getNode() == BasePtr.getNode() && 7197249423Sdim "Use of ADD/SUB but not an operand"); 7198249423Sdim 7199249423Sdim if (!isa<ConstantSDNode>(Op1)) { 7200249423Sdim OtherUses.clear(); 7201249423Sdim break; 7202249423Sdim } 7203249423Sdim 7204249423Sdim // FIXME: In some cases, we can be smarter about this. 7205249423Sdim if (Op1.getValueType() != Offset.getValueType()) { 7206249423Sdim OtherUses.clear(); 7207249423Sdim break; 7208249423Sdim } 7209249423Sdim 7210249423Sdim OtherUses.push_back(Use); 7211249423Sdim } 7212249423Sdim 7213249423Sdim if (Swapped) 7214249423Sdim std::swap(BasePtr, Offset); 7215249423Sdim 7216193323Sed // Now check for #3 and #4. 7217193323Sed bool RealUse = false; 7218224145Sdim 7219224145Sdim // Caches for hasPredecessorHelper 7220224145Sdim SmallPtrSet<const SDNode *, 32> Visited; 7221224145Sdim SmallVector<const SDNode *, 16> Worklist; 7222224145Sdim 7223193323Sed for (SDNode::use_iterator I = Ptr.getNode()->use_begin(), 7224193323Sed E = Ptr.getNode()->use_end(); I != E; ++I) { 7225193323Sed SDNode *Use = *I; 7226193323Sed if (Use == N) 7227193323Sed continue; 7228224145Sdim if (N->hasPredecessorHelper(Use, Visited, Worklist)) 7229193323Sed return false; 7230193323Sed 7231234353Sdim // If Ptr may be folded in addressing mode of other use, then it's 7232234353Sdim // not profitable to do this transformation. 7233234353Sdim if (!canFoldInAddressingMode(Ptr.getNode(), Use, DAG, TLI)) 7234193323Sed RealUse = true; 7235193323Sed } 7236193323Sed 7237193323Sed if (!RealUse) 7238193323Sed return false; 7239193323Sed 7240193323Sed SDValue Result; 7241193323Sed if (isLoad) 7242263508Sdim Result = DAG.getIndexedLoad(SDValue(N,0), SDLoc(N), 7243193323Sed BasePtr, Offset, AM); 7244193323Sed else 7245263508Sdim Result = DAG.getIndexedStore(SDValue(N,0), SDLoc(N), 7246193323Sed BasePtr, Offset, AM); 7247193323Sed ++PreIndexedNodes; 7248193323Sed ++NodesCombined; 7249202375Srdivacky DEBUG(dbgs() << "\nReplacing.4 "; 7250198090Srdivacky N->dump(&DAG); 7251202375Srdivacky dbgs() << "\nWith: "; 7252198090Srdivacky Result.getNode()->dump(&DAG); 7253202375Srdivacky dbgs() << '\n'); 7254193323Sed WorkListRemover DeadNodes(*this); 7255193323Sed if (isLoad) { 7256239462Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(0)); 7257239462Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Result.getValue(2)); 7258193323Sed } else { 7259239462Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(1)); 7260193323Sed } 7261193323Sed 7262193323Sed // Finally, since the node is now dead, remove it from the graph. 7263193323Sed DAG.DeleteNode(N); 7264193323Sed 7265249423Sdim if (Swapped) 7266249423Sdim std::swap(BasePtr, Offset); 7267249423Sdim 7268249423Sdim // Replace other uses of BasePtr that can be updated to use Ptr 7269249423Sdim for (unsigned i = 0, e = OtherUses.size(); i != e; ++i) { 7270249423Sdim unsigned OffsetIdx = 1; 7271249423Sdim if (OtherUses[i]->getOperand(OffsetIdx).getNode() == BasePtr.getNode()) 7272249423Sdim OffsetIdx = 0; 7273249423Sdim assert(OtherUses[i]->getOperand(!OffsetIdx).getNode() == 7274249423Sdim BasePtr.getNode() && "Expected BasePtr operand"); 7275249423Sdim 7276251662Sdim // We need to replace ptr0 in the following expression: 7277251662Sdim // x0 * offset0 + y0 * ptr0 = t0 7278251662Sdim // knowing that 7279251662Sdim // x1 * offset1 + y1 * ptr0 = t1 (the indexed load/store) 7280263508Sdim // 7281251662Sdim // where x0, x1, y0 and y1 in {-1, 1} are given by the types of the 7282251662Sdim // indexed load/store and the expresion that needs to be re-written. 7283251662Sdim // 7284251662Sdim // Therefore, we have: 7285251662Sdim // t0 = (x0 * offset0 - x1 * y0 * y1 *offset1) + (y0 * y1) * t1 7286249423Sdim 7287249423Sdim ConstantSDNode *CN = 7288249423Sdim cast<ConstantSDNode>(OtherUses[i]->getOperand(OffsetIdx)); 7289251662Sdim int X0, X1, Y0, Y1; 7290251662Sdim APInt Offset0 = CN->getAPIntValue(); 7291251662Sdim APInt Offset1 = cast<ConstantSDNode>(Offset)->getAPIntValue(); 7292249423Sdim 7293251662Sdim X0 = (OtherUses[i]->getOpcode() == ISD::SUB && OffsetIdx == 1) ? -1 : 1; 7294251662Sdim Y0 = (OtherUses[i]->getOpcode() == ISD::SUB && OffsetIdx == 0) ? -1 : 1; 7295251662Sdim X1 = (AM == ISD::PRE_DEC && !Swapped) ? -1 : 1; 7296251662Sdim Y1 = (AM == ISD::PRE_DEC && Swapped) ? -1 : 1; 7297249423Sdim 7298251662Sdim unsigned Opcode = (Y0 * Y1 < 0) ? ISD::SUB : ISD::ADD; 7299251662Sdim 7300251662Sdim APInt CNV = Offset0; 7301251662Sdim if (X0 < 0) CNV = -CNV; 7302251662Sdim if (X1 * Y0 * Y1 < 0) CNV = CNV + Offset1; 7303251662Sdim else CNV = CNV - Offset1; 7304251662Sdim 7305251662Sdim // We can now generate the new expression. 7306251662Sdim SDValue NewOp1 = DAG.getConstant(CNV, CN->getValueType(0)); 7307251662Sdim SDValue NewOp2 = Result.getValue(isLoad ? 1 : 0); 7308251662Sdim 7309251662Sdim SDValue NewUse = DAG.getNode(Opcode, 7310263508Sdim SDLoc(OtherUses[i]), 7311249423Sdim OtherUses[i]->getValueType(0), NewOp1, NewOp2); 7312249423Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(OtherUses[i], 0), NewUse); 7313249423Sdim removeFromWorkList(OtherUses[i]); 7314249423Sdim DAG.DeleteNode(OtherUses[i]); 7315249423Sdim } 7316249423Sdim 7317193323Sed // Replace the uses of Ptr with uses of the updated base value. 7318239462Sdim DAG.ReplaceAllUsesOfValueWith(Ptr, Result.getValue(isLoad ? 1 : 0)); 7319193323Sed removeFromWorkList(Ptr.getNode()); 7320193323Sed DAG.DeleteNode(Ptr.getNode()); 7321193323Sed 7322193323Sed return true; 7323193323Sed} 7324193323Sed 7325193323Sed/// CombineToPostIndexedLoadStore - Try to combine a load / store with a 7326193323Sed/// add / sub of the base pointer node into a post-indexed load / store. 7327193323Sed/// The transformation folded the add / subtract into the new indexed 7328193323Sed/// load / store effectively and all of its uses are redirected to the 7329193323Sed/// new load / store. 7330193323Sedbool DAGCombiner::CombineToPostIndexedLoadStore(SDNode *N) { 7331234353Sdim if (Level < AfterLegalizeDAG) 7332193323Sed return false; 7333193323Sed 7334193323Sed bool isLoad = true; 7335193323Sed SDValue Ptr; 7336198090Srdivacky EVT VT; 7337193323Sed if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) { 7338193323Sed if (LD->isIndexed()) 7339193323Sed return false; 7340193323Sed VT = LD->getMemoryVT(); 7341193323Sed if (!TLI.isIndexedLoadLegal(ISD::POST_INC, VT) && 7342193323Sed !TLI.isIndexedLoadLegal(ISD::POST_DEC, VT)) 7343193323Sed return false; 7344193323Sed Ptr = LD->getBasePtr(); 7345193323Sed } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) { 7346193323Sed if (ST->isIndexed()) 7347193323Sed return false; 7348193323Sed VT = ST->getMemoryVT(); 7349193323Sed if (!TLI.isIndexedStoreLegal(ISD::POST_INC, VT) && 7350193323Sed !TLI.isIndexedStoreLegal(ISD::POST_DEC, VT)) 7351193323Sed return false; 7352193323Sed Ptr = ST->getBasePtr(); 7353193323Sed isLoad = false; 7354193323Sed } else { 7355193323Sed return false; 7356193323Sed } 7357193323Sed 7358193323Sed if (Ptr.getNode()->hasOneUse()) 7359193323Sed return false; 7360193323Sed 7361193323Sed for (SDNode::use_iterator I = Ptr.getNode()->use_begin(), 7362193323Sed E = Ptr.getNode()->use_end(); I != E; ++I) { 7363193323Sed SDNode *Op = *I; 7364193323Sed if (Op == N || 7365193323Sed (Op->getOpcode() != ISD::ADD && Op->getOpcode() != ISD::SUB)) 7366193323Sed continue; 7367193323Sed 7368193323Sed SDValue BasePtr; 7369193323Sed SDValue Offset; 7370193323Sed ISD::MemIndexedMode AM = ISD::UNINDEXED; 7371193323Sed if (TLI.getPostIndexedAddressParts(N, Op, BasePtr, Offset, AM, DAG)) { 7372193323Sed // Don't create a indexed load / store with zero offset. 7373193323Sed if (isa<ConstantSDNode>(Offset) && 7374193323Sed cast<ConstantSDNode>(Offset)->isNullValue()) 7375193323Sed continue; 7376193323Sed 7377193323Sed // Try turning it into a post-indexed load / store except when 7378234353Sdim // 1) All uses are load / store ops that use it as base ptr (and 7379234353Sdim // it may be folded as addressing mmode). 7380193323Sed // 2) Op must be independent of N, i.e. Op is neither a predecessor 7381193323Sed // nor a successor of N. Otherwise, if Op is folded that would 7382193323Sed // create a cycle. 7383193323Sed 7384193323Sed if (isa<FrameIndexSDNode>(BasePtr) || isa<RegisterSDNode>(BasePtr)) 7385193323Sed continue; 7386193323Sed 7387193323Sed // Check for #1. 7388193323Sed bool TryNext = false; 7389193323Sed for (SDNode::use_iterator II = BasePtr.getNode()->use_begin(), 7390193323Sed EE = BasePtr.getNode()->use_end(); II != EE; ++II) { 7391193323Sed SDNode *Use = *II; 7392193323Sed if (Use == Ptr.getNode()) 7393193323Sed continue; 7394193323Sed 7395193323Sed // If all the uses are load / store addresses, then don't do the 7396193323Sed // transformation. 7397193323Sed if (Use->getOpcode() == ISD::ADD || Use->getOpcode() == ISD::SUB){ 7398193323Sed bool RealUse = false; 7399193323Sed for (SDNode::use_iterator III = Use->use_begin(), 7400193323Sed EEE = Use->use_end(); III != EEE; ++III) { 7401193323Sed SDNode *UseUse = *III; 7402263508Sdim if (!canFoldInAddressingMode(Use, UseUse, DAG, TLI)) 7403193323Sed RealUse = true; 7404193323Sed } 7405193323Sed 7406193323Sed if (!RealUse) { 7407193323Sed TryNext = true; 7408193323Sed break; 7409193323Sed } 7410193323Sed } 7411193323Sed } 7412193323Sed 7413193323Sed if (TryNext) 7414193323Sed continue; 7415193323Sed 7416193323Sed // Check for #2 7417193323Sed if (!Op->isPredecessorOf(N) && !N->isPredecessorOf(Op)) { 7418193323Sed SDValue Result = isLoad 7419263508Sdim ? DAG.getIndexedLoad(SDValue(N,0), SDLoc(N), 7420193323Sed BasePtr, Offset, AM) 7421263508Sdim : DAG.getIndexedStore(SDValue(N,0), SDLoc(N), 7422193323Sed BasePtr, Offset, AM); 7423193323Sed ++PostIndexedNodes; 7424193323Sed ++NodesCombined; 7425202375Srdivacky DEBUG(dbgs() << "\nReplacing.5 "; 7426198090Srdivacky N->dump(&DAG); 7427202375Srdivacky dbgs() << "\nWith: "; 7428198090Srdivacky Result.getNode()->dump(&DAG); 7429202375Srdivacky dbgs() << '\n'); 7430193323Sed WorkListRemover DeadNodes(*this); 7431193323Sed if (isLoad) { 7432239462Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(0)); 7433239462Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Result.getValue(2)); 7434193323Sed } else { 7435239462Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(1)); 7436193323Sed } 7437193323Sed 7438193323Sed // Finally, since the node is now dead, remove it from the graph. 7439193323Sed DAG.DeleteNode(N); 7440193323Sed 7441193323Sed // Replace the uses of Use with uses of the updated base value. 7442193323Sed DAG.ReplaceAllUsesOfValueWith(SDValue(Op, 0), 7443239462Sdim Result.getValue(isLoad ? 1 : 0)); 7444193323Sed removeFromWorkList(Op); 7445193323Sed DAG.DeleteNode(Op); 7446193323Sed return true; 7447193323Sed } 7448193323Sed } 7449193323Sed } 7450193323Sed 7451193323Sed return false; 7452193323Sed} 7453193323Sed 7454193323SedSDValue DAGCombiner::visitLOAD(SDNode *N) { 7455193323Sed LoadSDNode *LD = cast<LoadSDNode>(N); 7456193323Sed SDValue Chain = LD->getChain(); 7457193323Sed SDValue Ptr = LD->getBasePtr(); 7458193323Sed 7459193323Sed // If load is not volatile and there are no uses of the loaded value (and 7460193323Sed // the updated indexed value in case of indexed loads), change uses of the 7461193323Sed // chain value into uses of the chain input (i.e. delete the dead load). 7462193323Sed if (!LD->isVolatile()) { 7463193323Sed if (N->getValueType(1) == MVT::Other) { 7464193323Sed // Unindexed loads. 7465234353Sdim if (!N->hasAnyUseOfValue(0)) { 7466193323Sed // It's not safe to use the two value CombineTo variant here. e.g. 7467193323Sed // v1, chain2 = load chain1, loc 7468193323Sed // v2, chain3 = load chain2, loc 7469193323Sed // v3 = add v2, c 7470193323Sed // Now we replace use of chain2 with chain1. This makes the second load 7471193323Sed // isomorphic to the one we are deleting, and thus makes this load live. 7472202375Srdivacky DEBUG(dbgs() << "\nReplacing.6 "; 7473198090Srdivacky N->dump(&DAG); 7474202375Srdivacky dbgs() << "\nWith chain: "; 7475198090Srdivacky Chain.getNode()->dump(&DAG); 7476202375Srdivacky dbgs() << "\n"); 7477193323Sed WorkListRemover DeadNodes(*this); 7478239462Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Chain); 7479193323Sed 7480193323Sed if (N->use_empty()) { 7481193323Sed removeFromWorkList(N); 7482193323Sed DAG.DeleteNode(N); 7483193323Sed } 7484193323Sed 7485193323Sed return SDValue(N, 0); // Return N so it doesn't get rechecked! 7486193323Sed } 7487193323Sed } else { 7488193323Sed // Indexed loads. 7489193323Sed assert(N->getValueType(2) == MVT::Other && "Malformed indexed loads?"); 7490234353Sdim if (!N->hasAnyUseOfValue(0) && !N->hasAnyUseOfValue(1)) { 7491193323Sed SDValue Undef = DAG.getUNDEF(N->getValueType(0)); 7492204642Srdivacky DEBUG(dbgs() << "\nReplacing.7 "; 7493198090Srdivacky N->dump(&DAG); 7494202375Srdivacky dbgs() << "\nWith: "; 7495198090Srdivacky Undef.getNode()->dump(&DAG); 7496202375Srdivacky dbgs() << " and 2 other values\n"); 7497193323Sed WorkListRemover DeadNodes(*this); 7498239462Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Undef); 7499193323Sed DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), 7500239462Sdim DAG.getUNDEF(N->getValueType(1))); 7501239462Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(N, 2), Chain); 7502193323Sed removeFromWorkList(N); 7503193323Sed DAG.DeleteNode(N); 7504193323Sed return SDValue(N, 0); // Return N so it doesn't get rechecked! 7505193323Sed } 7506193323Sed } 7507193323Sed } 7508193323Sed 7509193323Sed // If this load is directly stored, replace the load value with the stored 7510193323Sed // value. 7511193323Sed // TODO: Handle store large -> read small portion. 7512193323Sed // TODO: Handle TRUNCSTORE/LOADEXT 7513221345Sdim if (ISD::isNormalLoad(N) && !LD->isVolatile()) { 7514193323Sed if (ISD::isNON_TRUNCStore(Chain.getNode())) { 7515193323Sed StoreSDNode *PrevST = cast<StoreSDNode>(Chain); 7516193323Sed if (PrevST->getBasePtr() == Ptr && 7517193323Sed PrevST->getValue().getValueType() == N->getValueType(0)) 7518193323Sed return CombineTo(N, Chain.getOperand(1), Chain); 7519193323Sed } 7520193323Sed } 7521193323Sed 7522206083Srdivacky // Try to infer better alignment information than the load already has. 7523206083Srdivacky if (OptLevel != CodeGenOpt::None && LD->isUnindexed()) { 7524206083Srdivacky if (unsigned Align = DAG.InferPtrAlignment(Ptr)) { 7525249423Sdim if (Align > LD->getMemOperand()->getBaseAlignment()) { 7526249423Sdim SDValue NewLoad = 7527263508Sdim DAG.getExtLoad(LD->getExtensionType(), SDLoc(N), 7528218893Sdim LD->getValueType(0), 7529218893Sdim Chain, Ptr, LD->getPointerInfo(), 7530218893Sdim LD->getMemoryVT(), 7531263508Sdim LD->isVolatile(), LD->isNonTemporal(), Align, 7532263508Sdim LD->getTBAAInfo()); 7533249423Sdim return CombineTo(N, NewLoad, SDValue(NewLoad.getNode(), 1), true); 7534249423Sdim } 7535206083Srdivacky } 7536206083Srdivacky } 7537206083Srdivacky 7538263508Sdim bool UseAA = CombinerAA.getNumOccurrences() > 0 ? CombinerAA : 7539263508Sdim TLI.getTargetMachine().getSubtarget<TargetSubtargetInfo>().useAA(); 7540263508Sdim if (UseAA) { 7541193323Sed // Walk up chain skipping non-aliasing memory nodes. 7542193323Sed SDValue BetterChain = FindBetterChain(N, Chain); 7543193323Sed 7544193323Sed // If there is a better chain. 7545193323Sed if (Chain != BetterChain) { 7546193323Sed SDValue ReplLoad; 7547193323Sed 7548193323Sed // Replace the chain to void dependency. 7549193323Sed if (LD->getExtensionType() == ISD::NON_EXTLOAD) { 7550263508Sdim ReplLoad = DAG.getLoad(N->getValueType(0), SDLoc(LD), 7551263508Sdim BetterChain, Ptr, LD->getMemOperand()); 7552193323Sed } else { 7553263508Sdim ReplLoad = DAG.getExtLoad(LD->getExtensionType(), SDLoc(LD), 7554218893Sdim LD->getValueType(0), 7555263508Sdim BetterChain, Ptr, LD->getMemoryVT(), 7556263508Sdim LD->getMemOperand()); 7557193323Sed } 7558193323Sed 7559193323Sed // Create token factor to keep old chain connected. 7560263508Sdim SDValue Token = DAG.getNode(ISD::TokenFactor, SDLoc(N), 7561193323Sed MVT::Other, Chain, ReplLoad.getValue(1)); 7562218893Sdim 7563198090Srdivacky // Make sure the new and old chains are cleaned up. 7564198090Srdivacky AddToWorkList(Token.getNode()); 7565218893Sdim 7566193323Sed // Replace uses with load result and token factor. Don't add users 7567193323Sed // to work list. 7568193323Sed return CombineTo(N, ReplLoad.getValue(0), Token, false); 7569193323Sed } 7570193323Sed } 7571193323Sed 7572193323Sed // Try transforming N to an indexed load. 7573193323Sed if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N)) 7574193323Sed return SDValue(N, 0); 7575193323Sed 7576263508Sdim // Try to slice up N to more direct loads if the slices are mapped to 7577263508Sdim // different register banks or pairing can take place. 7578263508Sdim if (SliceUpLoad(N)) 7579263508Sdim return SDValue(N, 0); 7580263508Sdim 7581193323Sed return SDValue(); 7582193323Sed} 7583193323Sed 7584263508Sdimnamespace { 7585263508Sdim/// \brief Helper structure used to slice a load in smaller loads. 7586263508Sdim/// Basically a slice is obtained from the following sequence: 7587263508Sdim/// Origin = load Ty1, Base 7588263508Sdim/// Shift = srl Ty1 Origin, CstTy Amount 7589263508Sdim/// Inst = trunc Shift to Ty2 7590263508Sdim/// 7591263508Sdim/// Then, it will be rewriten into: 7592263508Sdim/// Slice = load SliceTy, Base + SliceOffset 7593263508Sdim/// [Inst = zext Slice to Ty2], only if SliceTy <> Ty2 7594263508Sdim/// 7595263508Sdim/// SliceTy is deduced from the number of bits that are actually used to 7596263508Sdim/// build Inst. 7597263508Sdimstruct LoadedSlice { 7598263508Sdim /// \brief Helper structure used to compute the cost of a slice. 7599263508Sdim struct Cost { 7600263508Sdim /// Are we optimizing for code size. 7601263508Sdim bool ForCodeSize; 7602263508Sdim /// Various cost. 7603263508Sdim unsigned Loads; 7604263508Sdim unsigned Truncates; 7605263508Sdim unsigned CrossRegisterBanksCopies; 7606263508Sdim unsigned ZExts; 7607263508Sdim unsigned Shift; 7608263508Sdim 7609263508Sdim Cost(bool ForCodeSize = false) 7610263508Sdim : ForCodeSize(ForCodeSize), Loads(0), Truncates(0), 7611263508Sdim CrossRegisterBanksCopies(0), ZExts(0), Shift(0) {} 7612263508Sdim 7613263508Sdim /// \brief Get the cost of one isolated slice. 7614263508Sdim Cost(const LoadedSlice &LS, bool ForCodeSize = false) 7615263508Sdim : ForCodeSize(ForCodeSize), Loads(1), Truncates(0), 7616263508Sdim CrossRegisterBanksCopies(0), ZExts(0), Shift(0) { 7617263508Sdim EVT TruncType = LS.Inst->getValueType(0); 7618263508Sdim EVT LoadedType = LS.getLoadedType(); 7619263508Sdim if (TruncType != LoadedType && 7620263508Sdim !LS.DAG->getTargetLoweringInfo().isZExtFree(LoadedType, TruncType)) 7621263508Sdim ZExts = 1; 7622263508Sdim } 7623263508Sdim 7624263508Sdim /// \brief Account for slicing gain in the current cost. 7625263508Sdim /// Slicing provide a few gains like removing a shift or a 7626263508Sdim /// truncate. This method allows to grow the cost of the original 7627263508Sdim /// load with the gain from this slice. 7628263508Sdim void addSliceGain(const LoadedSlice &LS) { 7629263508Sdim // Each slice saves a truncate. 7630263508Sdim const TargetLowering &TLI = LS.DAG->getTargetLoweringInfo(); 7631263508Sdim if (!TLI.isTruncateFree(LS.Inst->getValueType(0), 7632263508Sdim LS.Inst->getOperand(0).getValueType())) 7633263508Sdim ++Truncates; 7634263508Sdim // If there is a shift amount, this slice gets rid of it. 7635263508Sdim if (LS.Shift) 7636263508Sdim ++Shift; 7637263508Sdim // If this slice can merge a cross register bank copy, account for it. 7638263508Sdim if (LS.canMergeExpensiveCrossRegisterBankCopy()) 7639263508Sdim ++CrossRegisterBanksCopies; 7640263508Sdim } 7641263508Sdim 7642263508Sdim Cost &operator+=(const Cost &RHS) { 7643263508Sdim Loads += RHS.Loads; 7644263508Sdim Truncates += RHS.Truncates; 7645263508Sdim CrossRegisterBanksCopies += RHS.CrossRegisterBanksCopies; 7646263508Sdim ZExts += RHS.ZExts; 7647263508Sdim Shift += RHS.Shift; 7648263508Sdim return *this; 7649263508Sdim } 7650263508Sdim 7651263508Sdim bool operator==(const Cost &RHS) const { 7652263508Sdim return Loads == RHS.Loads && Truncates == RHS.Truncates && 7653263508Sdim CrossRegisterBanksCopies == RHS.CrossRegisterBanksCopies && 7654263508Sdim ZExts == RHS.ZExts && Shift == RHS.Shift; 7655263508Sdim } 7656263508Sdim 7657263508Sdim bool operator!=(const Cost &RHS) const { return !(*this == RHS); } 7658263508Sdim 7659263508Sdim bool operator<(const Cost &RHS) const { 7660263508Sdim // Assume cross register banks copies are as expensive as loads. 7661263508Sdim // FIXME: Do we want some more target hooks? 7662263508Sdim unsigned ExpensiveOpsLHS = Loads + CrossRegisterBanksCopies; 7663263508Sdim unsigned ExpensiveOpsRHS = RHS.Loads + RHS.CrossRegisterBanksCopies; 7664263508Sdim // Unless we are optimizing for code size, consider the 7665263508Sdim // expensive operation first. 7666263508Sdim if (!ForCodeSize && ExpensiveOpsLHS != ExpensiveOpsRHS) 7667263508Sdim return ExpensiveOpsLHS < ExpensiveOpsRHS; 7668263508Sdim return (Truncates + ZExts + Shift + ExpensiveOpsLHS) < 7669263508Sdim (RHS.Truncates + RHS.ZExts + RHS.Shift + ExpensiveOpsRHS); 7670263508Sdim } 7671263508Sdim 7672263508Sdim bool operator>(const Cost &RHS) const { return RHS < *this; } 7673263508Sdim 7674263508Sdim bool operator<=(const Cost &RHS) const { return !(RHS < *this); } 7675263508Sdim 7676263508Sdim bool operator>=(const Cost &RHS) const { return !(*this < RHS); } 7677263508Sdim }; 7678263508Sdim // The last instruction that represent the slice. This should be a 7679263508Sdim // truncate instruction. 7680263508Sdim SDNode *Inst; 7681263508Sdim // The original load instruction. 7682263508Sdim LoadSDNode *Origin; 7683263508Sdim // The right shift amount in bits from the original load. 7684263508Sdim unsigned Shift; 7685263508Sdim // The DAG from which Origin came from. 7686263508Sdim // This is used to get some contextual information about legal types, etc. 7687263508Sdim SelectionDAG *DAG; 7688263508Sdim 7689263508Sdim LoadedSlice(SDNode *Inst = NULL, LoadSDNode *Origin = NULL, 7690263508Sdim unsigned Shift = 0, SelectionDAG *DAG = NULL) 7691263508Sdim : Inst(Inst), Origin(Origin), Shift(Shift), DAG(DAG) {} 7692263508Sdim 7693263508Sdim LoadedSlice(const LoadedSlice &LS) 7694263508Sdim : Inst(LS.Inst), Origin(LS.Origin), Shift(LS.Shift), DAG(LS.DAG) {} 7695263508Sdim 7696263508Sdim /// \brief Get the bits used in a chunk of bits \p BitWidth large. 7697263508Sdim /// \return Result is \p BitWidth and has used bits set to 1 and 7698263508Sdim /// not used bits set to 0. 7699263508Sdim APInt getUsedBits() const { 7700263508Sdim // Reproduce the trunc(lshr) sequence: 7701263508Sdim // - Start from the truncated value. 7702263508Sdim // - Zero extend to the desired bit width. 7703263508Sdim // - Shift left. 7704263508Sdim assert(Origin && "No original load to compare against."); 7705263508Sdim unsigned BitWidth = Origin->getValueSizeInBits(0); 7706263508Sdim assert(Inst && "This slice is not bound to an instruction"); 7707263508Sdim assert(Inst->getValueSizeInBits(0) <= BitWidth && 7708263508Sdim "Extracted slice is bigger than the whole type!"); 7709263508Sdim APInt UsedBits(Inst->getValueSizeInBits(0), 0); 7710263508Sdim UsedBits.setAllBits(); 7711263508Sdim UsedBits = UsedBits.zext(BitWidth); 7712263508Sdim UsedBits <<= Shift; 7713263508Sdim return UsedBits; 7714263508Sdim } 7715263508Sdim 7716263508Sdim /// \brief Get the size of the slice to be loaded in bytes. 7717263508Sdim unsigned getLoadedSize() const { 7718263508Sdim unsigned SliceSize = getUsedBits().countPopulation(); 7719263508Sdim assert(!(SliceSize & 0x7) && "Size is not a multiple of a byte."); 7720263508Sdim return SliceSize / 8; 7721263508Sdim } 7722263508Sdim 7723263508Sdim /// \brief Get the type that will be loaded for this slice. 7724263508Sdim /// Note: This may not be the final type for the slice. 7725263508Sdim EVT getLoadedType() const { 7726263508Sdim assert(DAG && "Missing context"); 7727263508Sdim LLVMContext &Ctxt = *DAG->getContext(); 7728263508Sdim return EVT::getIntegerVT(Ctxt, getLoadedSize() * 8); 7729263508Sdim } 7730263508Sdim 7731263508Sdim /// \brief Get the alignment of the load used for this slice. 7732263508Sdim unsigned getAlignment() const { 7733263508Sdim unsigned Alignment = Origin->getAlignment(); 7734263508Sdim unsigned Offset = getOffsetFromBase(); 7735263508Sdim if (Offset != 0) 7736263508Sdim Alignment = MinAlign(Alignment, Alignment + Offset); 7737263508Sdim return Alignment; 7738263508Sdim } 7739263508Sdim 7740263508Sdim /// \brief Check if this slice can be rewritten with legal operations. 7741263508Sdim bool isLegal() const { 7742263508Sdim // An invalid slice is not legal. 7743263508Sdim if (!Origin || !Inst || !DAG) 7744263508Sdim return false; 7745263508Sdim 7746263508Sdim // Offsets are for indexed load only, we do not handle that. 7747263508Sdim if (Origin->getOffset().getOpcode() != ISD::UNDEF) 7748263508Sdim return false; 7749263508Sdim 7750263508Sdim const TargetLowering &TLI = DAG->getTargetLoweringInfo(); 7751263508Sdim 7752263508Sdim // Check that the type is legal. 7753263508Sdim EVT SliceType = getLoadedType(); 7754263508Sdim if (!TLI.isTypeLegal(SliceType)) 7755263508Sdim return false; 7756263508Sdim 7757263508Sdim // Check that the load is legal for this type. 7758263508Sdim if (!TLI.isOperationLegal(ISD::LOAD, SliceType)) 7759263508Sdim return false; 7760263508Sdim 7761263508Sdim // Check that the offset can be computed. 7762263508Sdim // 1. Check its type. 7763263508Sdim EVT PtrType = Origin->getBasePtr().getValueType(); 7764263508Sdim if (PtrType == MVT::Untyped || PtrType.isExtended()) 7765263508Sdim return false; 7766263508Sdim 7767263508Sdim // 2. Check that it fits in the immediate. 7768263508Sdim if (!TLI.isLegalAddImmediate(getOffsetFromBase())) 7769263508Sdim return false; 7770263508Sdim 7771263508Sdim // 3. Check that the computation is legal. 7772263508Sdim if (!TLI.isOperationLegal(ISD::ADD, PtrType)) 7773263508Sdim return false; 7774263508Sdim 7775263508Sdim // Check that the zext is legal if it needs one. 7776263508Sdim EVT TruncateType = Inst->getValueType(0); 7777263508Sdim if (TruncateType != SliceType && 7778263508Sdim !TLI.isOperationLegal(ISD::ZERO_EXTEND, TruncateType)) 7779263508Sdim return false; 7780263508Sdim 7781263508Sdim return true; 7782263508Sdim } 7783263508Sdim 7784263508Sdim /// \brief Get the offset in bytes of this slice in the original chunk of 7785263508Sdim /// bits. 7786263508Sdim /// \pre DAG != NULL. 7787263508Sdim uint64_t getOffsetFromBase() const { 7788263508Sdim assert(DAG && "Missing context."); 7789263508Sdim bool IsBigEndian = 7790263508Sdim DAG->getTargetLoweringInfo().getDataLayout()->isBigEndian(); 7791263508Sdim assert(!(Shift & 0x7) && "Shifts not aligned on Bytes are not supported."); 7792263508Sdim uint64_t Offset = Shift / 8; 7793263508Sdim unsigned TySizeInBytes = Origin->getValueSizeInBits(0) / 8; 7794263508Sdim assert(!(Origin->getValueSizeInBits(0) & 0x7) && 7795263508Sdim "The size of the original loaded type is not a multiple of a" 7796263508Sdim " byte."); 7797263508Sdim // If Offset is bigger than TySizeInBytes, it means we are loading all 7798263508Sdim // zeros. This should have been optimized before in the process. 7799263508Sdim assert(TySizeInBytes > Offset && 7800263508Sdim "Invalid shift amount for given loaded size"); 7801263508Sdim if (IsBigEndian) 7802263508Sdim Offset = TySizeInBytes - Offset - getLoadedSize(); 7803263508Sdim return Offset; 7804263508Sdim } 7805263508Sdim 7806263508Sdim /// \brief Generate the sequence of instructions to load the slice 7807263508Sdim /// represented by this object and redirect the uses of this slice to 7808263508Sdim /// this new sequence of instructions. 7809263508Sdim /// \pre this->Inst && this->Origin are valid Instructions and this 7810263508Sdim /// object passed the legal check: LoadedSlice::isLegal returned true. 7811263508Sdim /// \return The last instruction of the sequence used to load the slice. 7812263508Sdim SDValue loadSlice() const { 7813263508Sdim assert(Inst && Origin && "Unable to replace a non-existing slice."); 7814263508Sdim const SDValue &OldBaseAddr = Origin->getBasePtr(); 7815263508Sdim SDValue BaseAddr = OldBaseAddr; 7816263508Sdim // Get the offset in that chunk of bytes w.r.t. the endianess. 7817263508Sdim int64_t Offset = static_cast<int64_t>(getOffsetFromBase()); 7818263508Sdim assert(Offset >= 0 && "Offset too big to fit in int64_t!"); 7819263508Sdim if (Offset) { 7820263508Sdim // BaseAddr = BaseAddr + Offset. 7821263508Sdim EVT ArithType = BaseAddr.getValueType(); 7822263508Sdim BaseAddr = DAG->getNode(ISD::ADD, SDLoc(Origin), ArithType, BaseAddr, 7823263508Sdim DAG->getConstant(Offset, ArithType)); 7824263508Sdim } 7825263508Sdim 7826263508Sdim // Create the type of the loaded slice according to its size. 7827263508Sdim EVT SliceType = getLoadedType(); 7828263508Sdim 7829263508Sdim // Create the load for the slice. 7830263508Sdim SDValue LastInst = DAG->getLoad( 7831263508Sdim SliceType, SDLoc(Origin), Origin->getChain(), BaseAddr, 7832263508Sdim Origin->getPointerInfo().getWithOffset(Offset), Origin->isVolatile(), 7833263508Sdim Origin->isNonTemporal(), Origin->isInvariant(), getAlignment()); 7834263508Sdim // If the final type is not the same as the loaded type, this means that 7835263508Sdim // we have to pad with zero. Create a zero extend for that. 7836263508Sdim EVT FinalType = Inst->getValueType(0); 7837263508Sdim if (SliceType != FinalType) 7838263508Sdim LastInst = 7839263508Sdim DAG->getNode(ISD::ZERO_EXTEND, SDLoc(LastInst), FinalType, LastInst); 7840263508Sdim return LastInst; 7841263508Sdim } 7842263508Sdim 7843263508Sdim /// \brief Check if this slice can be merged with an expensive cross register 7844263508Sdim /// bank copy. E.g., 7845263508Sdim /// i = load i32 7846263508Sdim /// f = bitcast i32 i to float 7847263508Sdim bool canMergeExpensiveCrossRegisterBankCopy() const { 7848263508Sdim if (!Inst || !Inst->hasOneUse()) 7849263508Sdim return false; 7850263508Sdim SDNode *Use = *Inst->use_begin(); 7851263508Sdim if (Use->getOpcode() != ISD::BITCAST) 7852263508Sdim return false; 7853263508Sdim assert(DAG && "Missing context"); 7854263508Sdim const TargetLowering &TLI = DAG->getTargetLoweringInfo(); 7855263508Sdim EVT ResVT = Use->getValueType(0); 7856263508Sdim const TargetRegisterClass *ResRC = TLI.getRegClassFor(ResVT.getSimpleVT()); 7857263508Sdim const TargetRegisterClass *ArgRC = 7858263508Sdim TLI.getRegClassFor(Use->getOperand(0).getValueType().getSimpleVT()); 7859263508Sdim if (ArgRC == ResRC || !TLI.isOperationLegal(ISD::LOAD, ResVT)) 7860263508Sdim return false; 7861263508Sdim 7862263508Sdim // At this point, we know that we perform a cross-register-bank copy. 7863263508Sdim // Check if it is expensive. 7864263508Sdim const TargetRegisterInfo *TRI = TLI.getTargetMachine().getRegisterInfo(); 7865263508Sdim // Assume bitcasts are cheap, unless both register classes do not 7866263508Sdim // explicitly share a common sub class. 7867263508Sdim if (!TRI || TRI->getCommonSubClass(ArgRC, ResRC)) 7868263508Sdim return false; 7869263508Sdim 7870263508Sdim // Check if it will be merged with the load. 7871263508Sdim // 1. Check the alignment constraint. 7872263508Sdim unsigned RequiredAlignment = TLI.getDataLayout()->getABITypeAlignment( 7873263508Sdim ResVT.getTypeForEVT(*DAG->getContext())); 7874263508Sdim 7875263508Sdim if (RequiredAlignment > getAlignment()) 7876263508Sdim return false; 7877263508Sdim 7878263508Sdim // 2. Check that the load is a legal operation for that type. 7879263508Sdim if (!TLI.isOperationLegal(ISD::LOAD, ResVT)) 7880263508Sdim return false; 7881263508Sdim 7882263508Sdim // 3. Check that we do not have a zext in the way. 7883263508Sdim if (Inst->getValueType(0) != getLoadedType()) 7884263508Sdim return false; 7885263508Sdim 7886263508Sdim return true; 7887263508Sdim } 7888263508Sdim}; 7889263508Sdim} 7890263508Sdim 7891263508Sdim/// \brief Sorts LoadedSlice according to their offset. 7892263508Sdimstruct LoadedSliceSorter { 7893263508Sdim bool operator()(const LoadedSlice &LHS, const LoadedSlice &RHS) { 7894263508Sdim assert(LHS.Origin == RHS.Origin && "Different bases not implemented."); 7895263508Sdim return LHS.getOffsetFromBase() < RHS.getOffsetFromBase(); 7896263508Sdim } 7897263508Sdim}; 7898263508Sdim 7899263508Sdim/// \brief Check that all bits set in \p UsedBits form a dense region, i.e., 7900263508Sdim/// \p UsedBits looks like 0..0 1..1 0..0. 7901263508Sdimstatic bool areUsedBitsDense(const APInt &UsedBits) { 7902263508Sdim // If all the bits are one, this is dense! 7903263508Sdim if (UsedBits.isAllOnesValue()) 7904263508Sdim return true; 7905263508Sdim 7906263508Sdim // Get rid of the unused bits on the right. 7907263508Sdim APInt NarrowedUsedBits = UsedBits.lshr(UsedBits.countTrailingZeros()); 7908263508Sdim // Get rid of the unused bits on the left. 7909263508Sdim if (NarrowedUsedBits.countLeadingZeros()) 7910263508Sdim NarrowedUsedBits = NarrowedUsedBits.trunc(NarrowedUsedBits.getActiveBits()); 7911263508Sdim // Check that the chunk of bits is completely used. 7912263508Sdim return NarrowedUsedBits.isAllOnesValue(); 7913263508Sdim} 7914263508Sdim 7915263508Sdim/// \brief Check whether or not \p First and \p Second are next to each other 7916263508Sdim/// in memory. This means that there is no hole between the bits loaded 7917263508Sdim/// by \p First and the bits loaded by \p Second. 7918263508Sdimstatic bool areSlicesNextToEachOther(const LoadedSlice &First, 7919263508Sdim const LoadedSlice &Second) { 7920263508Sdim assert(First.Origin == Second.Origin && First.Origin && 7921263508Sdim "Unable to match different memory origins."); 7922263508Sdim APInt UsedBits = First.getUsedBits(); 7923263508Sdim assert((UsedBits & Second.getUsedBits()) == 0 && 7924263508Sdim "Slices are not supposed to overlap."); 7925263508Sdim UsedBits |= Second.getUsedBits(); 7926263508Sdim return areUsedBitsDense(UsedBits); 7927263508Sdim} 7928263508Sdim 7929263508Sdim/// \brief Adjust the \p GlobalLSCost according to the target 7930263508Sdim/// paring capabilities and the layout of the slices. 7931263508Sdim/// \pre \p GlobalLSCost should account for at least as many loads as 7932263508Sdim/// there is in the slices in \p LoadedSlices. 7933263508Sdimstatic void adjustCostForPairing(SmallVectorImpl<LoadedSlice> &LoadedSlices, 7934263508Sdim LoadedSlice::Cost &GlobalLSCost) { 7935263508Sdim unsigned NumberOfSlices = LoadedSlices.size(); 7936263508Sdim // If there is less than 2 elements, no pairing is possible. 7937263508Sdim if (NumberOfSlices < 2) 7938263508Sdim return; 7939263508Sdim 7940263508Sdim // Sort the slices so that elements that are likely to be next to each 7941263508Sdim // other in memory are next to each other in the list. 7942263508Sdim std::sort(LoadedSlices.begin(), LoadedSlices.end(), LoadedSliceSorter()); 7943263508Sdim const TargetLowering &TLI = LoadedSlices[0].DAG->getTargetLoweringInfo(); 7944263508Sdim // First (resp. Second) is the first (resp. Second) potentially candidate 7945263508Sdim // to be placed in a paired load. 7946263508Sdim const LoadedSlice *First = NULL; 7947263508Sdim const LoadedSlice *Second = NULL; 7948263508Sdim for (unsigned CurrSlice = 0; CurrSlice < NumberOfSlices; ++CurrSlice, 7949263508Sdim // Set the beginning of the pair. 7950263508Sdim First = Second) { 7951263508Sdim 7952263508Sdim Second = &LoadedSlices[CurrSlice]; 7953263508Sdim 7954263508Sdim // If First is NULL, it means we start a new pair. 7955263508Sdim // Get to the next slice. 7956263508Sdim if (!First) 7957263508Sdim continue; 7958263508Sdim 7959263508Sdim EVT LoadedType = First->getLoadedType(); 7960263508Sdim 7961263508Sdim // If the types of the slices are different, we cannot pair them. 7962263508Sdim if (LoadedType != Second->getLoadedType()) 7963263508Sdim continue; 7964263508Sdim 7965263508Sdim // Check if the target supplies paired loads for this type. 7966263508Sdim unsigned RequiredAlignment = 0; 7967263508Sdim if (!TLI.hasPairedLoad(LoadedType, RequiredAlignment)) { 7968263508Sdim // move to the next pair, this type is hopeless. 7969263508Sdim Second = NULL; 7970263508Sdim continue; 7971263508Sdim } 7972263508Sdim // Check if we meet the alignment requirement. 7973263508Sdim if (RequiredAlignment > First->getAlignment()) 7974263508Sdim continue; 7975263508Sdim 7976263508Sdim // Check that both loads are next to each other in memory. 7977263508Sdim if (!areSlicesNextToEachOther(*First, *Second)) 7978263508Sdim continue; 7979263508Sdim 7980263508Sdim assert(GlobalLSCost.Loads > 0 && "We save more loads than we created!"); 7981263508Sdim --GlobalLSCost.Loads; 7982263508Sdim // Move to the next pair. 7983263508Sdim Second = NULL; 7984263508Sdim } 7985263508Sdim} 7986263508Sdim 7987263508Sdim/// \brief Check the profitability of all involved LoadedSlice. 7988263508Sdim/// Currently, it is considered profitable if there is exactly two 7989263508Sdim/// involved slices (1) which are (2) next to each other in memory, and 7990263508Sdim/// whose cost (\see LoadedSlice::Cost) is smaller than the original load (3). 7991263508Sdim/// 7992263508Sdim/// Note: The order of the elements in \p LoadedSlices may be modified, but not 7993263508Sdim/// the elements themselves. 7994263508Sdim/// 7995263508Sdim/// FIXME: When the cost model will be mature enough, we can relax 7996263508Sdim/// constraints (1) and (2). 7997263508Sdimstatic bool isSlicingProfitable(SmallVectorImpl<LoadedSlice> &LoadedSlices, 7998263508Sdim const APInt &UsedBits, bool ForCodeSize) { 7999263508Sdim unsigned NumberOfSlices = LoadedSlices.size(); 8000263508Sdim if (StressLoadSlicing) 8001263508Sdim return NumberOfSlices > 1; 8002263508Sdim 8003263508Sdim // Check (1). 8004263508Sdim if (NumberOfSlices != 2) 8005263508Sdim return false; 8006263508Sdim 8007263508Sdim // Check (2). 8008263508Sdim if (!areUsedBitsDense(UsedBits)) 8009263508Sdim return false; 8010263508Sdim 8011263508Sdim // Check (3). 8012263508Sdim LoadedSlice::Cost OrigCost(ForCodeSize), GlobalSlicingCost(ForCodeSize); 8013263508Sdim // The original code has one big load. 8014263508Sdim OrigCost.Loads = 1; 8015263508Sdim for (unsigned CurrSlice = 0; CurrSlice < NumberOfSlices; ++CurrSlice) { 8016263508Sdim const LoadedSlice &LS = LoadedSlices[CurrSlice]; 8017263508Sdim // Accumulate the cost of all the slices. 8018263508Sdim LoadedSlice::Cost SliceCost(LS, ForCodeSize); 8019263508Sdim GlobalSlicingCost += SliceCost; 8020263508Sdim 8021263508Sdim // Account as cost in the original configuration the gain obtained 8022263508Sdim // with the current slices. 8023263508Sdim OrigCost.addSliceGain(LS); 8024263508Sdim } 8025263508Sdim 8026263508Sdim // If the target supports paired load, adjust the cost accordingly. 8027263508Sdim adjustCostForPairing(LoadedSlices, GlobalSlicingCost); 8028263508Sdim return OrigCost > GlobalSlicingCost; 8029263508Sdim} 8030263508Sdim 8031263508Sdim/// \brief If the given load, \p LI, is used only by trunc or trunc(lshr) 8032263508Sdim/// operations, split it in the various pieces being extracted. 8033263508Sdim/// 8034263508Sdim/// This sort of thing is introduced by SROA. 8035263508Sdim/// This slicing takes care not to insert overlapping loads. 8036263508Sdim/// \pre LI is a simple load (i.e., not an atomic or volatile load). 8037263508Sdimbool DAGCombiner::SliceUpLoad(SDNode *N) { 8038263508Sdim if (Level < AfterLegalizeDAG) 8039263508Sdim return false; 8040263508Sdim 8041263508Sdim LoadSDNode *LD = cast<LoadSDNode>(N); 8042263508Sdim if (LD->isVolatile() || !ISD::isNormalLoad(LD) || 8043263508Sdim !LD->getValueType(0).isInteger()) 8044263508Sdim return false; 8045263508Sdim 8046263508Sdim // Keep track of already used bits to detect overlapping values. 8047263508Sdim // In that case, we will just abort the transformation. 8048263508Sdim APInt UsedBits(LD->getValueSizeInBits(0), 0); 8049263508Sdim 8050263508Sdim SmallVector<LoadedSlice, 4> LoadedSlices; 8051263508Sdim 8052263508Sdim // Check if this load is used as several smaller chunks of bits. 8053263508Sdim // Basically, look for uses in trunc or trunc(lshr) and record a new chain 8054263508Sdim // of computation for each trunc. 8055263508Sdim for (SDNode::use_iterator UI = LD->use_begin(), UIEnd = LD->use_end(); 8056263508Sdim UI != UIEnd; ++UI) { 8057263508Sdim // Skip the uses of the chain. 8058263508Sdim if (UI.getUse().getResNo() != 0) 8059263508Sdim continue; 8060263508Sdim 8061263508Sdim SDNode *User = *UI; 8062263508Sdim unsigned Shift = 0; 8063263508Sdim 8064263508Sdim // Check if this is a trunc(lshr). 8065263508Sdim if (User->getOpcode() == ISD::SRL && User->hasOneUse() && 8066263508Sdim isa<ConstantSDNode>(User->getOperand(1))) { 8067263508Sdim Shift = cast<ConstantSDNode>(User->getOperand(1))->getZExtValue(); 8068263508Sdim User = *User->use_begin(); 8069263508Sdim } 8070263508Sdim 8071263508Sdim // At this point, User is a Truncate, iff we encountered, trunc or 8072263508Sdim // trunc(lshr). 8073263508Sdim if (User->getOpcode() != ISD::TRUNCATE) 8074263508Sdim return false; 8075263508Sdim 8076263508Sdim // The width of the type must be a power of 2 and greater than 8-bits. 8077263508Sdim // Otherwise the load cannot be represented in LLVM IR. 8078263508Sdim // Moreover, if we shifted with a non 8-bits multiple, the slice 8079263508Sdim // will be accross several bytes. We do not support that. 8080263508Sdim unsigned Width = User->getValueSizeInBits(0); 8081263508Sdim if (Width < 8 || !isPowerOf2_32(Width) || (Shift & 0x7)) 8082263508Sdim return 0; 8083263508Sdim 8084263508Sdim // Build the slice for this chain of computations. 8085263508Sdim LoadedSlice LS(User, LD, Shift, &DAG); 8086263508Sdim APInt CurrentUsedBits = LS.getUsedBits(); 8087263508Sdim 8088263508Sdim // Check if this slice overlaps with another. 8089263508Sdim if ((CurrentUsedBits & UsedBits) != 0) 8090263508Sdim return false; 8091263508Sdim // Update the bits used globally. 8092263508Sdim UsedBits |= CurrentUsedBits; 8093263508Sdim 8094263508Sdim // Check if the new slice would be legal. 8095263508Sdim if (!LS.isLegal()) 8096263508Sdim return false; 8097263508Sdim 8098263508Sdim // Record the slice. 8099263508Sdim LoadedSlices.push_back(LS); 8100263508Sdim } 8101263508Sdim 8102263508Sdim // Abort slicing if it does not seem to be profitable. 8103263508Sdim if (!isSlicingProfitable(LoadedSlices, UsedBits, ForCodeSize)) 8104263508Sdim return false; 8105263508Sdim 8106263508Sdim ++SlicedLoads; 8107263508Sdim 8108263508Sdim // Rewrite each chain to use an independent load. 8109263508Sdim // By construction, each chain can be represented by a unique load. 8110263508Sdim 8111263508Sdim // Prepare the argument for the new token factor for all the slices. 8112263508Sdim SmallVector<SDValue, 8> ArgChains; 8113263508Sdim for (SmallVectorImpl<LoadedSlice>::const_iterator 8114263508Sdim LSIt = LoadedSlices.begin(), 8115263508Sdim LSItEnd = LoadedSlices.end(); 8116263508Sdim LSIt != LSItEnd; ++LSIt) { 8117263508Sdim SDValue SliceInst = LSIt->loadSlice(); 8118263508Sdim CombineTo(LSIt->Inst, SliceInst, true); 8119263508Sdim if (SliceInst.getNode()->getOpcode() != ISD::LOAD) 8120263508Sdim SliceInst = SliceInst.getOperand(0); 8121263508Sdim assert(SliceInst->getOpcode() == ISD::LOAD && 8122263508Sdim "It takes more than a zext to get to the loaded slice!!"); 8123263508Sdim ArgChains.push_back(SliceInst.getValue(1)); 8124263508Sdim } 8125263508Sdim 8126263508Sdim SDValue Chain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, 8127263508Sdim &ArgChains[0], ArgChains.size()); 8128263508Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Chain); 8129263508Sdim return true; 8130263508Sdim} 8131263508Sdim 8132207618Srdivacky/// CheckForMaskedLoad - Check to see if V is (and load (ptr), imm), where the 8133207618Srdivacky/// load is having specific bytes cleared out. If so, return the byte size 8134207618Srdivacky/// being masked out and the shift amount. 8135207618Srdivackystatic std::pair<unsigned, unsigned> 8136207618SrdivackyCheckForMaskedLoad(SDValue V, SDValue Ptr, SDValue Chain) { 8137207618Srdivacky std::pair<unsigned, unsigned> Result(0, 0); 8138218893Sdim 8139207618Srdivacky // Check for the structure we're looking for. 8140207618Srdivacky if (V->getOpcode() != ISD::AND || 8141207618Srdivacky !isa<ConstantSDNode>(V->getOperand(1)) || 8142207618Srdivacky !ISD::isNormalLoad(V->getOperand(0).getNode())) 8143207618Srdivacky return Result; 8144218893Sdim 8145207618Srdivacky // Check the chain and pointer. 8146207618Srdivacky LoadSDNode *LD = cast<LoadSDNode>(V->getOperand(0)); 8147207618Srdivacky if (LD->getBasePtr() != Ptr) return Result; // Not from same pointer. 8148218893Sdim 8149207618Srdivacky // The store should be chained directly to the load or be an operand of a 8150207618Srdivacky // tokenfactor. 8151207618Srdivacky if (LD == Chain.getNode()) 8152207618Srdivacky ; // ok. 8153207618Srdivacky else if (Chain->getOpcode() != ISD::TokenFactor) 8154207618Srdivacky return Result; // Fail. 8155207618Srdivacky else { 8156207618Srdivacky bool isOk = false; 8157207618Srdivacky for (unsigned i = 0, e = Chain->getNumOperands(); i != e; ++i) 8158207618Srdivacky if (Chain->getOperand(i).getNode() == LD) { 8159207618Srdivacky isOk = true; 8160207618Srdivacky break; 8161207618Srdivacky } 8162207618Srdivacky if (!isOk) return Result; 8163207618Srdivacky } 8164218893Sdim 8165207618Srdivacky // This only handles simple types. 8166207618Srdivacky if (V.getValueType() != MVT::i16 && 8167207618Srdivacky V.getValueType() != MVT::i32 && 8168207618Srdivacky V.getValueType() != MVT::i64) 8169207618Srdivacky return Result; 8170193323Sed 8171207618Srdivacky // Check the constant mask. Invert it so that the bits being masked out are 8172207618Srdivacky // 0 and the bits being kept are 1. Use getSExtValue so that leading bits 8173207618Srdivacky // follow the sign bit for uniformity. 8174207618Srdivacky uint64_t NotMask = ~cast<ConstantSDNode>(V->getOperand(1))->getSExtValue(); 8175263508Sdim unsigned NotMaskLZ = countLeadingZeros(NotMask); 8176207618Srdivacky if (NotMaskLZ & 7) return Result; // Must be multiple of a byte. 8177263508Sdim unsigned NotMaskTZ = countTrailingZeros(NotMask); 8178207618Srdivacky if (NotMaskTZ & 7) return Result; // Must be multiple of a byte. 8179207618Srdivacky if (NotMaskLZ == 64) return Result; // All zero mask. 8180218893Sdim 8181207618Srdivacky // See if we have a continuous run of bits. If so, we have 0*1+0* 8182207618Srdivacky if (CountTrailingOnes_64(NotMask >> NotMaskTZ)+NotMaskTZ+NotMaskLZ != 64) 8183207618Srdivacky return Result; 8184207618Srdivacky 8185207618Srdivacky // Adjust NotMaskLZ down to be from the actual size of the int instead of i64. 8186207618Srdivacky if (V.getValueType() != MVT::i64 && NotMaskLZ) 8187207618Srdivacky NotMaskLZ -= 64-V.getValueSizeInBits(); 8188218893Sdim 8189207618Srdivacky unsigned MaskedBytes = (V.getValueSizeInBits()-NotMaskLZ-NotMaskTZ)/8; 8190207618Srdivacky switch (MaskedBytes) { 8191218893Sdim case 1: 8192218893Sdim case 2: 8193207618Srdivacky case 4: break; 8194207618Srdivacky default: return Result; // All one mask, or 5-byte mask. 8195207618Srdivacky } 8196218893Sdim 8197207618Srdivacky // Verify that the first bit starts at a multiple of mask so that the access 8198207618Srdivacky // is aligned the same as the access width. 8199207618Srdivacky if (NotMaskTZ && NotMaskTZ/8 % MaskedBytes) return Result; 8200218893Sdim 8201207618Srdivacky Result.first = MaskedBytes; 8202207618Srdivacky Result.second = NotMaskTZ/8; 8203207618Srdivacky return Result; 8204207618Srdivacky} 8205207618Srdivacky 8206207618Srdivacky 8207207618Srdivacky/// ShrinkLoadReplaceStoreWithStore - Check to see if IVal is something that 8208207618Srdivacky/// provides a value as specified by MaskInfo. If so, replace the specified 8209207618Srdivacky/// store with a narrower store of truncated IVal. 8210207618Srdivackystatic SDNode * 8211207618SrdivackyShrinkLoadReplaceStoreWithStore(const std::pair<unsigned, unsigned> &MaskInfo, 8212207618Srdivacky SDValue IVal, StoreSDNode *St, 8213207618Srdivacky DAGCombiner *DC) { 8214207618Srdivacky unsigned NumBytes = MaskInfo.first; 8215207618Srdivacky unsigned ByteShift = MaskInfo.second; 8216207618Srdivacky SelectionDAG &DAG = DC->getDAG(); 8217218893Sdim 8218207618Srdivacky // Check to see if IVal is all zeros in the part being masked in by the 'or' 8219207618Srdivacky // that uses this. If not, this is not a replacement. 8220207618Srdivacky APInt Mask = ~APInt::getBitsSet(IVal.getValueSizeInBits(), 8221207618Srdivacky ByteShift*8, (ByteShift+NumBytes)*8); 8222207618Srdivacky if (!DAG.MaskedValueIsZero(IVal, Mask)) return 0; 8223218893Sdim 8224207618Srdivacky // Check that it is legal on the target to do this. It is legal if the new 8225207618Srdivacky // VT we're shrinking to (i8/i16/i32) is legal or we're still before type 8226207618Srdivacky // legalization. 8227207618Srdivacky MVT VT = MVT::getIntegerVT(NumBytes*8); 8228207618Srdivacky if (!DC->isTypeLegal(VT)) 8229207618Srdivacky return 0; 8230218893Sdim 8231207618Srdivacky // Okay, we can do this! Replace the 'St' store with a store of IVal that is 8232207618Srdivacky // shifted by ByteShift and truncated down to NumBytes. 8233207618Srdivacky if (ByteShift) 8234263508Sdim IVal = DAG.getNode(ISD::SRL, SDLoc(IVal), IVal.getValueType(), IVal, 8235219077Sdim DAG.getConstant(ByteShift*8, 8236219077Sdim DC->getShiftAmountTy(IVal.getValueType()))); 8237207618Srdivacky 8238207618Srdivacky // Figure out the offset for the store and the alignment of the access. 8239207618Srdivacky unsigned StOffset; 8240207618Srdivacky unsigned NewAlign = St->getAlignment(); 8241207618Srdivacky 8242207618Srdivacky if (DAG.getTargetLoweringInfo().isLittleEndian()) 8243207618Srdivacky StOffset = ByteShift; 8244207618Srdivacky else 8245207618Srdivacky StOffset = IVal.getValueType().getStoreSize() - ByteShift - NumBytes; 8246218893Sdim 8247207618Srdivacky SDValue Ptr = St->getBasePtr(); 8248207618Srdivacky if (StOffset) { 8249263508Sdim Ptr = DAG.getNode(ISD::ADD, SDLoc(IVal), Ptr.getValueType(), 8250207618Srdivacky Ptr, DAG.getConstant(StOffset, Ptr.getValueType())); 8251207618Srdivacky NewAlign = MinAlign(NewAlign, StOffset); 8252207618Srdivacky } 8253218893Sdim 8254207618Srdivacky // Truncate down to the new size. 8255263508Sdim IVal = DAG.getNode(ISD::TRUNCATE, SDLoc(IVal), VT, IVal); 8256218893Sdim 8257207618Srdivacky ++OpsNarrowed; 8258263508Sdim return DAG.getStore(St->getChain(), SDLoc(St), IVal, Ptr, 8259218893Sdim St->getPointerInfo().getWithOffset(StOffset), 8260207618Srdivacky false, false, NewAlign).getNode(); 8261207618Srdivacky} 8262207618Srdivacky 8263207618Srdivacky 8264193323Sed/// ReduceLoadOpStoreWidth - Look for sequence of load / op / store where op is 8265193323Sed/// one of 'or', 'xor', and 'and' of immediates. If 'op' is only touching some 8266193323Sed/// of the loaded bits, try narrowing the load and store if it would end up 8267193323Sed/// being a win for performance or code size. 8268193323SedSDValue DAGCombiner::ReduceLoadOpStoreWidth(SDNode *N) { 8269193323Sed StoreSDNode *ST = cast<StoreSDNode>(N); 8270193323Sed if (ST->isVolatile()) 8271193323Sed return SDValue(); 8272193323Sed 8273193323Sed SDValue Chain = ST->getChain(); 8274193323Sed SDValue Value = ST->getValue(); 8275193323Sed SDValue Ptr = ST->getBasePtr(); 8276198090Srdivacky EVT VT = Value.getValueType(); 8277193323Sed 8278193323Sed if (ST->isTruncatingStore() || VT.isVector() || !Value.hasOneUse()) 8279193323Sed return SDValue(); 8280193323Sed 8281193323Sed unsigned Opc = Value.getOpcode(); 8282218893Sdim 8283207618Srdivacky // If this is "store (or X, Y), P" and X is "(and (load P), cst)", where cst 8284207618Srdivacky // is a byte mask indicating a consecutive number of bytes, check to see if 8285207618Srdivacky // Y is known to provide just those bytes. If so, we try to replace the 8286207618Srdivacky // load + replace + store sequence with a single (narrower) store, which makes 8287207618Srdivacky // the load dead. 8288207618Srdivacky if (Opc == ISD::OR) { 8289207618Srdivacky std::pair<unsigned, unsigned> MaskedLoad; 8290207618Srdivacky MaskedLoad = CheckForMaskedLoad(Value.getOperand(0), Ptr, Chain); 8291207618Srdivacky if (MaskedLoad.first) 8292207618Srdivacky if (SDNode *NewST = ShrinkLoadReplaceStoreWithStore(MaskedLoad, 8293207618Srdivacky Value.getOperand(1), ST,this)) 8294207618Srdivacky return SDValue(NewST, 0); 8295218893Sdim 8296207618Srdivacky // Or is commutative, so try swapping X and Y. 8297207618Srdivacky MaskedLoad = CheckForMaskedLoad(Value.getOperand(1), Ptr, Chain); 8298207618Srdivacky if (MaskedLoad.first) 8299207618Srdivacky if (SDNode *NewST = ShrinkLoadReplaceStoreWithStore(MaskedLoad, 8300207618Srdivacky Value.getOperand(0), ST,this)) 8301207618Srdivacky return SDValue(NewST, 0); 8302207618Srdivacky } 8303218893Sdim 8304193323Sed if ((Opc != ISD::OR && Opc != ISD::XOR && Opc != ISD::AND) || 8305193323Sed Value.getOperand(1).getOpcode() != ISD::Constant) 8306193323Sed return SDValue(); 8307193323Sed 8308193323Sed SDValue N0 = Value.getOperand(0); 8309212904Sdim if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && 8310212904Sdim Chain == SDValue(N0.getNode(), 1)) { 8311193323Sed LoadSDNode *LD = cast<LoadSDNode>(N0); 8312218893Sdim if (LD->getBasePtr() != Ptr || 8313218893Sdim LD->getPointerInfo().getAddrSpace() != 8314218893Sdim ST->getPointerInfo().getAddrSpace()) 8315193323Sed return SDValue(); 8316193323Sed 8317193323Sed // Find the type to narrow it the load / op / store to. 8318193323Sed SDValue N1 = Value.getOperand(1); 8319193323Sed unsigned BitWidth = N1.getValueSizeInBits(); 8320193323Sed APInt Imm = cast<ConstantSDNode>(N1)->getAPIntValue(); 8321193323Sed if (Opc == ISD::AND) 8322193323Sed Imm ^= APInt::getAllOnesValue(BitWidth); 8323193323Sed if (Imm == 0 || Imm.isAllOnesValue()) 8324193323Sed return SDValue(); 8325193323Sed unsigned ShAmt = Imm.countTrailingZeros(); 8326193323Sed unsigned MSB = BitWidth - Imm.countLeadingZeros() - 1; 8327193323Sed unsigned NewBW = NextPowerOf2(MSB - ShAmt); 8328198090Srdivacky EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), NewBW); 8329193323Sed while (NewBW < BitWidth && 8330193323Sed !(TLI.isOperationLegalOrCustom(Opc, NewVT) && 8331193323Sed TLI.isNarrowingProfitable(VT, NewVT))) { 8332193323Sed NewBW = NextPowerOf2(NewBW); 8333198090Srdivacky NewVT = EVT::getIntegerVT(*DAG.getContext(), NewBW); 8334193323Sed } 8335193323Sed if (NewBW >= BitWidth) 8336193323Sed return SDValue(); 8337193323Sed 8338193323Sed // If the lsb changed does not start at the type bitwidth boundary, 8339193323Sed // start at the previous one. 8340193323Sed if (ShAmt % NewBW) 8341193323Sed ShAmt = (((ShAmt + NewBW - 1) / NewBW) * NewBW) - NewBW; 8342249423Sdim APInt Mask = APInt::getBitsSet(BitWidth, ShAmt, 8343249423Sdim std::min(BitWidth, ShAmt + NewBW)); 8344193323Sed if ((Imm & Mask) == Imm) { 8345193323Sed APInt NewImm = (Imm & Mask).lshr(ShAmt).trunc(NewBW); 8346193323Sed if (Opc == ISD::AND) 8347193323Sed NewImm ^= APInt::getAllOnesValue(NewBW); 8348193323Sed uint64_t PtrOff = ShAmt / 8; 8349193323Sed // For big endian targets, we need to adjust the offset to the pointer to 8350193323Sed // load the correct bytes. 8351193323Sed if (TLI.isBigEndian()) 8352193323Sed PtrOff = (BitWidth + 7 - NewBW) / 8 - PtrOff; 8353193323Sed 8354193323Sed unsigned NewAlign = MinAlign(LD->getAlignment(), PtrOff); 8355226633Sdim Type *NewVTTy = NewVT.getTypeForEVT(*DAG.getContext()); 8356243830Sdim if (NewAlign < TLI.getDataLayout()->getABITypeAlignment(NewVTTy)) 8357193323Sed return SDValue(); 8358193323Sed 8359263508Sdim SDValue NewPtr = DAG.getNode(ISD::ADD, SDLoc(LD), 8360193323Sed Ptr.getValueType(), Ptr, 8361193323Sed DAG.getConstant(PtrOff, Ptr.getValueType())); 8362263508Sdim SDValue NewLD = DAG.getLoad(NewVT, SDLoc(N0), 8363193323Sed LD->getChain(), NewPtr, 8364218893Sdim LD->getPointerInfo().getWithOffset(PtrOff), 8365203954Srdivacky LD->isVolatile(), LD->isNonTemporal(), 8366263508Sdim LD->isInvariant(), NewAlign, 8367263508Sdim LD->getTBAAInfo()); 8368263508Sdim SDValue NewVal = DAG.getNode(Opc, SDLoc(Value), NewVT, NewLD, 8369193323Sed DAG.getConstant(NewImm, NewVT)); 8370263508Sdim SDValue NewST = DAG.getStore(Chain, SDLoc(N), 8371193323Sed NewVal, NewPtr, 8372218893Sdim ST->getPointerInfo().getWithOffset(PtrOff), 8373203954Srdivacky false, false, NewAlign); 8374193323Sed 8375193323Sed AddToWorkList(NewPtr.getNode()); 8376193323Sed AddToWorkList(NewLD.getNode()); 8377193323Sed AddToWorkList(NewVal.getNode()); 8378193323Sed WorkListRemover DeadNodes(*this); 8379239462Sdim DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), NewLD.getValue(1)); 8380193323Sed ++OpsNarrowed; 8381193323Sed return NewST; 8382193323Sed } 8383193323Sed } 8384193323Sed 8385193323Sed return SDValue(); 8386193323Sed} 8387193323Sed 8388218893Sdim/// TransformFPLoadStorePair - For a given floating point load / store pair, 8389218893Sdim/// if the load value isn't used by any other operations, then consider 8390218893Sdim/// transforming the pair to integer load / store operations if the target 8391218893Sdim/// deems the transformation profitable. 8392218893SdimSDValue DAGCombiner::TransformFPLoadStorePair(SDNode *N) { 8393218893Sdim StoreSDNode *ST = cast<StoreSDNode>(N); 8394218893Sdim SDValue Chain = ST->getChain(); 8395218893Sdim SDValue Value = ST->getValue(); 8396218893Sdim if (ISD::isNormalStore(ST) && ISD::isNormalLoad(Value.getNode()) && 8397218893Sdim Value.hasOneUse() && 8398218893Sdim Chain == SDValue(Value.getNode(), 1)) { 8399218893Sdim LoadSDNode *LD = cast<LoadSDNode>(Value); 8400218893Sdim EVT VT = LD->getMemoryVT(); 8401218893Sdim if (!VT.isFloatingPoint() || 8402218893Sdim VT != ST->getMemoryVT() || 8403218893Sdim LD->isNonTemporal() || 8404218893Sdim ST->isNonTemporal() || 8405218893Sdim LD->getPointerInfo().getAddrSpace() != 0 || 8406218893Sdim ST->getPointerInfo().getAddrSpace() != 0) 8407218893Sdim return SDValue(); 8408218893Sdim 8409218893Sdim EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 8410218893Sdim if (!TLI.isOperationLegal(ISD::LOAD, IntVT) || 8411218893Sdim !TLI.isOperationLegal(ISD::STORE, IntVT) || 8412218893Sdim !TLI.isDesirableToTransformToIntegerOp(ISD::LOAD, VT) || 8413218893Sdim !TLI.isDesirableToTransformToIntegerOp(ISD::STORE, VT)) 8414218893Sdim return SDValue(); 8415218893Sdim 8416218893Sdim unsigned LDAlign = LD->getAlignment(); 8417218893Sdim unsigned STAlign = ST->getAlignment(); 8418226633Sdim Type *IntVTTy = IntVT.getTypeForEVT(*DAG.getContext()); 8419243830Sdim unsigned ABIAlign = TLI.getDataLayout()->getABITypeAlignment(IntVTTy); 8420218893Sdim if (LDAlign < ABIAlign || STAlign < ABIAlign) 8421218893Sdim return SDValue(); 8422218893Sdim 8423263508Sdim SDValue NewLD = DAG.getLoad(IntVT, SDLoc(Value), 8424218893Sdim LD->getChain(), LD->getBasePtr(), 8425218893Sdim LD->getPointerInfo(), 8426234353Sdim false, false, false, LDAlign); 8427218893Sdim 8428263508Sdim SDValue NewST = DAG.getStore(NewLD.getValue(1), SDLoc(N), 8429218893Sdim NewLD, ST->getBasePtr(), 8430218893Sdim ST->getPointerInfo(), 8431218893Sdim false, false, STAlign); 8432218893Sdim 8433218893Sdim AddToWorkList(NewLD.getNode()); 8434218893Sdim AddToWorkList(NewST.getNode()); 8435218893Sdim WorkListRemover DeadNodes(*this); 8436239462Sdim DAG.ReplaceAllUsesOfValueWith(Value.getValue(1), NewLD.getValue(1)); 8437218893Sdim ++LdStFP2Int; 8438218893Sdim return NewST; 8439218893Sdim } 8440218893Sdim 8441218893Sdim return SDValue(); 8442218893Sdim} 8443218893Sdim 8444249423Sdim/// Helper struct to parse and store a memory address as base + index + offset. 8445249423Sdim/// We ignore sign extensions when it is safe to do so. 8446249423Sdim/// The following two expressions are not equivalent. To differentiate we need 8447249423Sdim/// to store whether there was a sign extension involved in the index 8448249423Sdim/// computation. 8449249423Sdim/// (load (i64 add (i64 copyfromreg %c) 8450249423Sdim/// (i64 signextend (add (i8 load %index) 8451249423Sdim/// (i8 1)))) 8452249423Sdim/// vs 8453249423Sdim/// 8454249423Sdim/// (load (i64 add (i64 copyfromreg %c) 8455249423Sdim/// (i64 signextend (i32 add (i32 signextend (i8 load %index)) 8456249423Sdim/// (i32 1))))) 8457249423Sdimstruct BaseIndexOffset { 8458249423Sdim SDValue Base; 8459249423Sdim SDValue Index; 8460249423Sdim int64_t Offset; 8461249423Sdim bool IsIndexSignExt; 8462249423Sdim 8463249423Sdim BaseIndexOffset() : Offset(0), IsIndexSignExt(false) {} 8464249423Sdim 8465249423Sdim BaseIndexOffset(SDValue Base, SDValue Index, int64_t Offset, 8466249423Sdim bool IsIndexSignExt) : 8467249423Sdim Base(Base), Index(Index), Offset(Offset), IsIndexSignExt(IsIndexSignExt) {} 8468249423Sdim 8469249423Sdim bool equalBaseIndex(const BaseIndexOffset &Other) { 8470249423Sdim return Other.Base == Base && Other.Index == Index && 8471249423Sdim Other.IsIndexSignExt == IsIndexSignExt; 8472249423Sdim } 8473249423Sdim 8474249423Sdim /// Parses tree in Ptr for base, index, offset addresses. 8475249423Sdim static BaseIndexOffset match(SDValue Ptr) { 8476249423Sdim bool IsIndexSignExt = false; 8477249423Sdim 8478263508Sdim // We only can pattern match BASE + INDEX + OFFSET. If Ptr is not an ADD 8479263508Sdim // instruction, then it could be just the BASE or everything else we don't 8480263508Sdim // know how to handle. Just use Ptr as BASE and give up. 8481249423Sdim if (Ptr->getOpcode() != ISD::ADD) 8482249423Sdim return BaseIndexOffset(Ptr, SDValue(), 0, IsIndexSignExt); 8483249423Sdim 8484263508Sdim // We know that we have at least an ADD instruction. Try to pattern match 8485263508Sdim // the simple case of BASE + OFFSET. 8486249423Sdim if (isa<ConstantSDNode>(Ptr->getOperand(1))) { 8487249423Sdim int64_t Offset = cast<ConstantSDNode>(Ptr->getOperand(1))->getSExtValue(); 8488249423Sdim return BaseIndexOffset(Ptr->getOperand(0), SDValue(), Offset, 8489249423Sdim IsIndexSignExt); 8490249423Sdim } 8491249423Sdim 8492263508Sdim // Inside a loop the current BASE pointer is calculated using an ADD and a 8493263508Sdim // MUL instruction. In this case Ptr is the actual BASE pointer. 8494263508Sdim // (i64 add (i64 %array_ptr) 8495263508Sdim // (i64 mul (i64 %induction_var) 8496263508Sdim // (i64 %element_size))) 8497263508Sdim if (Ptr->getOperand(1)->getOpcode() == ISD::MUL) 8498263508Sdim return BaseIndexOffset(Ptr, SDValue(), 0, IsIndexSignExt); 8499263508Sdim 8500249423Sdim // Look at Base + Index + Offset cases. 8501243830Sdim SDValue Base = Ptr->getOperand(0); 8502249423Sdim SDValue IndexOffset = Ptr->getOperand(1); 8503249423Sdim 8504249423Sdim // Skip signextends. 8505249423Sdim if (IndexOffset->getOpcode() == ISD::SIGN_EXTEND) { 8506249423Sdim IndexOffset = IndexOffset->getOperand(0); 8507249423Sdim IsIndexSignExt = true; 8508249423Sdim } 8509249423Sdim 8510249423Sdim // Either the case of Base + Index (no offset) or something else. 8511249423Sdim if (IndexOffset->getOpcode() != ISD::ADD) 8512249423Sdim return BaseIndexOffset(Base, IndexOffset, 0, IsIndexSignExt); 8513249423Sdim 8514249423Sdim // Now we have the case of Base + Index + offset. 8515249423Sdim SDValue Index = IndexOffset->getOperand(0); 8516249423Sdim SDValue Offset = IndexOffset->getOperand(1); 8517249423Sdim 8518249423Sdim if (!isa<ConstantSDNode>(Offset)) 8519249423Sdim return BaseIndexOffset(Ptr, SDValue(), 0, IsIndexSignExt); 8520249423Sdim 8521249423Sdim // Ignore signextends. 8522249423Sdim if (Index->getOpcode() == ISD::SIGN_EXTEND) { 8523249423Sdim Index = Index->getOperand(0); 8524249423Sdim IsIndexSignExt = true; 8525249423Sdim } else IsIndexSignExt = false; 8526249423Sdim 8527249423Sdim int64_t Off = cast<ConstantSDNode>(Offset)->getSExtValue(); 8528249423Sdim return BaseIndexOffset(Base, Index, Off, IsIndexSignExt); 8529243830Sdim } 8530249423Sdim}; 8531243830Sdim 8532243830Sdim/// Holds a pointer to an LSBaseSDNode as well as information on where it 8533243830Sdim/// is located in a sequence of memory operations connected by a chain. 8534243830Sdimstruct MemOpLink { 8535243830Sdim MemOpLink (LSBaseSDNode *N, int64_t Offset, unsigned Seq): 8536243830Sdim MemNode(N), OffsetFromBase(Offset), SequenceNum(Seq) { } 8537243830Sdim // Ptr to the mem node. 8538243830Sdim LSBaseSDNode *MemNode; 8539243830Sdim // Offset from the base ptr. 8540243830Sdim int64_t OffsetFromBase; 8541243830Sdim // What is the sequence number of this mem node. 8542243830Sdim // Lowest mem operand in the DAG starts at zero. 8543243830Sdim unsigned SequenceNum; 8544243830Sdim}; 8545243830Sdim 8546243830Sdim/// Sorts store nodes in a link according to their offset from a shared 8547243830Sdim// base ptr. 8548243830Sdimstruct ConsecutiveMemoryChainSorter { 8549243830Sdim bool operator()(MemOpLink LHS, MemOpLink RHS) { 8550266715Sdim return 8551266715Sdim LHS.OffsetFromBase < RHS.OffsetFromBase || 8552266715Sdim (LHS.OffsetFromBase == RHS.OffsetFromBase && 8553266715Sdim LHS.SequenceNum > RHS.SequenceNum); 8554243830Sdim } 8555243830Sdim}; 8556243830Sdim 8557243830Sdimbool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) { 8558243830Sdim EVT MemVT = St->getMemoryVT(); 8559243830Sdim int64_t ElementSizeBytes = MemVT.getSizeInBits()/8; 8560249423Sdim bool NoVectors = DAG.getMachineFunction().getFunction()->getAttributes(). 8561249423Sdim hasAttribute(AttributeSet::FunctionIndex, Attribute::NoImplicitFloat); 8562243830Sdim 8563243830Sdim // Don't merge vectors into wider inputs. 8564243830Sdim if (MemVT.isVector() || !MemVT.isSimple()) 8565243830Sdim return false; 8566243830Sdim 8567243830Sdim // Perform an early exit check. Do not bother looking at stored values that 8568243830Sdim // are not constants or loads. 8569243830Sdim SDValue StoredVal = St->getValue(); 8570243830Sdim bool IsLoadSrc = isa<LoadSDNode>(StoredVal); 8571243830Sdim if (!isa<ConstantSDNode>(StoredVal) && !isa<ConstantFPSDNode>(StoredVal) && 8572243830Sdim !IsLoadSrc) 8573243830Sdim return false; 8574243830Sdim 8575243830Sdim // Only look at ends of store sequences. 8576243830Sdim SDValue Chain = SDValue(St, 1); 8577243830Sdim if (Chain->hasOneUse() && Chain->use_begin()->getOpcode() == ISD::STORE) 8578243830Sdim return false; 8579243830Sdim 8580249423Sdim // This holds the base pointer, index, and the offset in bytes from the base 8581249423Sdim // pointer. 8582249423Sdim BaseIndexOffset BasePtr = BaseIndexOffset::match(St->getBasePtr()); 8583243830Sdim 8584243830Sdim // We must have a base and an offset. 8585249423Sdim if (!BasePtr.Base.getNode()) 8586243830Sdim return false; 8587243830Sdim 8588243830Sdim // Do not handle stores to undef base pointers. 8589249423Sdim if (BasePtr.Base.getOpcode() == ISD::UNDEF) 8590243830Sdim return false; 8591243830Sdim 8592249423Sdim // Save the LoadSDNodes that we find in the chain. 8593249423Sdim // We need to make sure that these nodes do not interfere with 8594249423Sdim // any of the store nodes. 8595249423Sdim SmallVector<LSBaseSDNode*, 8> AliasLoadNodes; 8596249423Sdim 8597249423Sdim // Save the StoreSDNodes that we find in the chain. 8598243830Sdim SmallVector<MemOpLink, 8> StoreNodes; 8599249423Sdim 8600243830Sdim // Walk up the chain and look for nodes with offsets from the same 8601243830Sdim // base pointer. Stop when reaching an instruction with a different kind 8602243830Sdim // or instruction which has a different base pointer. 8603243830Sdim unsigned Seq = 0; 8604243830Sdim StoreSDNode *Index = St; 8605243830Sdim while (Index) { 8606243830Sdim // If the chain has more than one use, then we can't reorder the mem ops. 8607243830Sdim if (Index != St && !SDValue(Index, 1)->hasOneUse()) 8608243830Sdim break; 8609243830Sdim 8610243830Sdim // Find the base pointer and offset for this memory node. 8611249423Sdim BaseIndexOffset Ptr = BaseIndexOffset::match(Index->getBasePtr()); 8612243830Sdim 8613243830Sdim // Check that the base pointer is the same as the original one. 8614249423Sdim if (!Ptr.equalBaseIndex(BasePtr)) 8615243830Sdim break; 8616243830Sdim 8617243830Sdim // Check that the alignment is the same. 8618243830Sdim if (Index->getAlignment() != St->getAlignment()) 8619243830Sdim break; 8620243830Sdim 8621243830Sdim // The memory operands must not be volatile. 8622243830Sdim if (Index->isVolatile() || Index->isIndexed()) 8623243830Sdim break; 8624243830Sdim 8625243830Sdim // No truncation. 8626243830Sdim if (StoreSDNode *St = dyn_cast<StoreSDNode>(Index)) 8627243830Sdim if (St->isTruncatingStore()) 8628243830Sdim break; 8629243830Sdim 8630243830Sdim // The stored memory type must be the same. 8631243830Sdim if (Index->getMemoryVT() != MemVT) 8632243830Sdim break; 8633243830Sdim 8634243830Sdim // We do not allow unaligned stores because we want to prevent overriding 8635243830Sdim // stores. 8636243830Sdim if (Index->getAlignment()*8 != MemVT.getSizeInBits()) 8637243830Sdim break; 8638243830Sdim 8639243830Sdim // We found a potential memory operand to merge. 8640249423Sdim StoreNodes.push_back(MemOpLink(Index, Ptr.Offset, Seq++)); 8641243830Sdim 8642249423Sdim // Find the next memory operand in the chain. If the next operand in the 8643249423Sdim // chain is a store then move up and continue the scan with the next 8644249423Sdim // memory operand. If the next operand is a load save it and use alias 8645249423Sdim // information to check if it interferes with anything. 8646249423Sdim SDNode *NextInChain = Index->getChain().getNode(); 8647249423Sdim while (1) { 8648249423Sdim if (StoreSDNode *STn = dyn_cast<StoreSDNode>(NextInChain)) { 8649249423Sdim // We found a store node. Use it for the next iteration. 8650249423Sdim Index = STn; 8651249423Sdim break; 8652249423Sdim } else if (LoadSDNode *Ldn = dyn_cast<LoadSDNode>(NextInChain)) { 8653263508Sdim if (Ldn->isVolatile()) { 8654263508Sdim Index = NULL; 8655263508Sdim break; 8656263508Sdim } 8657263508Sdim 8658249423Sdim // Save the load node for later. Continue the scan. 8659249423Sdim AliasLoadNodes.push_back(Ldn); 8660249423Sdim NextInChain = Ldn->getChain().getNode(); 8661249423Sdim continue; 8662249423Sdim } else { 8663249423Sdim Index = NULL; 8664249423Sdim break; 8665249423Sdim } 8666249423Sdim } 8667243830Sdim } 8668243830Sdim 8669243830Sdim // Check if there is anything to merge. 8670243830Sdim if (StoreNodes.size() < 2) 8671243830Sdim return false; 8672243830Sdim 8673243830Sdim // Sort the memory operands according to their distance from the base pointer. 8674243830Sdim std::sort(StoreNodes.begin(), StoreNodes.end(), 8675243830Sdim ConsecutiveMemoryChainSorter()); 8676243830Sdim 8677243830Sdim // Scan the memory operations on the chain and find the first non-consecutive 8678243830Sdim // store memory address. 8679243830Sdim unsigned LastConsecutiveStore = 0; 8680243830Sdim int64_t StartAddress = StoreNodes[0].OffsetFromBase; 8681249423Sdim for (unsigned i = 0, e = StoreNodes.size(); i < e; ++i) { 8682249423Sdim 8683249423Sdim // Check that the addresses are consecutive starting from the second 8684249423Sdim // element in the list of stores. 8685249423Sdim if (i > 0) { 8686249423Sdim int64_t CurrAddress = StoreNodes[i].OffsetFromBase; 8687249423Sdim if (CurrAddress - StartAddress != (ElementSizeBytes * i)) 8688249423Sdim break; 8689249423Sdim } 8690249423Sdim 8691249423Sdim bool Alias = false; 8692249423Sdim // Check if this store interferes with any of the loads that we found. 8693249423Sdim for (unsigned ld = 0, lde = AliasLoadNodes.size(); ld < lde; ++ld) 8694249423Sdim if (isAlias(AliasLoadNodes[ld], StoreNodes[i].MemNode)) { 8695249423Sdim Alias = true; 8696249423Sdim break; 8697249423Sdim } 8698249423Sdim // We found a load that alias with this store. Stop the sequence. 8699249423Sdim if (Alias) 8700243830Sdim break; 8701243830Sdim 8702243830Sdim // Mark this node as useful. 8703243830Sdim LastConsecutiveStore = i; 8704243830Sdim } 8705243830Sdim 8706243830Sdim // The node with the lowest store address. 8707243830Sdim LSBaseSDNode *FirstInChain = StoreNodes[0].MemNode; 8708243830Sdim 8709243830Sdim // Store the constants into memory as one consecutive store. 8710243830Sdim if (!IsLoadSrc) { 8711243830Sdim unsigned LastLegalType = 0; 8712243830Sdim unsigned LastLegalVectorType = 0; 8713243830Sdim bool NonZero = false; 8714243830Sdim for (unsigned i=0; i<LastConsecutiveStore+1; ++i) { 8715243830Sdim StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode); 8716243830Sdim SDValue StoredVal = St->getValue(); 8717243830Sdim 8718243830Sdim if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(StoredVal)) { 8719243830Sdim NonZero |= !C->isNullValue(); 8720243830Sdim } else if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(StoredVal)) { 8721243830Sdim NonZero |= !C->getConstantFPValue()->isNullValue(); 8722243830Sdim } else { 8723243830Sdim // Non constant. 8724243830Sdim break; 8725243830Sdim } 8726243830Sdim 8727243830Sdim // Find a legal type for the constant store. 8728243830Sdim unsigned StoreBW = (i+1) * ElementSizeBytes * 8; 8729243830Sdim EVT StoreTy = EVT::getIntegerVT(*DAG.getContext(), StoreBW); 8730243830Sdim if (TLI.isTypeLegal(StoreTy)) 8731243830Sdim LastLegalType = i+1; 8732249423Sdim // Or check whether a truncstore is legal. 8733249423Sdim else if (TLI.getTypeAction(*DAG.getContext(), StoreTy) == 8734249423Sdim TargetLowering::TypePromoteInteger) { 8735249423Sdim EVT LegalizedStoredValueTy = 8736249423Sdim TLI.getTypeToTransformTo(*DAG.getContext(), StoredVal.getValueType()); 8737249423Sdim if (TLI.isTruncStoreLegal(LegalizedStoredValueTy, StoreTy)) 8738249423Sdim LastLegalType = i+1; 8739249423Sdim } 8740243830Sdim 8741243830Sdim // Find a legal type for the vector store. 8742243830Sdim EVT Ty = EVT::getVectorVT(*DAG.getContext(), MemVT, i+1); 8743243830Sdim if (TLI.isTypeLegal(Ty)) 8744243830Sdim LastLegalVectorType = i + 1; 8745243830Sdim } 8746243830Sdim 8747249423Sdim // We only use vectors if the constant is known to be zero and the 8748249423Sdim // function is not marked with the noimplicitfloat attribute. 8749249423Sdim if (NonZero || NoVectors) 8750243830Sdim LastLegalVectorType = 0; 8751243830Sdim 8752243830Sdim // Check if we found a legal integer type to store. 8753243830Sdim if (LastLegalType == 0 && LastLegalVectorType == 0) 8754243830Sdim return false; 8755243830Sdim 8756249423Sdim bool UseVector = (LastLegalVectorType > LastLegalType) && !NoVectors; 8757243830Sdim unsigned NumElem = UseVector ? LastLegalVectorType : LastLegalType; 8758243830Sdim 8759243830Sdim // Make sure we have something to merge. 8760243830Sdim if (NumElem < 2) 8761243830Sdim return false; 8762243830Sdim 8763243830Sdim unsigned EarliestNodeUsed = 0; 8764243830Sdim for (unsigned i=0; i < NumElem; ++i) { 8765243830Sdim // Find a chain for the new wide-store operand. Notice that some 8766243830Sdim // of the store nodes that we found may not be selected for inclusion 8767243830Sdim // in the wide store. The chain we use needs to be the chain of the 8768243830Sdim // earliest store node which is *used* and replaced by the wide store. 8769243830Sdim if (StoreNodes[i].SequenceNum > StoreNodes[EarliestNodeUsed].SequenceNum) 8770243830Sdim EarliestNodeUsed = i; 8771243830Sdim } 8772243830Sdim 8773243830Sdim // The earliest Node in the DAG. 8774243830Sdim LSBaseSDNode *EarliestOp = StoreNodes[EarliestNodeUsed].MemNode; 8775263508Sdim SDLoc DL(StoreNodes[0].MemNode); 8776243830Sdim 8777243830Sdim SDValue StoredVal; 8778243830Sdim if (UseVector) { 8779243830Sdim // Find a legal type for the vector store. 8780243830Sdim EVT Ty = EVT::getVectorVT(*DAG.getContext(), MemVT, NumElem); 8781243830Sdim assert(TLI.isTypeLegal(Ty) && "Illegal vector store"); 8782243830Sdim StoredVal = DAG.getConstant(0, Ty); 8783243830Sdim } else { 8784243830Sdim unsigned StoreBW = NumElem * ElementSizeBytes * 8; 8785243830Sdim APInt StoreInt(StoreBW, 0); 8786243830Sdim 8787243830Sdim // Construct a single integer constant which is made of the smaller 8788243830Sdim // constant inputs. 8789243830Sdim bool IsLE = TLI.isLittleEndian(); 8790243830Sdim for (unsigned i = 0; i < NumElem ; ++i) { 8791243830Sdim unsigned Idx = IsLE ?(NumElem - 1 - i) : i; 8792243830Sdim StoreSDNode *St = cast<StoreSDNode>(StoreNodes[Idx].MemNode); 8793243830Sdim SDValue Val = St->getValue(); 8794243830Sdim StoreInt<<=ElementSizeBytes*8; 8795243830Sdim if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val)) { 8796243830Sdim StoreInt|=C->getAPIntValue().zext(StoreBW); 8797243830Sdim } else if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val)) { 8798243830Sdim StoreInt|= C->getValueAPF().bitcastToAPInt().zext(StoreBW); 8799243830Sdim } else { 8800243830Sdim assert(false && "Invalid constant element type"); 8801243830Sdim } 8802243830Sdim } 8803243830Sdim 8804243830Sdim // Create the new Load and Store operations. 8805243830Sdim EVT StoreTy = EVT::getIntegerVT(*DAG.getContext(), StoreBW); 8806243830Sdim StoredVal = DAG.getConstant(StoreInt, StoreTy); 8807243830Sdim } 8808243830Sdim 8809243830Sdim SDValue NewStore = DAG.getStore(EarliestOp->getChain(), DL, StoredVal, 8810243830Sdim FirstInChain->getBasePtr(), 8811243830Sdim FirstInChain->getPointerInfo(), 8812243830Sdim false, false, 8813243830Sdim FirstInChain->getAlignment()); 8814243830Sdim 8815243830Sdim // Replace the first store with the new store 8816243830Sdim CombineTo(EarliestOp, NewStore); 8817243830Sdim // Erase all other stores. 8818243830Sdim for (unsigned i = 0; i < NumElem ; ++i) { 8819243830Sdim if (StoreNodes[i].MemNode == EarliestOp) 8820243830Sdim continue; 8821243830Sdim StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode); 8822243830Sdim // ReplaceAllUsesWith will replace all uses that existed when it was 8823243830Sdim // called, but graph optimizations may cause new ones to appear. For 8824243830Sdim // example, the case in pr14333 looks like 8825243830Sdim // 8826243830Sdim // St's chain -> St -> another store -> X 8827243830Sdim // 8828243830Sdim // And the only difference from St to the other store is the chain. 8829243830Sdim // When we change it's chain to be St's chain they become identical, 8830243830Sdim // get CSEed and the net result is that X is now a use of St. 8831243830Sdim // Since we know that St is redundant, just iterate. 8832243830Sdim while (!St->use_empty()) 8833243830Sdim DAG.ReplaceAllUsesWith(SDValue(St, 0), St->getChain()); 8834243830Sdim removeFromWorkList(St); 8835243830Sdim DAG.DeleteNode(St); 8836243830Sdim } 8837243830Sdim 8838243830Sdim return true; 8839243830Sdim } 8840243830Sdim 8841243830Sdim // Below we handle the case of multiple consecutive stores that 8842243830Sdim // come from multiple consecutive loads. We merge them into a single 8843243830Sdim // wide load and a single wide store. 8844243830Sdim 8845243830Sdim // Look for load nodes which are used by the stored values. 8846243830Sdim SmallVector<MemOpLink, 8> LoadNodes; 8847243830Sdim 8848243830Sdim // Find acceptable loads. Loads need to have the same chain (token factor), 8849243830Sdim // must not be zext, volatile, indexed, and they must be consecutive. 8850249423Sdim BaseIndexOffset LdBasePtr; 8851243830Sdim for (unsigned i=0; i<LastConsecutiveStore+1; ++i) { 8852243830Sdim StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode); 8853243830Sdim LoadSDNode *Ld = dyn_cast<LoadSDNode>(St->getValue()); 8854243830Sdim if (!Ld) break; 8855243830Sdim 8856243830Sdim // Loads must only have one use. 8857243830Sdim if (!Ld->hasNUsesOfValue(1, 0)) 8858243830Sdim break; 8859243830Sdim 8860243830Sdim // Check that the alignment is the same as the stores. 8861243830Sdim if (Ld->getAlignment() != St->getAlignment()) 8862243830Sdim break; 8863243830Sdim 8864243830Sdim // The memory operands must not be volatile. 8865243830Sdim if (Ld->isVolatile() || Ld->isIndexed()) 8866243830Sdim break; 8867243830Sdim 8868243830Sdim // We do not accept ext loads. 8869243830Sdim if (Ld->getExtensionType() != ISD::NON_EXTLOAD) 8870243830Sdim break; 8871243830Sdim 8872243830Sdim // The stored memory type must be the same. 8873243830Sdim if (Ld->getMemoryVT() != MemVT) 8874243830Sdim break; 8875243830Sdim 8876249423Sdim BaseIndexOffset LdPtr = BaseIndexOffset::match(Ld->getBasePtr()); 8877243830Sdim // If this is not the first ptr that we check. 8878249423Sdim if (LdBasePtr.Base.getNode()) { 8879243830Sdim // The base ptr must be the same. 8880249423Sdim if (!LdPtr.equalBaseIndex(LdBasePtr)) 8881243830Sdim break; 8882243830Sdim } else { 8883243830Sdim // Check that all other base pointers are the same as this one. 8884249423Sdim LdBasePtr = LdPtr; 8885243830Sdim } 8886243830Sdim 8887243830Sdim // We found a potential memory operand to merge. 8888249423Sdim LoadNodes.push_back(MemOpLink(Ld, LdPtr.Offset, 0)); 8889243830Sdim } 8890243830Sdim 8891243830Sdim if (LoadNodes.size() < 2) 8892243830Sdim return false; 8893243830Sdim 8894243830Sdim // Scan the memory operations on the chain and find the first non-consecutive 8895243830Sdim // load memory address. These variables hold the index in the store node 8896243830Sdim // array. 8897243830Sdim unsigned LastConsecutiveLoad = 0; 8898243830Sdim // This variable refers to the size and not index in the array. 8899243830Sdim unsigned LastLegalVectorType = 0; 8900243830Sdim unsigned LastLegalIntegerType = 0; 8901243830Sdim StartAddress = LoadNodes[0].OffsetFromBase; 8902243830Sdim SDValue FirstChain = LoadNodes[0].MemNode->getChain(); 8903243830Sdim for (unsigned i = 1; i < LoadNodes.size(); ++i) { 8904243830Sdim // All loads much share the same chain. 8905243830Sdim if (LoadNodes[i].MemNode->getChain() != FirstChain) 8906243830Sdim break; 8907249423Sdim 8908243830Sdim int64_t CurrAddress = LoadNodes[i].OffsetFromBase; 8909243830Sdim if (CurrAddress - StartAddress != (ElementSizeBytes * i)) 8910243830Sdim break; 8911243830Sdim LastConsecutiveLoad = i; 8912243830Sdim 8913243830Sdim // Find a legal type for the vector store. 8914243830Sdim EVT StoreTy = EVT::getVectorVT(*DAG.getContext(), MemVT, i+1); 8915243830Sdim if (TLI.isTypeLegal(StoreTy)) 8916243830Sdim LastLegalVectorType = i + 1; 8917243830Sdim 8918243830Sdim // Find a legal type for the integer store. 8919243830Sdim unsigned StoreBW = (i+1) * ElementSizeBytes * 8; 8920243830Sdim StoreTy = EVT::getIntegerVT(*DAG.getContext(), StoreBW); 8921243830Sdim if (TLI.isTypeLegal(StoreTy)) 8922243830Sdim LastLegalIntegerType = i + 1; 8923249423Sdim // Or check whether a truncstore and extload is legal. 8924249423Sdim else if (TLI.getTypeAction(*DAG.getContext(), StoreTy) == 8925249423Sdim TargetLowering::TypePromoteInteger) { 8926249423Sdim EVT LegalizedStoredValueTy = 8927249423Sdim TLI.getTypeToTransformTo(*DAG.getContext(), StoreTy); 8928249423Sdim if (TLI.isTruncStoreLegal(LegalizedStoredValueTy, StoreTy) && 8929249423Sdim TLI.isLoadExtLegal(ISD::ZEXTLOAD, StoreTy) && 8930249423Sdim TLI.isLoadExtLegal(ISD::SEXTLOAD, StoreTy) && 8931249423Sdim TLI.isLoadExtLegal(ISD::EXTLOAD, StoreTy)) 8932249423Sdim LastLegalIntegerType = i+1; 8933249423Sdim } 8934243830Sdim } 8935243830Sdim 8936243830Sdim // Only use vector types if the vector type is larger than the integer type. 8937243830Sdim // If they are the same, use integers. 8938249423Sdim bool UseVectorTy = LastLegalVectorType > LastLegalIntegerType && !NoVectors; 8939243830Sdim unsigned LastLegalType = std::max(LastLegalVectorType, LastLegalIntegerType); 8940243830Sdim 8941243830Sdim // We add +1 here because the LastXXX variables refer to location while 8942243830Sdim // the NumElem refers to array/index size. 8943243830Sdim unsigned NumElem = std::min(LastConsecutiveStore, LastConsecutiveLoad) + 1; 8944243830Sdim NumElem = std::min(LastLegalType, NumElem); 8945243830Sdim 8946243830Sdim if (NumElem < 2) 8947243830Sdim return false; 8948243830Sdim 8949243830Sdim // The earliest Node in the DAG. 8950243830Sdim unsigned EarliestNodeUsed = 0; 8951243830Sdim LSBaseSDNode *EarliestOp = StoreNodes[EarliestNodeUsed].MemNode; 8952243830Sdim for (unsigned i=1; i<NumElem; ++i) { 8953243830Sdim // Find a chain for the new wide-store operand. Notice that some 8954243830Sdim // of the store nodes that we found may not be selected for inclusion 8955243830Sdim // in the wide store. The chain we use needs to be the chain of the 8956243830Sdim // earliest store node which is *used* and replaced by the wide store. 8957243830Sdim if (StoreNodes[i].SequenceNum > StoreNodes[EarliestNodeUsed].SequenceNum) 8958243830Sdim EarliestNodeUsed = i; 8959243830Sdim } 8960243830Sdim 8961243830Sdim // Find if it is better to use vectors or integers to load and store 8962243830Sdim // to memory. 8963243830Sdim EVT JointMemOpVT; 8964243830Sdim if (UseVectorTy) { 8965243830Sdim JointMemOpVT = EVT::getVectorVT(*DAG.getContext(), MemVT, NumElem); 8966243830Sdim } else { 8967243830Sdim unsigned StoreBW = NumElem * ElementSizeBytes * 8; 8968243830Sdim JointMemOpVT = EVT::getIntegerVT(*DAG.getContext(), StoreBW); 8969243830Sdim } 8970243830Sdim 8971263508Sdim SDLoc LoadDL(LoadNodes[0].MemNode); 8972263508Sdim SDLoc StoreDL(StoreNodes[0].MemNode); 8973243830Sdim 8974243830Sdim LoadSDNode *FirstLoad = cast<LoadSDNode>(LoadNodes[0].MemNode); 8975243830Sdim SDValue NewLoad = DAG.getLoad(JointMemOpVT, LoadDL, 8976243830Sdim FirstLoad->getChain(), 8977243830Sdim FirstLoad->getBasePtr(), 8978243830Sdim FirstLoad->getPointerInfo(), 8979243830Sdim false, false, false, 8980243830Sdim FirstLoad->getAlignment()); 8981243830Sdim 8982243830Sdim SDValue NewStore = DAG.getStore(EarliestOp->getChain(), StoreDL, NewLoad, 8983243830Sdim FirstInChain->getBasePtr(), 8984243830Sdim FirstInChain->getPointerInfo(), false, false, 8985243830Sdim FirstInChain->getAlignment()); 8986243830Sdim 8987243830Sdim // Replace one of the loads with the new load. 8988243830Sdim LoadSDNode *Ld = cast<LoadSDNode>(LoadNodes[0].MemNode); 8989243830Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), 8990243830Sdim SDValue(NewLoad.getNode(), 1)); 8991243830Sdim 8992243830Sdim // Remove the rest of the load chains. 8993243830Sdim for (unsigned i = 1; i < NumElem ; ++i) { 8994243830Sdim // Replace all chain users of the old load nodes with the chain of the new 8995243830Sdim // load node. 8996243830Sdim LoadSDNode *Ld = cast<LoadSDNode>(LoadNodes[i].MemNode); 8997243830Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), Ld->getChain()); 8998243830Sdim } 8999243830Sdim 9000243830Sdim // Replace the first store with the new store. 9001243830Sdim CombineTo(EarliestOp, NewStore); 9002243830Sdim // Erase all other stores. 9003243830Sdim for (unsigned i = 0; i < NumElem ; ++i) { 9004243830Sdim // Remove all Store nodes. 9005243830Sdim if (StoreNodes[i].MemNode == EarliestOp) 9006243830Sdim continue; 9007243830Sdim StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode); 9008243830Sdim DAG.ReplaceAllUsesOfValueWith(SDValue(St, 0), St->getChain()); 9009243830Sdim removeFromWorkList(St); 9010243830Sdim DAG.DeleteNode(St); 9011243830Sdim } 9012243830Sdim 9013243830Sdim return true; 9014243830Sdim} 9015243830Sdim 9016193323SedSDValue DAGCombiner::visitSTORE(SDNode *N) { 9017193323Sed StoreSDNode *ST = cast<StoreSDNode>(N); 9018193323Sed SDValue Chain = ST->getChain(); 9019193323Sed SDValue Value = ST->getValue(); 9020193323Sed SDValue Ptr = ST->getBasePtr(); 9021193323Sed 9022193323Sed // If this is a store of a bit convert, store the input value if the 9023193323Sed // resultant store does not need a higher alignment than the original. 9024218893Sdim if (Value.getOpcode() == ISD::BITCAST && !ST->isTruncatingStore() && 9025193323Sed ST->isUnindexed()) { 9026193323Sed unsigned OrigAlign = ST->getAlignment(); 9027198090Srdivacky EVT SVT = Value.getOperand(0).getValueType(); 9028243830Sdim unsigned Align = TLI.getDataLayout()-> 9029198090Srdivacky getABITypeAlignment(SVT.getTypeForEVT(*DAG.getContext())); 9030193323Sed if (Align <= OrigAlign && 9031193323Sed ((!LegalOperations && !ST->isVolatile()) || 9032193323Sed TLI.isOperationLegalOrCustom(ISD::STORE, SVT))) 9033263508Sdim return DAG.getStore(Chain, SDLoc(N), Value.getOperand(0), 9034218893Sdim Ptr, ST->getPointerInfo(), ST->isVolatile(), 9035263508Sdim ST->isNonTemporal(), OrigAlign, 9036263508Sdim ST->getTBAAInfo()); 9037193323Sed } 9038193323Sed 9039221345Sdim // Turn 'store undef, Ptr' -> nothing. 9040221345Sdim if (Value.getOpcode() == ISD::UNDEF && ST->isUnindexed()) 9041221345Sdim return Chain; 9042221345Sdim 9043193323Sed // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr' 9044193323Sed if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Value)) { 9045193323Sed // NOTE: If the original store is volatile, this transform must not increase 9046193323Sed // the number of stores. For example, on x86-32 an f64 can be stored in one 9047193323Sed // processor operation but an i64 (which is not legal) requires two. So the 9048193323Sed // transform should not be done in this case. 9049193323Sed if (Value.getOpcode() != ISD::TargetConstantFP) { 9050193323Sed SDValue Tmp; 9051263508Sdim switch (CFP->getSimpleValueType(0).SimpleTy) { 9052198090Srdivacky default: llvm_unreachable("Unknown FP type"); 9053239462Sdim case MVT::f16: // We don't do this for these yet. 9054239462Sdim case MVT::f80: 9055193323Sed case MVT::f128: 9056193323Sed case MVT::ppcf128: 9057193323Sed break; 9058193323Sed case MVT::f32: 9059207618Srdivacky if ((isTypeLegal(MVT::i32) && !LegalOperations && !ST->isVolatile()) || 9060193323Sed TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32)) { 9061193323Sed Tmp = DAG.getConstant((uint32_t)CFP->getValueAPF(). 9062193323Sed bitcastToAPInt().getZExtValue(), MVT::i32); 9063263508Sdim return DAG.getStore(Chain, SDLoc(N), Tmp, 9064263508Sdim Ptr, ST->getMemOperand()); 9065193323Sed } 9066193323Sed break; 9067193323Sed case MVT::f64: 9068207618Srdivacky if ((TLI.isTypeLegal(MVT::i64) && !LegalOperations && 9069193323Sed !ST->isVolatile()) || 9070193323Sed TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i64)) { 9071193323Sed Tmp = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). 9072193323Sed getZExtValue(), MVT::i64); 9073263508Sdim return DAG.getStore(Chain, SDLoc(N), Tmp, 9074263508Sdim Ptr, ST->getMemOperand()); 9075221345Sdim } 9076221345Sdim 9077221345Sdim if (!ST->isVolatile() && 9078221345Sdim TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32)) { 9079193323Sed // Many FP stores are not made apparent until after legalize, e.g. for 9080193323Sed // argument passing. Since this is so common, custom legalize the 9081193323Sed // 64-bit integer store into two 32-bit stores. 9082193323Sed uint64_t Val = CFP->getValueAPF().bitcastToAPInt().getZExtValue(); 9083193323Sed SDValue Lo = DAG.getConstant(Val & 0xFFFFFFFF, MVT::i32); 9084193323Sed SDValue Hi = DAG.getConstant(Val >> 32, MVT::i32); 9085193323Sed if (TLI.isBigEndian()) std::swap(Lo, Hi); 9086193323Sed 9087193323Sed unsigned Alignment = ST->getAlignment(); 9088193323Sed bool isVolatile = ST->isVolatile(); 9089203954Srdivacky bool isNonTemporal = ST->isNonTemporal(); 9090263508Sdim const MDNode *TBAAInfo = ST->getTBAAInfo(); 9091193323Sed 9092263508Sdim SDValue St0 = DAG.getStore(Chain, SDLoc(ST), Lo, 9093218893Sdim Ptr, ST->getPointerInfo(), 9094203954Srdivacky isVolatile, isNonTemporal, 9095263508Sdim ST->getAlignment(), TBAAInfo); 9096263508Sdim Ptr = DAG.getNode(ISD::ADD, SDLoc(N), Ptr.getValueType(), Ptr, 9097193323Sed DAG.getConstant(4, Ptr.getValueType())); 9098193323Sed Alignment = MinAlign(Alignment, 4U); 9099263508Sdim SDValue St1 = DAG.getStore(Chain, SDLoc(ST), Hi, 9100218893Sdim Ptr, ST->getPointerInfo().getWithOffset(4), 9101218893Sdim isVolatile, isNonTemporal, 9102263508Sdim Alignment, TBAAInfo); 9103263508Sdim return DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, 9104193323Sed St0, St1); 9105193323Sed } 9106193323Sed 9107193323Sed break; 9108193323Sed } 9109193323Sed } 9110193323Sed } 9111193323Sed 9112206083Srdivacky // Try to infer better alignment information than the store already has. 9113206083Srdivacky if (OptLevel != CodeGenOpt::None && ST->isUnindexed()) { 9114206083Srdivacky if (unsigned Align = DAG.InferPtrAlignment(Ptr)) { 9115206083Srdivacky if (Align > ST->getAlignment()) 9116263508Sdim return DAG.getTruncStore(Chain, SDLoc(N), Value, 9117218893Sdim Ptr, ST->getPointerInfo(), ST->getMemoryVT(), 9118263508Sdim ST->isVolatile(), ST->isNonTemporal(), Align, 9119263508Sdim ST->getTBAAInfo()); 9120206083Srdivacky } 9121206083Srdivacky } 9122206083Srdivacky 9123218893Sdim // Try transforming a pair floating point load / store ops to integer 9124218893Sdim // load / store ops. 9125218893Sdim SDValue NewST = TransformFPLoadStorePair(N); 9126218893Sdim if (NewST.getNode()) 9127218893Sdim return NewST; 9128218893Sdim 9129263508Sdim bool UseAA = CombinerAA.getNumOccurrences() > 0 ? CombinerAA : 9130263508Sdim TLI.getTargetMachine().getSubtarget<TargetSubtargetInfo>().useAA(); 9131263508Sdim if (UseAA) { 9132193323Sed // Walk up chain skipping non-aliasing memory nodes. 9133193323Sed SDValue BetterChain = FindBetterChain(N, Chain); 9134193323Sed 9135193323Sed // If there is a better chain. 9136193323Sed if (Chain != BetterChain) { 9137198090Srdivacky SDValue ReplStore; 9138198090Srdivacky 9139193323Sed // Replace the chain to avoid dependency. 9140193323Sed if (ST->isTruncatingStore()) { 9141263508Sdim ReplStore = DAG.getTruncStore(BetterChain, SDLoc(N), Value, Ptr, 9142263508Sdim ST->getMemoryVT(), ST->getMemOperand()); 9143193323Sed } else { 9144263508Sdim ReplStore = DAG.getStore(BetterChain, SDLoc(N), Value, Ptr, 9145263508Sdim ST->getMemOperand()); 9146193323Sed } 9147193323Sed 9148193323Sed // Create token to keep both nodes around. 9149263508Sdim SDValue Token = DAG.getNode(ISD::TokenFactor, SDLoc(N), 9150193323Sed MVT::Other, Chain, ReplStore); 9151193323Sed 9152198090Srdivacky // Make sure the new and old chains are cleaned up. 9153198090Srdivacky AddToWorkList(Token.getNode()); 9154198090Srdivacky 9155193323Sed // Don't add users to work list. 9156193323Sed return CombineTo(N, Token, false); 9157193323Sed } 9158193323Sed } 9159193323Sed 9160193323Sed // Try transforming N to an indexed store. 9161193323Sed if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N)) 9162193323Sed return SDValue(N, 0); 9163193323Sed 9164193323Sed // FIXME: is there such a thing as a truncating indexed store? 9165193323Sed if (ST->isTruncatingStore() && ST->isUnindexed() && 9166193323Sed Value.getValueType().isInteger()) { 9167193323Sed // See if we can simplify the input to this truncstore with knowledge that 9168193323Sed // only the low bits are being used. For example: 9169193323Sed // "truncstore (or (shl x, 8), y), i8" -> "truncstore y, i8" 9170193323Sed SDValue Shorter = 9171193323Sed GetDemandedBits(Value, 9172224145Sdim APInt::getLowBitsSet( 9173224145Sdim Value.getValueType().getScalarType().getSizeInBits(), 9174224145Sdim ST->getMemoryVT().getScalarType().getSizeInBits())); 9175193323Sed AddToWorkList(Value.getNode()); 9176193323Sed if (Shorter.getNode()) 9177263508Sdim return DAG.getTruncStore(Chain, SDLoc(N), Shorter, 9178263508Sdim Ptr, ST->getMemoryVT(), ST->getMemOperand()); 9179193323Sed 9180193323Sed // Otherwise, see if we can simplify the operation with 9181193323Sed // SimplifyDemandedBits, which only works if the value has a single use. 9182193323Sed if (SimplifyDemandedBits(Value, 9183218893Sdim APInt::getLowBitsSet( 9184218893Sdim Value.getValueType().getScalarType().getSizeInBits(), 9185218893Sdim ST->getMemoryVT().getScalarType().getSizeInBits()))) 9186193323Sed return SDValue(N, 0); 9187193323Sed } 9188193323Sed 9189193323Sed // If this is a load followed by a store to the same location, then the store 9190193323Sed // is dead/noop. 9191193323Sed if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Value)) { 9192193323Sed if (Ld->getBasePtr() == Ptr && ST->getMemoryVT() == Ld->getMemoryVT() && 9193193323Sed ST->isUnindexed() && !ST->isVolatile() && 9194193323Sed // There can't be any side effects between the load and store, such as 9195193323Sed // a call or store. 9196193323Sed Chain.reachesChainWithoutSideEffects(SDValue(Ld, 1))) { 9197193323Sed // The store is dead, remove it. 9198193323Sed return Chain; 9199193323Sed } 9200193323Sed } 9201193323Sed 9202193323Sed // If this is an FP_ROUND or TRUNC followed by a store, fold this into a 9203193323Sed // truncating store. We can do this even if this is already a truncstore. 9204193323Sed if ((Value.getOpcode() == ISD::FP_ROUND || Value.getOpcode() == ISD::TRUNCATE) 9205193323Sed && Value.getNode()->hasOneUse() && ST->isUnindexed() && 9206193323Sed TLI.isTruncStoreLegal(Value.getOperand(0).getValueType(), 9207193323Sed ST->getMemoryVT())) { 9208263508Sdim return DAG.getTruncStore(Chain, SDLoc(N), Value.getOperand(0), 9209263508Sdim Ptr, ST->getMemoryVT(), ST->getMemOperand()); 9210193323Sed } 9211193323Sed 9212243830Sdim // Only perform this optimization before the types are legal, because we 9213243830Sdim // don't want to perform this optimization on every DAGCombine invocation. 9214249423Sdim if (!LegalTypes) { 9215249423Sdim bool EverChanged = false; 9216243830Sdim 9217249423Sdim do { 9218249423Sdim // There can be multiple store sequences on the same chain. 9219249423Sdim // Keep trying to merge store sequences until we are unable to do so 9220249423Sdim // or until we merge the last store on the chain. 9221249423Sdim bool Changed = MergeConsecutiveStores(ST); 9222249423Sdim EverChanged |= Changed; 9223249423Sdim if (!Changed) break; 9224249423Sdim } while (ST->getOpcode() != ISD::DELETED_NODE); 9225249423Sdim 9226249423Sdim if (EverChanged) 9227249423Sdim return SDValue(N, 0); 9228249423Sdim } 9229249423Sdim 9230193323Sed return ReduceLoadOpStoreWidth(N); 9231193323Sed} 9232193323Sed 9233193323SedSDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) { 9234193323Sed SDValue InVec = N->getOperand(0); 9235193323Sed SDValue InVal = N->getOperand(1); 9236193323Sed SDValue EltNo = N->getOperand(2); 9237263508Sdim SDLoc dl(N); 9238193323Sed 9239208599Srdivacky // If the inserted element is an UNDEF, just use the input vector. 9240208599Srdivacky if (InVal.getOpcode() == ISD::UNDEF) 9241208599Srdivacky return InVec; 9242208599Srdivacky 9243218893Sdim EVT VT = InVec.getValueType(); 9244218893Sdim 9245219077Sdim // If we can't generate a legal BUILD_VECTOR, exit 9246218893Sdim if (LegalOperations && !TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) 9247218893Sdim return SDValue(); 9248218893Sdim 9249226633Sdim // Check that we know which element is being inserted 9250226633Sdim if (!isa<ConstantSDNode>(EltNo)) 9251226633Sdim return SDValue(); 9252226633Sdim unsigned Elt = cast<ConstantSDNode>(EltNo)->getZExtValue(); 9253226633Sdim 9254226633Sdim // Check that the operand is a BUILD_VECTOR (or UNDEF, which can essentially 9255226633Sdim // be converted to a BUILD_VECTOR). Fill in the Ops vector with the 9256226633Sdim // vector elements. 9257226633Sdim SmallVector<SDValue, 8> Ops; 9258263508Sdim // Do not combine these two vectors if the output vector will not replace 9259263508Sdim // the input vector. 9260263508Sdim if (InVec.getOpcode() == ISD::BUILD_VECTOR && InVec.hasOneUse()) { 9261226633Sdim Ops.append(InVec.getNode()->op_begin(), 9262226633Sdim InVec.getNode()->op_end()); 9263226633Sdim } else if (InVec.getOpcode() == ISD::UNDEF) { 9264226633Sdim unsigned NElts = VT.getVectorNumElements(); 9265226633Sdim Ops.append(NElts, DAG.getUNDEF(InVal.getValueType())); 9266226633Sdim } else { 9267226633Sdim return SDValue(); 9268193323Sed } 9269193323Sed 9270226633Sdim // Insert the element 9271226633Sdim if (Elt < Ops.size()) { 9272226633Sdim // All the operands of BUILD_VECTOR must have the same type; 9273226633Sdim // we enforce that here. 9274226633Sdim EVT OpVT = Ops[0].getValueType(); 9275226633Sdim if (InVal.getValueType() != OpVT) 9276226633Sdim InVal = OpVT.bitsGT(InVal.getValueType()) ? 9277226633Sdim DAG.getNode(ISD::ANY_EXTEND, dl, OpVT, InVal) : 9278226633Sdim DAG.getNode(ISD::TRUNCATE, dl, OpVT, InVal); 9279226633Sdim Ops[Elt] = InVal; 9280193323Sed } 9281226633Sdim 9282226633Sdim // Return the new vector 9283226633Sdim return DAG.getNode(ISD::BUILD_VECTOR, dl, 9284226633Sdim VT, &Ops[0], Ops.size()); 9285193323Sed} 9286193323Sed 9287193323SedSDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) { 9288193323Sed // (vextract (scalar_to_vector val, 0) -> val 9289193323Sed SDValue InVec = N->getOperand(0); 9290234353Sdim EVT VT = InVec.getValueType(); 9291234353Sdim EVT NVT = N->getValueType(0); 9292193323Sed 9293223017Sdim if (InVec.getOpcode() == ISD::SCALAR_TO_VECTOR) { 9294223017Sdim // Check if the result type doesn't match the inserted element type. A 9295223017Sdim // SCALAR_TO_VECTOR may truncate the inserted element and the 9296223017Sdim // EXTRACT_VECTOR_ELT may widen the extracted vector. 9297223017Sdim SDValue InOp = InVec.getOperand(0); 9298223017Sdim if (InOp.getValueType() != NVT) { 9299223017Sdim assert(InOp.getValueType().isInteger() && NVT.isInteger()); 9300263508Sdim return DAG.getSExtOrTrunc(InOp, SDLoc(InVec), NVT); 9301223017Sdim } 9302223017Sdim return InOp; 9303223017Sdim } 9304193323Sed 9305234353Sdim SDValue EltNo = N->getOperand(1); 9306234353Sdim bool ConstEltNo = isa<ConstantSDNode>(EltNo); 9307234353Sdim 9308234353Sdim // Transform: (EXTRACT_VECTOR_ELT( VECTOR_SHUFFLE )) -> EXTRACT_VECTOR_ELT. 9309234353Sdim // We only perform this optimization before the op legalization phase because 9310243830Sdim // we may introduce new vector instructions which are not backed by TD 9311243830Sdim // patterns. For example on AVX, extracting elements from a wide vector 9312243830Sdim // without using extract_subvector. 9313234353Sdim if (InVec.getOpcode() == ISD::VECTOR_SHUFFLE 9314234353Sdim && ConstEltNo && !LegalOperations) { 9315234353Sdim int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue(); 9316234353Sdim int NumElem = VT.getVectorNumElements(); 9317234353Sdim ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(InVec); 9318234353Sdim // Find the new index to extract from. 9319234353Sdim int OrigElt = SVOp->getMaskElt(Elt); 9320234353Sdim 9321234353Sdim // Extracting an undef index is undef. 9322234353Sdim if (OrigElt == -1) 9323234353Sdim return DAG.getUNDEF(NVT); 9324234353Sdim 9325234353Sdim // Select the right vector half to extract from. 9326234353Sdim if (OrigElt < NumElem) { 9327234353Sdim InVec = InVec->getOperand(0); 9328234353Sdim } else { 9329234353Sdim InVec = InVec->getOperand(1); 9330234353Sdim OrigElt -= NumElem; 9331234353Sdim } 9332234353Sdim 9333263508Sdim EVT IndexTy = TLI.getVectorIdxTy(); 9334263508Sdim return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), NVT, 9335239462Sdim InVec, DAG.getConstant(OrigElt, IndexTy)); 9336234353Sdim } 9337234353Sdim 9338193323Sed // Perform only after legalization to ensure build_vector / vector_shuffle 9339193323Sed // optimizations have already been done. 9340193323Sed if (!LegalOperations) return SDValue(); 9341193323Sed 9342193323Sed // (vextract (v4f32 load $addr), c) -> (f32 load $addr+c*size) 9343193323Sed // (vextract (v4f32 s2v (f32 load $addr)), c) -> (f32 load $addr+c*size) 9344193323Sed // (vextract (v4f32 shuffle (load $addr), <1,u,u,u>), 0) -> (f32 load $addr) 9345193323Sed 9346234353Sdim if (ConstEltNo) { 9347218893Sdim int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue(); 9348193323Sed bool NewLoad = false; 9349193323Sed bool BCNumEltsChanged = false; 9350198090Srdivacky EVT ExtVT = VT.getVectorElementType(); 9351198090Srdivacky EVT LVT = ExtVT; 9352193323Sed 9353234353Sdim // If the result of load has to be truncated, then it's not necessarily 9354234353Sdim // profitable. 9355234353Sdim if (NVT.bitsLT(LVT) && !TLI.isTruncateFree(LVT, NVT)) 9356234353Sdim return SDValue(); 9357234353Sdim 9358218893Sdim if (InVec.getOpcode() == ISD::BITCAST) { 9359234353Sdim // Don't duplicate a load with other uses. 9360234353Sdim if (!InVec.hasOneUse()) 9361234353Sdim return SDValue(); 9362234353Sdim 9363198090Srdivacky EVT BCVT = InVec.getOperand(0).getValueType(); 9364198090Srdivacky if (!BCVT.isVector() || ExtVT.bitsGT(BCVT.getVectorElementType())) 9365193323Sed return SDValue(); 9366193323Sed if (VT.getVectorNumElements() != BCVT.getVectorNumElements()) 9367193323Sed BCNumEltsChanged = true; 9368193323Sed InVec = InVec.getOperand(0); 9369198090Srdivacky ExtVT = BCVT.getVectorElementType(); 9370193323Sed NewLoad = true; 9371193323Sed } 9372193323Sed 9373193323Sed LoadSDNode *LN0 = NULL; 9374193323Sed const ShuffleVectorSDNode *SVN = NULL; 9375193323Sed if (ISD::isNormalLoad(InVec.getNode())) { 9376193323Sed LN0 = cast<LoadSDNode>(InVec); 9377193323Sed } else if (InVec.getOpcode() == ISD::SCALAR_TO_VECTOR && 9378198090Srdivacky InVec.getOperand(0).getValueType() == ExtVT && 9379193323Sed ISD::isNormalLoad(InVec.getOperand(0).getNode())) { 9380234353Sdim // Don't duplicate a load with other uses. 9381234353Sdim if (!InVec.hasOneUse()) 9382234353Sdim return SDValue(); 9383234353Sdim 9384193323Sed LN0 = cast<LoadSDNode>(InVec.getOperand(0)); 9385193323Sed } else if ((SVN = dyn_cast<ShuffleVectorSDNode>(InVec))) { 9386193323Sed // (vextract (vector_shuffle (load $addr), v2, <1, u, u, u>), 1) 9387193323Sed // => 9388193323Sed // (load $addr+1*size) 9389193323Sed 9390234353Sdim // Don't duplicate a load with other uses. 9391234353Sdim if (!InVec.hasOneUse()) 9392234353Sdim return SDValue(); 9393234353Sdim 9394193323Sed // If the bit convert changed the number of elements, it is unsafe 9395193323Sed // to examine the mask. 9396193323Sed if (BCNumEltsChanged) 9397193323Sed return SDValue(); 9398193323Sed 9399193323Sed // Select the input vector, guarding against out of range extract vector. 9400193323Sed unsigned NumElems = VT.getVectorNumElements(); 9401218893Sdim int Idx = (Elt > (int)NumElems) ? -1 : SVN->getMaskElt(Elt); 9402193323Sed InVec = (Idx < (int)NumElems) ? InVec.getOperand(0) : InVec.getOperand(1); 9403193323Sed 9404234353Sdim if (InVec.getOpcode() == ISD::BITCAST) { 9405234353Sdim // Don't duplicate a load with other uses. 9406234353Sdim if (!InVec.hasOneUse()) 9407234353Sdim return SDValue(); 9408234353Sdim 9409193323Sed InVec = InVec.getOperand(0); 9410234353Sdim } 9411193323Sed if (ISD::isNormalLoad(InVec.getNode())) { 9412193323Sed LN0 = cast<LoadSDNode>(InVec); 9413207618Srdivacky Elt = (Idx < (int)NumElems) ? Idx : Idx - (int)NumElems; 9414193323Sed } 9415193323Sed } 9416193323Sed 9417234353Sdim // Make sure we found a non-volatile load and the extractelement is 9418234353Sdim // the only use. 9419223017Sdim if (!LN0 || !LN0->hasNUsesOfValue(1,0) || LN0->isVolatile()) 9420193323Sed return SDValue(); 9421193323Sed 9422218893Sdim // If Idx was -1 above, Elt is going to be -1, so just return undef. 9423218893Sdim if (Elt == -1) 9424226633Sdim return DAG.getUNDEF(LVT); 9425218893Sdim 9426193323Sed unsigned Align = LN0->getAlignment(); 9427193323Sed if (NewLoad) { 9428193323Sed // Check the resultant load doesn't need a higher alignment than the 9429193323Sed // original load. 9430193323Sed unsigned NewAlign = 9431243830Sdim TLI.getDataLayout() 9432218893Sdim ->getABITypeAlignment(LVT.getTypeForEVT(*DAG.getContext())); 9433193323Sed 9434193323Sed if (NewAlign > Align || !TLI.isOperationLegalOrCustom(ISD::LOAD, LVT)) 9435193323Sed return SDValue(); 9436193323Sed 9437193323Sed Align = NewAlign; 9438193323Sed } 9439193323Sed 9440193323Sed SDValue NewPtr = LN0->getBasePtr(); 9441218893Sdim unsigned PtrOff = 0; 9442218893Sdim 9443193323Sed if (Elt) { 9444218893Sdim PtrOff = LVT.getSizeInBits() * Elt / 8; 9445198090Srdivacky EVT PtrType = NewPtr.getValueType(); 9446193323Sed if (TLI.isBigEndian()) 9447193323Sed PtrOff = VT.getSizeInBits() / 8 - PtrOff; 9448263508Sdim NewPtr = DAG.getNode(ISD::ADD, SDLoc(N), PtrType, NewPtr, 9449193323Sed DAG.getConstant(PtrOff, PtrType)); 9450193323Sed } 9451193323Sed 9452234353Sdim // The replacement we need to do here is a little tricky: we need to 9453234353Sdim // replace an extractelement of a load with a load. 9454234353Sdim // Use ReplaceAllUsesOfValuesWith to do the replacement. 9455234353Sdim // Note that this replacement assumes that the extractvalue is the only 9456234353Sdim // use of the load; that's okay because we don't want to perform this 9457234353Sdim // transformation in other cases anyway. 9458234353Sdim SDValue Load; 9459234353Sdim SDValue Chain; 9460234353Sdim if (NVT.bitsGT(LVT)) { 9461234353Sdim // If the result type of vextract is wider than the load, then issue an 9462234353Sdim // extending load instead. 9463234353Sdim ISD::LoadExtType ExtType = TLI.isLoadExtLegal(ISD::ZEXTLOAD, LVT) 9464234353Sdim ? ISD::ZEXTLOAD : ISD::EXTLOAD; 9465263508Sdim Load = DAG.getExtLoad(ExtType, SDLoc(N), NVT, LN0->getChain(), 9466234353Sdim NewPtr, LN0->getPointerInfo().getWithOffset(PtrOff), 9467263508Sdim LVT, LN0->isVolatile(), LN0->isNonTemporal(), 9468263508Sdim Align, LN0->getTBAAInfo()); 9469234353Sdim Chain = Load.getValue(1); 9470234353Sdim } else { 9471263508Sdim Load = DAG.getLoad(LVT, SDLoc(N), LN0->getChain(), NewPtr, 9472234353Sdim LN0->getPointerInfo().getWithOffset(PtrOff), 9473263508Sdim LN0->isVolatile(), LN0->isNonTemporal(), 9474263508Sdim LN0->isInvariant(), Align, LN0->getTBAAInfo()); 9475234353Sdim Chain = Load.getValue(1); 9476234353Sdim if (NVT.bitsLT(LVT)) 9477263508Sdim Load = DAG.getNode(ISD::TRUNCATE, SDLoc(N), NVT, Load); 9478234353Sdim else 9479263508Sdim Load = DAG.getNode(ISD::BITCAST, SDLoc(N), NVT, Load); 9480234353Sdim } 9481234353Sdim WorkListRemover DeadNodes(*this); 9482234353Sdim SDValue From[] = { SDValue(N, 0), SDValue(LN0,1) }; 9483234353Sdim SDValue To[] = { Load, Chain }; 9484239462Sdim DAG.ReplaceAllUsesOfValuesWith(From, To, 2); 9485234353Sdim // Since we're explcitly calling ReplaceAllUses, add the new node to the 9486234353Sdim // worklist explicitly as well. 9487234353Sdim AddToWorkList(Load.getNode()); 9488234353Sdim AddUsersToWorkList(Load.getNode()); // Add users too 9489234353Sdim // Make sure to revisit this node to clean it up; it will usually be dead. 9490234353Sdim AddToWorkList(N); 9491234353Sdim return SDValue(N, 0); 9492193323Sed } 9493193323Sed 9494193323Sed return SDValue(); 9495193323Sed} 9496193323Sed 9497243830Sdim// Simplify (build_vec (ext )) to (bitcast (build_vec )) 9498243830SdimSDValue DAGCombiner::reduceBuildVecExtToExtBuildVec(SDNode *N) { 9499243830Sdim // We perform this optimization post type-legalization because 9500243830Sdim // the type-legalizer often scalarizes integer-promoted vectors. 9501243830Sdim // Performing this optimization before may create bit-casts which 9502243830Sdim // will be type-legalized to complex code sequences. 9503243830Sdim // We perform this optimization only before the operation legalizer because we 9504243830Sdim // may introduce illegal operations. 9505243830Sdim if (Level != AfterLegalizeVectorOps && Level != AfterLegalizeTypes) 9506243830Sdim return SDValue(); 9507243830Sdim 9508193323Sed unsigned NumInScalars = N->getNumOperands(); 9509263508Sdim SDLoc dl(N); 9510198090Srdivacky EVT VT = N->getValueType(0); 9511239462Sdim 9512234353Sdim // Check to see if this is a BUILD_VECTOR of a bunch of values 9513234353Sdim // which come from any_extend or zero_extend nodes. If so, we can create 9514234353Sdim // a new BUILD_VECTOR using bit-casts which may enable other BUILD_VECTOR 9515234353Sdim // optimizations. We do not handle sign-extend because we can't fill the sign 9516234353Sdim // using shuffles. 9517234353Sdim EVT SourceType = MVT::Other; 9518234353Sdim bool AllAnyExt = true; 9519239462Sdim 9520234353Sdim for (unsigned i = 0; i != NumInScalars; ++i) { 9521234353Sdim SDValue In = N->getOperand(i); 9522234353Sdim // Ignore undef inputs. 9523234353Sdim if (In.getOpcode() == ISD::UNDEF) continue; 9524193323Sed 9525234353Sdim bool AnyExt = In.getOpcode() == ISD::ANY_EXTEND; 9526234353Sdim bool ZeroExt = In.getOpcode() == ISD::ZERO_EXTEND; 9527234353Sdim 9528234353Sdim // Abort if the element is not an extension. 9529234353Sdim if (!ZeroExt && !AnyExt) { 9530234353Sdim SourceType = MVT::Other; 9531234353Sdim break; 9532234353Sdim } 9533234353Sdim 9534234353Sdim // The input is a ZeroExt or AnyExt. Check the original type. 9535234353Sdim EVT InTy = In.getOperand(0).getValueType(); 9536234353Sdim 9537234353Sdim // Check that all of the widened source types are the same. 9538234353Sdim if (SourceType == MVT::Other) 9539234353Sdim // First time. 9540234353Sdim SourceType = InTy; 9541234353Sdim else if (InTy != SourceType) { 9542234353Sdim // Multiple income types. Abort. 9543234353Sdim SourceType = MVT::Other; 9544234353Sdim break; 9545234353Sdim } 9546234353Sdim 9547234353Sdim // Check if all of the extends are ANY_EXTENDs. 9548234353Sdim AllAnyExt &= AnyExt; 9549234353Sdim } 9550234353Sdim 9551234353Sdim // In order to have valid types, all of the inputs must be extended from the 9552234353Sdim // same source type and all of the inputs must be any or zero extend. 9553234353Sdim // Scalar sizes must be a power of two. 9554243830Sdim EVT OutScalarTy = VT.getScalarType(); 9555234353Sdim bool ValidTypes = SourceType != MVT::Other && 9556234353Sdim isPowerOf2_32(OutScalarTy.getSizeInBits()) && 9557234353Sdim isPowerOf2_32(SourceType.getSizeInBits()); 9558234353Sdim 9559234353Sdim // Create a new simpler BUILD_VECTOR sequence which other optimizations can 9560234353Sdim // turn into a single shuffle instruction. 9561243830Sdim if (!ValidTypes) 9562243830Sdim return SDValue(); 9563234353Sdim 9564243830Sdim bool isLE = TLI.isLittleEndian(); 9565243830Sdim unsigned ElemRatio = OutScalarTy.getSizeInBits()/SourceType.getSizeInBits(); 9566243830Sdim assert(ElemRatio > 1 && "Invalid element size ratio"); 9567243830Sdim SDValue Filler = AllAnyExt ? DAG.getUNDEF(SourceType): 9568243830Sdim DAG.getConstant(0, SourceType); 9569234353Sdim 9570243830Sdim unsigned NewBVElems = ElemRatio * VT.getVectorNumElements(); 9571243830Sdim SmallVector<SDValue, 8> Ops(NewBVElems, Filler); 9572234353Sdim 9573243830Sdim // Populate the new build_vector 9574243830Sdim for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 9575243830Sdim SDValue Cast = N->getOperand(i); 9576243830Sdim assert((Cast.getOpcode() == ISD::ANY_EXTEND || 9577243830Sdim Cast.getOpcode() == ISD::ZERO_EXTEND || 9578243830Sdim Cast.getOpcode() == ISD::UNDEF) && "Invalid cast opcode"); 9579243830Sdim SDValue In; 9580243830Sdim if (Cast.getOpcode() == ISD::UNDEF) 9581243830Sdim In = DAG.getUNDEF(SourceType); 9582243830Sdim else 9583243830Sdim In = Cast->getOperand(0); 9584243830Sdim unsigned Index = isLE ? (i * ElemRatio) : 9585243830Sdim (i * ElemRatio + (ElemRatio - 1)); 9586243830Sdim 9587243830Sdim assert(Index < Ops.size() && "Invalid index"); 9588243830Sdim Ops[Index] = In; 9589243830Sdim } 9590243830Sdim 9591243830Sdim // The type of the new BUILD_VECTOR node. 9592243830Sdim EVT VecVT = EVT::getVectorVT(*DAG.getContext(), SourceType, NewBVElems); 9593243830Sdim assert(VecVT.getSizeInBits() == VT.getSizeInBits() && 9594243830Sdim "Invalid vector size"); 9595243830Sdim // Check if the new vector type is legal. 9596243830Sdim if (!isTypeLegal(VecVT)) return SDValue(); 9597243830Sdim 9598243830Sdim // Make the new BUILD_VECTOR. 9599243830Sdim SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, dl, VecVT, &Ops[0], Ops.size()); 9600243830Sdim 9601243830Sdim // The new BUILD_VECTOR node has the potential to be further optimized. 9602243830Sdim AddToWorkList(BV.getNode()); 9603243830Sdim // Bitcast to the desired type. 9604243830Sdim return DAG.getNode(ISD::BITCAST, dl, VT, BV); 9605243830Sdim} 9606243830Sdim 9607243830SdimSDValue DAGCombiner::reduceBuildVecConvertToConvertBuildVec(SDNode *N) { 9608243830Sdim EVT VT = N->getValueType(0); 9609243830Sdim 9610243830Sdim unsigned NumInScalars = N->getNumOperands(); 9611263508Sdim SDLoc dl(N); 9612243830Sdim 9613243830Sdim EVT SrcVT = MVT::Other; 9614243830Sdim unsigned Opcode = ISD::DELETED_NODE; 9615243830Sdim unsigned NumDefs = 0; 9616243830Sdim 9617243830Sdim for (unsigned i = 0; i != NumInScalars; ++i) { 9618243830Sdim SDValue In = N->getOperand(i); 9619243830Sdim unsigned Opc = In.getOpcode(); 9620243830Sdim 9621243830Sdim if (Opc == ISD::UNDEF) 9622243830Sdim continue; 9623243830Sdim 9624243830Sdim // If all scalar values are floats and converted from integers. 9625243830Sdim if (Opcode == ISD::DELETED_NODE && 9626243830Sdim (Opc == ISD::UINT_TO_FP || Opc == ISD::SINT_TO_FP)) { 9627243830Sdim Opcode = Opc; 9628234353Sdim } 9629249423Sdim 9630243830Sdim if (Opc != Opcode) 9631243830Sdim return SDValue(); 9632234353Sdim 9633243830Sdim EVT InVT = In.getOperand(0).getValueType(); 9634234353Sdim 9635243830Sdim // If all scalar values are typed differently, bail out. It's chosen to 9636243830Sdim // simplify BUILD_VECTOR of integer types. 9637243830Sdim if (SrcVT == MVT::Other) 9638243830Sdim SrcVT = InVT; 9639243830Sdim if (SrcVT != InVT) 9640243830Sdim return SDValue(); 9641243830Sdim NumDefs++; 9642243830Sdim } 9643234353Sdim 9644243830Sdim // If the vector has just one element defined, it's not worth to fold it into 9645243830Sdim // a vectorized one. 9646243830Sdim if (NumDefs < 2) 9647243830Sdim return SDValue(); 9648243830Sdim 9649243830Sdim assert((Opcode == ISD::UINT_TO_FP || Opcode == ISD::SINT_TO_FP) 9650243830Sdim && "Should only handle conversion from integer to float."); 9651243830Sdim assert(SrcVT != MVT::Other && "Cannot determine source type!"); 9652243830Sdim 9653243830Sdim EVT NVT = EVT::getVectorVT(*DAG.getContext(), SrcVT, NumInScalars); 9654249423Sdim 9655249423Sdim if (!TLI.isOperationLegalOrCustom(Opcode, NVT)) 9656249423Sdim return SDValue(); 9657249423Sdim 9658243830Sdim SmallVector<SDValue, 8> Opnds; 9659243830Sdim for (unsigned i = 0; i != NumInScalars; ++i) { 9660243830Sdim SDValue In = N->getOperand(i); 9661243830Sdim 9662243830Sdim if (In.getOpcode() == ISD::UNDEF) 9663243830Sdim Opnds.push_back(DAG.getUNDEF(SrcVT)); 9664243830Sdim else 9665243830Sdim Opnds.push_back(In.getOperand(0)); 9666234353Sdim } 9667243830Sdim SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, 9668243830Sdim &Opnds[0], Opnds.size()); 9669243830Sdim AddToWorkList(BV.getNode()); 9670234353Sdim 9671243830Sdim return DAG.getNode(Opcode, dl, VT, BV); 9672243830Sdim} 9673243830Sdim 9674243830SdimSDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) { 9675243830Sdim unsigned NumInScalars = N->getNumOperands(); 9676263508Sdim SDLoc dl(N); 9677243830Sdim EVT VT = N->getValueType(0); 9678243830Sdim 9679243830Sdim // A vector built entirely of undefs is undef. 9680243830Sdim if (ISD::allOperandsUndef(N)) 9681243830Sdim return DAG.getUNDEF(VT); 9682243830Sdim 9683243830Sdim SDValue V = reduceBuildVecExtToExtBuildVec(N); 9684243830Sdim if (V.getNode()) 9685243830Sdim return V; 9686243830Sdim 9687243830Sdim V = reduceBuildVecConvertToConvertBuildVec(N); 9688243830Sdim if (V.getNode()) 9689243830Sdim return V; 9690243830Sdim 9691193323Sed // Check to see if this is a BUILD_VECTOR of a bunch of EXTRACT_VECTOR_ELT 9692193323Sed // operations. If so, and if the EXTRACT_VECTOR_ELT vector inputs come from 9693193323Sed // at most two distinct vectors, turn this into a shuffle node. 9694234353Sdim 9695234353Sdim // May only combine to shuffle after legalize if shuffle is legal. 9696234353Sdim if (LegalOperations && 9697234353Sdim !TLI.isOperationLegalOrCustom(ISD::VECTOR_SHUFFLE, VT)) 9698234353Sdim return SDValue(); 9699234353Sdim 9700193323Sed SDValue VecIn1, VecIn2; 9701193323Sed for (unsigned i = 0; i != NumInScalars; ++i) { 9702193323Sed // Ignore undef inputs. 9703193323Sed if (N->getOperand(i).getOpcode() == ISD::UNDEF) continue; 9704193323Sed 9705193323Sed // If this input is something other than a EXTRACT_VECTOR_ELT with a 9706193323Sed // constant index, bail out. 9707193323Sed if (N->getOperand(i).getOpcode() != ISD::EXTRACT_VECTOR_ELT || 9708193323Sed !isa<ConstantSDNode>(N->getOperand(i).getOperand(1))) { 9709193323Sed VecIn1 = VecIn2 = SDValue(0, 0); 9710193323Sed break; 9711193323Sed } 9712193323Sed 9713234353Sdim // We allow up to two distinct input vectors. 9714193323Sed SDValue ExtractedFromVec = N->getOperand(i).getOperand(0); 9715193323Sed if (ExtractedFromVec == VecIn1 || ExtractedFromVec == VecIn2) 9716193323Sed continue; 9717193323Sed 9718193323Sed if (VecIn1.getNode() == 0) { 9719193323Sed VecIn1 = ExtractedFromVec; 9720193323Sed } else if (VecIn2.getNode() == 0) { 9721193323Sed VecIn2 = ExtractedFromVec; 9722193323Sed } else { 9723193323Sed // Too many inputs. 9724193323Sed VecIn1 = VecIn2 = SDValue(0, 0); 9725193323Sed break; 9726193323Sed } 9727193323Sed } 9728193323Sed 9729234353Sdim // If everything is good, we can make a shuffle operation. 9730193323Sed if (VecIn1.getNode()) { 9731193323Sed SmallVector<int, 8> Mask; 9732193323Sed for (unsigned i = 0; i != NumInScalars; ++i) { 9733193323Sed if (N->getOperand(i).getOpcode() == ISD::UNDEF) { 9734193323Sed Mask.push_back(-1); 9735193323Sed continue; 9736193323Sed } 9737193323Sed 9738193323Sed // If extracting from the first vector, just use the index directly. 9739193323Sed SDValue Extract = N->getOperand(i); 9740193323Sed SDValue ExtVal = Extract.getOperand(1); 9741193323Sed if (Extract.getOperand(0) == VecIn1) { 9742193323Sed unsigned ExtIndex = cast<ConstantSDNode>(ExtVal)->getZExtValue(); 9743193323Sed if (ExtIndex > VT.getVectorNumElements()) 9744193323Sed return SDValue(); 9745218893Sdim 9746193323Sed Mask.push_back(ExtIndex); 9747193323Sed continue; 9748193323Sed } 9749193323Sed 9750193323Sed // Otherwise, use InIdx + VecSize 9751193323Sed unsigned Idx = cast<ConstantSDNode>(ExtVal)->getZExtValue(); 9752193323Sed Mask.push_back(Idx+NumInScalars); 9753193323Sed } 9754193323Sed 9755234353Sdim // We can't generate a shuffle node with mismatched input and output types. 9756234353Sdim // Attempt to transform a single input vector to the correct type. 9757234353Sdim if ((VT != VecIn1.getValueType())) { 9758234353Sdim // We don't support shuffeling between TWO values of different types. 9759234353Sdim if (VecIn2.getNode() != 0) 9760234353Sdim return SDValue(); 9761234353Sdim 9762234353Sdim // We only support widening of vectors which are half the size of the 9763234353Sdim // output registers. For example XMM->YMM widening on X86 with AVX. 9764234353Sdim if (VecIn1.getValueType().getSizeInBits()*2 != VT.getSizeInBits()) 9765234353Sdim return SDValue(); 9766234353Sdim 9767243830Sdim // If the input vector type has a different base type to the output 9768243830Sdim // vector type, bail out. 9769243830Sdim if (VecIn1.getValueType().getVectorElementType() != 9770243830Sdim VT.getVectorElementType()) 9771243830Sdim return SDValue(); 9772243830Sdim 9773234353Sdim // Widen the input vector by adding undef values. 9774243830Sdim VecIn1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, 9775234353Sdim VecIn1, DAG.getUNDEF(VecIn1.getValueType())); 9776234353Sdim } 9777234353Sdim 9778234353Sdim // If VecIn2 is unused then change it to undef. 9779234353Sdim VecIn2 = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT); 9780234353Sdim 9781243830Sdim // Check that we were able to transform all incoming values to the same 9782243830Sdim // type. 9783234353Sdim if (VecIn2.getValueType() != VecIn1.getValueType() || 9784234353Sdim VecIn1.getValueType() != VT) 9785234353Sdim return SDValue(); 9786234353Sdim 9787234353Sdim // Only type-legal BUILD_VECTOR nodes are converted to shuffle nodes. 9788207618Srdivacky if (!isTypeLegal(VT)) 9789193323Sed return SDValue(); 9790193323Sed 9791193323Sed // Return the new VECTOR_SHUFFLE node. 9792193323Sed SDValue Ops[2]; 9793193323Sed Ops[0] = VecIn1; 9794234353Sdim Ops[1] = VecIn2; 9795243830Sdim return DAG.getVectorShuffle(VT, dl, Ops[0], Ops[1], &Mask[0]); 9796193323Sed } 9797193323Sed 9798193323Sed return SDValue(); 9799193323Sed} 9800193323Sed 9801193323SedSDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) { 9802193323Sed // TODO: Check to see if this is a CONCAT_VECTORS of a bunch of 9803193323Sed // EXTRACT_SUBVECTOR operations. If so, and if the EXTRACT_SUBVECTOR vector 9804193323Sed // inputs come from at most two distinct vectors, turn this into a shuffle 9805193323Sed // node. 9806193323Sed 9807193323Sed // If we only have one input vector, we don't need to do any concatenation. 9808193323Sed if (N->getNumOperands() == 1) 9809193323Sed return N->getOperand(0); 9810193323Sed 9811239462Sdim // Check if all of the operands are undefs. 9812263508Sdim EVT VT = N->getValueType(0); 9813239462Sdim if (ISD::allOperandsUndef(N)) 9814263508Sdim return DAG.getUNDEF(VT); 9815239462Sdim 9816263508Sdim // Optimize concat_vectors where one of the vectors is undef. 9817263508Sdim if (N->getNumOperands() == 2 && 9818263508Sdim N->getOperand(1)->getOpcode() == ISD::UNDEF) { 9819263508Sdim SDValue In = N->getOperand(0); 9820263508Sdim assert(In.getValueType().isVector() && "Must concat vectors"); 9821263508Sdim 9822263508Sdim // Transform: concat_vectors(scalar, undef) -> scalar_to_vector(sclr). 9823263508Sdim if (In->getOpcode() == ISD::BITCAST && 9824263508Sdim !In->getOperand(0)->getValueType(0).isVector()) { 9825263508Sdim SDValue Scalar = In->getOperand(0); 9826263508Sdim EVT SclTy = Scalar->getValueType(0); 9827263508Sdim 9828263508Sdim if (!SclTy.isFloatingPoint() && !SclTy.isInteger()) 9829263508Sdim return SDValue(); 9830263508Sdim 9831263508Sdim EVT NVT = EVT::getVectorVT(*DAG.getContext(), SclTy, 9832263508Sdim VT.getSizeInBits() / SclTy.getSizeInBits()); 9833263508Sdim if (!TLI.isTypeLegal(NVT) || !TLI.isTypeLegal(Scalar.getValueType())) 9834263508Sdim return SDValue(); 9835263508Sdim 9836263508Sdim SDLoc dl = SDLoc(N); 9837263508Sdim SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NVT, Scalar); 9838263508Sdim return DAG.getNode(ISD::BITCAST, dl, VT, Res); 9839263508Sdim } 9840263508Sdim } 9841263508Sdim 9842251662Sdim // Type legalization of vectors and DAG canonicalization of SHUFFLE_VECTOR 9843251662Sdim // nodes often generate nop CONCAT_VECTOR nodes. 9844251662Sdim // Scan the CONCAT_VECTOR operands and look for a CONCAT operations that 9845251662Sdim // place the incoming vectors at the exact same location. 9846251662Sdim SDValue SingleSource = SDValue(); 9847251662Sdim unsigned PartNumElem = N->getOperand(0).getValueType().getVectorNumElements(); 9848251662Sdim 9849251662Sdim for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 9850251662Sdim SDValue Op = N->getOperand(i); 9851251662Sdim 9852251662Sdim if (Op.getOpcode() == ISD::UNDEF) 9853251662Sdim continue; 9854251662Sdim 9855251662Sdim // Check if this is the identity extract: 9856251662Sdim if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR) 9857251662Sdim return SDValue(); 9858251662Sdim 9859251662Sdim // Find the single incoming vector for the extract_subvector. 9860251662Sdim if (SingleSource.getNode()) { 9861251662Sdim if (Op.getOperand(0) != SingleSource) 9862251662Sdim return SDValue(); 9863251662Sdim } else { 9864251662Sdim SingleSource = Op.getOperand(0); 9865251662Sdim 9866251662Sdim // Check the source type is the same as the type of the result. 9867251662Sdim // If not, this concat may extend the vector, so we can not 9868251662Sdim // optimize it away. 9869251662Sdim if (SingleSource.getValueType() != N->getValueType(0)) 9870251662Sdim return SDValue(); 9871251662Sdim } 9872251662Sdim 9873251662Sdim unsigned IdentityIndex = i * PartNumElem; 9874251662Sdim ConstantSDNode *CS = dyn_cast<ConstantSDNode>(Op.getOperand(1)); 9875251662Sdim // The extract index must be constant. 9876251662Sdim if (!CS) 9877251662Sdim return SDValue(); 9878263508Sdim 9879251662Sdim // Check that we are reading from the identity index. 9880251662Sdim if (CS->getZExtValue() != IdentityIndex) 9881251662Sdim return SDValue(); 9882251662Sdim } 9883251662Sdim 9884251662Sdim if (SingleSource.getNode()) 9885251662Sdim return SingleSource; 9886263508Sdim 9887193323Sed return SDValue(); 9888193323Sed} 9889193323Sed 9890226633SdimSDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode* N) { 9891226633Sdim EVT NVT = N->getValueType(0); 9892226633Sdim SDValue V = N->getOperand(0); 9893226633Sdim 9894249423Sdim if (V->getOpcode() == ISD::CONCAT_VECTORS) { 9895249423Sdim // Combine: 9896249423Sdim // (extract_subvec (concat V1, V2, ...), i) 9897249423Sdim // Into: 9898249423Sdim // Vi if possible 9899263508Sdim // Only operand 0 is checked as 'concat' assumes all inputs of the same 9900263508Sdim // type. 9901249423Sdim if (V->getOperand(0).getValueType() != NVT) 9902249423Sdim return SDValue(); 9903249423Sdim unsigned Idx = dyn_cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 9904249423Sdim unsigned NumElems = NVT.getVectorNumElements(); 9905249423Sdim assert((Idx % NumElems) == 0 && 9906249423Sdim "IDX in concat is not a multiple of the result vector length."); 9907249423Sdim return V->getOperand(Idx / NumElems); 9908249423Sdim } 9909249423Sdim 9910249423Sdim // Skip bitcasting 9911249423Sdim if (V->getOpcode() == ISD::BITCAST) 9912249423Sdim V = V.getOperand(0); 9913249423Sdim 9914226633Sdim if (V->getOpcode() == ISD::INSERT_SUBVECTOR) { 9915263508Sdim SDLoc dl(N); 9916226633Sdim // Handle only simple case where vector being inserted and vector 9917226633Sdim // being extracted are of same type, and are half size of larger vectors. 9918226633Sdim EVT BigVT = V->getOperand(0).getValueType(); 9919226633Sdim EVT SmallVT = V->getOperand(1).getValueType(); 9920249423Sdim if (!NVT.bitsEq(SmallVT) || NVT.getSizeInBits()*2 != BigVT.getSizeInBits()) 9921226633Sdim return SDValue(); 9922226633Sdim 9923234353Sdim // Only handle cases where both indexes are constants with the same type. 9924243830Sdim ConstantSDNode *ExtIdx = dyn_cast<ConstantSDNode>(N->getOperand(1)); 9925243830Sdim ConstantSDNode *InsIdx = dyn_cast<ConstantSDNode>(V->getOperand(2)); 9926226633Sdim 9927234353Sdim if (InsIdx && ExtIdx && 9928234353Sdim InsIdx->getValueType(0).getSizeInBits() <= 64 && 9929234353Sdim ExtIdx->getValueType(0).getSizeInBits() <= 64) { 9930234353Sdim // Combine: 9931234353Sdim // (extract_subvec (insert_subvec V1, V2, InsIdx), ExtIdx) 9932234353Sdim // Into: 9933249423Sdim // indices are equal or bit offsets are equal => V1 9934234353Sdim // otherwise => (extract_subvec V1, ExtIdx) 9935249423Sdim if (InsIdx->getZExtValue() * SmallVT.getScalarType().getSizeInBits() == 9936249423Sdim ExtIdx->getZExtValue() * NVT.getScalarType().getSizeInBits()) 9937249423Sdim return DAG.getNode(ISD::BITCAST, dl, NVT, V->getOperand(1)); 9938249423Sdim return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, 9939249423Sdim DAG.getNode(ISD::BITCAST, dl, 9940249423Sdim N->getOperand(0).getValueType(), 9941249423Sdim V->getOperand(0)), N->getOperand(1)); 9942234353Sdim } 9943226633Sdim } 9944226633Sdim 9945226633Sdim return SDValue(); 9946226633Sdim} 9947226633Sdim 9948251662Sdim// Tries to turn a shuffle of two CONCAT_VECTORS into a single concat. 9949251662Sdimstatic SDValue partitionShuffleOfConcats(SDNode *N, SelectionDAG &DAG) { 9950251662Sdim EVT VT = N->getValueType(0); 9951251662Sdim unsigned NumElts = VT.getVectorNumElements(); 9952251662Sdim 9953251662Sdim SDValue N0 = N->getOperand(0); 9954251662Sdim SDValue N1 = N->getOperand(1); 9955251662Sdim ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N); 9956251662Sdim 9957251662Sdim SmallVector<SDValue, 4> Ops; 9958251662Sdim EVT ConcatVT = N0.getOperand(0).getValueType(); 9959251662Sdim unsigned NumElemsPerConcat = ConcatVT.getVectorNumElements(); 9960251662Sdim unsigned NumConcats = NumElts / NumElemsPerConcat; 9961251662Sdim 9962251662Sdim // Look at every vector that's inserted. We're looking for exact 9963251662Sdim // subvector-sized copies from a concatenated vector 9964251662Sdim for (unsigned I = 0; I != NumConcats; ++I) { 9965251662Sdim // Make sure we're dealing with a copy. 9966251662Sdim unsigned Begin = I * NumElemsPerConcat; 9967263508Sdim bool AllUndef = true, NoUndef = true; 9968263508Sdim for (unsigned J = Begin; J != Begin + NumElemsPerConcat; ++J) { 9969263508Sdim if (SVN->getMaskElt(J) >= 0) 9970263508Sdim AllUndef = false; 9971263508Sdim else 9972263508Sdim NoUndef = false; 9973263508Sdim } 9974251662Sdim 9975263508Sdim if (NoUndef) { 9976263508Sdim if (SVN->getMaskElt(Begin) % NumElemsPerConcat != 0) 9977251662Sdim return SDValue(); 9978263508Sdim 9979263508Sdim for (unsigned J = 1; J != NumElemsPerConcat; ++J) 9980263508Sdim if (SVN->getMaskElt(Begin + J - 1) + 1 != SVN->getMaskElt(Begin + J)) 9981263508Sdim return SDValue(); 9982263508Sdim 9983263508Sdim unsigned FirstElt = SVN->getMaskElt(Begin) / NumElemsPerConcat; 9984263508Sdim if (FirstElt < N0.getNumOperands()) 9985263508Sdim Ops.push_back(N0.getOperand(FirstElt)); 9986263508Sdim else 9987263508Sdim Ops.push_back(N1.getOperand(FirstElt - N0.getNumOperands())); 9988263508Sdim 9989263508Sdim } else if (AllUndef) { 9990263508Sdim Ops.push_back(DAG.getUNDEF(N0.getOperand(0).getValueType())); 9991263508Sdim } else { // Mixed with general masks and undefs, can't do optimization. 9992263508Sdim return SDValue(); 9993251662Sdim } 9994251662Sdim } 9995251662Sdim 9996263508Sdim return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops.data(), 9997251662Sdim Ops.size()); 9998251662Sdim} 9999251662Sdim 10000193323SedSDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { 10001198090Srdivacky EVT VT = N->getValueType(0); 10002193323Sed unsigned NumElts = VT.getVectorNumElements(); 10003193323Sed 10004193323Sed SDValue N0 = N->getOperand(0); 10005234353Sdim SDValue N1 = N->getOperand(1); 10006193323Sed 10007234353Sdim assert(N0.getValueType() == VT && "Vector shuffle must be normalized in DAG"); 10008193323Sed 10009234353Sdim // Canonicalize shuffle undef, undef -> undef 10010234353Sdim if (N0.getOpcode() == ISD::UNDEF && N1.getOpcode() == ISD::UNDEF) 10011234353Sdim return DAG.getUNDEF(VT); 10012193323Sed 10013234353Sdim ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N); 10014234353Sdim 10015234353Sdim // Canonicalize shuffle v, v -> v, undef 10016234353Sdim if (N0 == N1) { 10017234353Sdim SmallVector<int, 8> NewMask; 10018234353Sdim for (unsigned i = 0; i != NumElts; ++i) { 10019234353Sdim int Idx = SVN->getMaskElt(i); 10020234353Sdim if (Idx >= (int)NumElts) Idx -= NumElts; 10021234353Sdim NewMask.push_back(Idx); 10022234353Sdim } 10023263508Sdim return DAG.getVectorShuffle(VT, SDLoc(N), N0, DAG.getUNDEF(VT), 10024234353Sdim &NewMask[0]); 10025234353Sdim } 10026234353Sdim 10027234353Sdim // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask. 10028234353Sdim if (N0.getOpcode() == ISD::UNDEF) { 10029234353Sdim SmallVector<int, 8> NewMask; 10030234353Sdim for (unsigned i = 0; i != NumElts; ++i) { 10031234353Sdim int Idx = SVN->getMaskElt(i); 10032234353Sdim if (Idx >= 0) { 10033263508Sdim if (Idx >= (int)NumElts) 10034263508Sdim Idx -= NumElts; 10035234353Sdim else 10036263508Sdim Idx = -1; // remove reference to lhs 10037234353Sdim } 10038234353Sdim NewMask.push_back(Idx); 10039234353Sdim } 10040263508Sdim return DAG.getVectorShuffle(VT, SDLoc(N), N1, DAG.getUNDEF(VT), 10041234353Sdim &NewMask[0]); 10042234353Sdim } 10043234353Sdim 10044234353Sdim // Remove references to rhs if it is undef 10045234353Sdim if (N1.getOpcode() == ISD::UNDEF) { 10046234353Sdim bool Changed = false; 10047234353Sdim SmallVector<int, 8> NewMask; 10048234353Sdim for (unsigned i = 0; i != NumElts; ++i) { 10049234353Sdim int Idx = SVN->getMaskElt(i); 10050234353Sdim if (Idx >= (int)NumElts) { 10051234353Sdim Idx = -1; 10052234353Sdim Changed = true; 10053234353Sdim } 10054234353Sdim NewMask.push_back(Idx); 10055234353Sdim } 10056234353Sdim if (Changed) 10057263508Sdim return DAG.getVectorShuffle(VT, SDLoc(N), N0, N1, &NewMask[0]); 10058234353Sdim } 10059234353Sdim 10060218893Sdim // If it is a splat, check if the argument vector is another splat or a 10061218893Sdim // build_vector with all scalar elements the same. 10062218893Sdim if (SVN->isSplat() && SVN->getSplatIndex() < (int)NumElts) { 10063193323Sed SDNode *V = N0.getNode(); 10064193323Sed 10065193323Sed // If this is a bit convert that changes the element type of the vector but 10066193323Sed // not the number of vector elements, look through it. Be careful not to 10067193323Sed // look though conversions that change things like v4f32 to v2f64. 10068218893Sdim if (V->getOpcode() == ISD::BITCAST) { 10069193323Sed SDValue ConvInput = V->getOperand(0); 10070193323Sed if (ConvInput.getValueType().isVector() && 10071193323Sed ConvInput.getValueType().getVectorNumElements() == NumElts) 10072193323Sed V = ConvInput.getNode(); 10073193323Sed } 10074193323Sed 10075193323Sed if (V->getOpcode() == ISD::BUILD_VECTOR) { 10076218893Sdim assert(V->getNumOperands() == NumElts && 10077218893Sdim "BUILD_VECTOR has wrong number of operands"); 10078218893Sdim SDValue Base; 10079218893Sdim bool AllSame = true; 10080218893Sdim for (unsigned i = 0; i != NumElts; ++i) { 10081218893Sdim if (V->getOperand(i).getOpcode() != ISD::UNDEF) { 10082218893Sdim Base = V->getOperand(i); 10083218893Sdim break; 10084193323Sed } 10085218893Sdim } 10086218893Sdim // Splat of <u, u, u, u>, return <u, u, u, u> 10087218893Sdim if (!Base.getNode()) 10088218893Sdim return N0; 10089218893Sdim for (unsigned i = 0; i != NumElts; ++i) { 10090218893Sdim if (V->getOperand(i) != Base) { 10091218893Sdim AllSame = false; 10092218893Sdim break; 10093193323Sed } 10094193323Sed } 10095218893Sdim // Splat of <x, x, x, x>, return <x, x, x, x> 10096218893Sdim if (AllSame) 10097218893Sdim return N0; 10098193323Sed } 10099193323Sed } 10100234353Sdim 10101251662Sdim if (N0.getOpcode() == ISD::CONCAT_VECTORS && 10102251662Sdim Level < AfterLegalizeVectorOps && 10103251662Sdim (N1.getOpcode() == ISD::UNDEF || 10104251662Sdim (N1.getOpcode() == ISD::CONCAT_VECTORS && 10105251662Sdim N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType()))) { 10106251662Sdim SDValue V = partitionShuffleOfConcats(N, DAG); 10107251662Sdim 10108251662Sdim if (V.getNode()) 10109251662Sdim return V; 10110251662Sdim } 10111251662Sdim 10112234353Sdim // If this shuffle node is simply a swizzle of another shuffle node, 10113234353Sdim // and it reverses the swizzle of the previous shuffle then we can 10114234353Sdim // optimize shuffle(shuffle(x, undef), undef) -> x. 10115234353Sdim if (N0.getOpcode() == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG && 10116234353Sdim N1.getOpcode() == ISD::UNDEF) { 10117234353Sdim 10118234353Sdim ShuffleVectorSDNode *OtherSV = cast<ShuffleVectorSDNode>(N0); 10119234353Sdim 10120234353Sdim // Shuffle nodes can only reverse shuffles with a single non-undef value. 10121234353Sdim if (N0.getOperand(1).getOpcode() != ISD::UNDEF) 10122234353Sdim return SDValue(); 10123234353Sdim 10124234353Sdim // The incoming shuffle must be of the same type as the result of the 10125234353Sdim // current shuffle. 10126234353Sdim assert(OtherSV->getOperand(0).getValueType() == VT && 10127234353Sdim "Shuffle types don't match"); 10128234353Sdim 10129234353Sdim for (unsigned i = 0; i != NumElts; ++i) { 10130234353Sdim int Idx = SVN->getMaskElt(i); 10131234353Sdim assert(Idx < (int)NumElts && "Index references undef operand"); 10132234353Sdim // Next, this index comes from the first value, which is the incoming 10133234353Sdim // shuffle. Adopt the incoming index. 10134234353Sdim if (Idx >= 0) 10135234353Sdim Idx = OtherSV->getMaskElt(Idx); 10136234353Sdim 10137234353Sdim // The combined shuffle must map each index to itself. 10138234353Sdim if (Idx >= 0 && (unsigned)Idx != i) 10139234353Sdim return SDValue(); 10140234353Sdim } 10141234353Sdim 10142234353Sdim return OtherSV->getOperand(0); 10143234353Sdim } 10144234353Sdim 10145193323Sed return SDValue(); 10146193323Sed} 10147193323Sed 10148193323Sed/// XformToShuffleWithZero - Returns a vector_shuffle if it able to transform 10149193323Sed/// an AND to a vector_shuffle with the destination vector and a zero vector. 10150193323Sed/// e.g. AND V, <0xffffffff, 0, 0xffffffff, 0>. ==> 10151193323Sed/// vector_shuffle V, Zero, <0, 4, 2, 4> 10152193323SedSDValue DAGCombiner::XformToShuffleWithZero(SDNode *N) { 10153198090Srdivacky EVT VT = N->getValueType(0); 10154263508Sdim SDLoc dl(N); 10155193323Sed SDValue LHS = N->getOperand(0); 10156193323Sed SDValue RHS = N->getOperand(1); 10157193323Sed if (N->getOpcode() == ISD::AND) { 10158218893Sdim if (RHS.getOpcode() == ISD::BITCAST) 10159193323Sed RHS = RHS.getOperand(0); 10160193323Sed if (RHS.getOpcode() == ISD::BUILD_VECTOR) { 10161193323Sed SmallVector<int, 8> Indices; 10162193323Sed unsigned NumElts = RHS.getNumOperands(); 10163193323Sed for (unsigned i = 0; i != NumElts; ++i) { 10164193323Sed SDValue Elt = RHS.getOperand(i); 10165193323Sed if (!isa<ConstantSDNode>(Elt)) 10166193323Sed return SDValue(); 10167234353Sdim 10168234353Sdim if (cast<ConstantSDNode>(Elt)->isAllOnesValue()) 10169193323Sed Indices.push_back(i); 10170193323Sed else if (cast<ConstantSDNode>(Elt)->isNullValue()) 10171193323Sed Indices.push_back(NumElts); 10172193323Sed else 10173193323Sed return SDValue(); 10174193323Sed } 10175193323Sed 10176193323Sed // Let's see if the target supports this vector_shuffle. 10177198090Srdivacky EVT RVT = RHS.getValueType(); 10178193323Sed if (!TLI.isVectorClearMaskLegal(Indices, RVT)) 10179193323Sed return SDValue(); 10180193323Sed 10181193323Sed // Return the new VECTOR_SHUFFLE node. 10182198090Srdivacky EVT EltVT = RVT.getVectorElementType(); 10183193323Sed SmallVector<SDValue,8> ZeroOps(RVT.getVectorNumElements(), 10184198090Srdivacky DAG.getConstant(0, EltVT)); 10185263508Sdim SDValue Zero = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), 10186193323Sed RVT, &ZeroOps[0], ZeroOps.size()); 10187218893Sdim LHS = DAG.getNode(ISD::BITCAST, dl, RVT, LHS); 10188193323Sed SDValue Shuf = DAG.getVectorShuffle(RVT, dl, LHS, Zero, &Indices[0]); 10189218893Sdim return DAG.getNode(ISD::BITCAST, dl, VT, Shuf); 10190193323Sed } 10191193323Sed } 10192193323Sed 10193193323Sed return SDValue(); 10194193323Sed} 10195193323Sed 10196193323Sed/// SimplifyVBinOp - Visit a binary vector operation, like ADD. 10197193323SedSDValue DAGCombiner::SimplifyVBinOp(SDNode *N) { 10198218893Sdim assert(N->getValueType(0).isVector() && 10199218893Sdim "SimplifyVBinOp only works on vectors!"); 10200193323Sed 10201193323Sed SDValue LHS = N->getOperand(0); 10202193323Sed SDValue RHS = N->getOperand(1); 10203193323Sed SDValue Shuffle = XformToShuffleWithZero(N); 10204193323Sed if (Shuffle.getNode()) return Shuffle; 10205193323Sed 10206193323Sed // If the LHS and RHS are BUILD_VECTOR nodes, see if we can constant fold 10207193323Sed // this operation. 10208193323Sed if (LHS.getOpcode() == ISD::BUILD_VECTOR && 10209193323Sed RHS.getOpcode() == ISD::BUILD_VECTOR) { 10210193323Sed SmallVector<SDValue, 8> Ops; 10211193323Sed for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) { 10212193323Sed SDValue LHSOp = LHS.getOperand(i); 10213193323Sed SDValue RHSOp = RHS.getOperand(i); 10214193323Sed // If these two elements can't be folded, bail out. 10215193323Sed if ((LHSOp.getOpcode() != ISD::UNDEF && 10216193323Sed LHSOp.getOpcode() != ISD::Constant && 10217193323Sed LHSOp.getOpcode() != ISD::ConstantFP) || 10218193323Sed (RHSOp.getOpcode() != ISD::UNDEF && 10219193323Sed RHSOp.getOpcode() != ISD::Constant && 10220193323Sed RHSOp.getOpcode() != ISD::ConstantFP)) 10221193323Sed break; 10222193323Sed 10223193323Sed // Can't fold divide by zero. 10224193323Sed if (N->getOpcode() == ISD::SDIV || N->getOpcode() == ISD::UDIV || 10225193323Sed N->getOpcode() == ISD::FDIV) { 10226193323Sed if ((RHSOp.getOpcode() == ISD::Constant && 10227193323Sed cast<ConstantSDNode>(RHSOp.getNode())->isNullValue()) || 10228193323Sed (RHSOp.getOpcode() == ISD::ConstantFP && 10229193323Sed cast<ConstantFPSDNode>(RHSOp.getNode())->getValueAPF().isZero())) 10230193323Sed break; 10231193323Sed } 10232193323Sed 10233218893Sdim EVT VT = LHSOp.getValueType(); 10234234353Sdim EVT RVT = RHSOp.getValueType(); 10235234353Sdim if (RVT != VT) { 10236234353Sdim // Integer BUILD_VECTOR operands may have types larger than the element 10237234353Sdim // size (e.g., when the element type is not legal). Prior to type 10238234353Sdim // legalization, the types may not match between the two BUILD_VECTORS. 10239234353Sdim // Truncate one of the operands to make them match. 10240234353Sdim if (RVT.getSizeInBits() > VT.getSizeInBits()) { 10241263508Sdim RHSOp = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, RHSOp); 10242234353Sdim } else { 10243263508Sdim LHSOp = DAG.getNode(ISD::TRUNCATE, SDLoc(N), RVT, LHSOp); 10244234353Sdim VT = RVT; 10245234353Sdim } 10246234353Sdim } 10247263508Sdim SDValue FoldOp = DAG.getNode(N->getOpcode(), SDLoc(LHS), VT, 10248208599Srdivacky LHSOp, RHSOp); 10249208599Srdivacky if (FoldOp.getOpcode() != ISD::UNDEF && 10250208599Srdivacky FoldOp.getOpcode() != ISD::Constant && 10251208599Srdivacky FoldOp.getOpcode() != ISD::ConstantFP) 10252208599Srdivacky break; 10253208599Srdivacky Ops.push_back(FoldOp); 10254208599Srdivacky AddToWorkList(FoldOp.getNode()); 10255193323Sed } 10256193323Sed 10257218893Sdim if (Ops.size() == LHS.getNumOperands()) 10258263508Sdim return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), 10259218893Sdim LHS.getValueType(), &Ops[0], Ops.size()); 10260193323Sed } 10261193323Sed 10262193323Sed return SDValue(); 10263193323Sed} 10264193323Sed 10265243830Sdim/// SimplifyVUnaryOp - Visit a binary vector operation, like FABS/FNEG. 10266243830SdimSDValue DAGCombiner::SimplifyVUnaryOp(SDNode *N) { 10267243830Sdim assert(N->getValueType(0).isVector() && 10268243830Sdim "SimplifyVUnaryOp only works on vectors!"); 10269243830Sdim 10270243830Sdim SDValue N0 = N->getOperand(0); 10271243830Sdim 10272243830Sdim if (N0.getOpcode() != ISD::BUILD_VECTOR) 10273243830Sdim return SDValue(); 10274243830Sdim 10275243830Sdim // Operand is a BUILD_VECTOR node, see if we can constant fold it. 10276243830Sdim SmallVector<SDValue, 8> Ops; 10277243830Sdim for (unsigned i = 0, e = N0.getNumOperands(); i != e; ++i) { 10278243830Sdim SDValue Op = N0.getOperand(i); 10279243830Sdim if (Op.getOpcode() != ISD::UNDEF && 10280243830Sdim Op.getOpcode() != ISD::ConstantFP) 10281243830Sdim break; 10282243830Sdim EVT EltVT = Op.getValueType(); 10283263508Sdim SDValue FoldOp = DAG.getNode(N->getOpcode(), SDLoc(N0), EltVT, Op); 10284243830Sdim if (FoldOp.getOpcode() != ISD::UNDEF && 10285243830Sdim FoldOp.getOpcode() != ISD::ConstantFP) 10286243830Sdim break; 10287243830Sdim Ops.push_back(FoldOp); 10288243830Sdim AddToWorkList(FoldOp.getNode()); 10289243830Sdim } 10290243830Sdim 10291243830Sdim if (Ops.size() != N0.getNumOperands()) 10292243830Sdim return SDValue(); 10293243830Sdim 10294263508Sdim return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), 10295243830Sdim N0.getValueType(), &Ops[0], Ops.size()); 10296243830Sdim} 10297243830Sdim 10298263508SdimSDValue DAGCombiner::SimplifySelect(SDLoc DL, SDValue N0, 10299193323Sed SDValue N1, SDValue N2){ 10300193323Sed assert(N0.getOpcode() ==ISD::SETCC && "First argument must be a SetCC node!"); 10301193323Sed 10302193323Sed SDValue SCC = SimplifySelectCC(DL, N0.getOperand(0), N0.getOperand(1), N1, N2, 10303193323Sed cast<CondCodeSDNode>(N0.getOperand(2))->get()); 10304193323Sed 10305193323Sed // If we got a simplified select_cc node back from SimplifySelectCC, then 10306193323Sed // break it down into a new SETCC node, and a new SELECT node, and then return 10307193323Sed // the SELECT node, since we were called with a SELECT node. 10308193323Sed if (SCC.getNode()) { 10309193323Sed // Check to see if we got a select_cc back (to turn into setcc/select). 10310193323Sed // Otherwise, just return whatever node we got back, like fabs. 10311193323Sed if (SCC.getOpcode() == ISD::SELECT_CC) { 10312263508Sdim SDValue SETCC = DAG.getNode(ISD::SETCC, SDLoc(N0), 10313193323Sed N0.getValueType(), 10314193323Sed SCC.getOperand(0), SCC.getOperand(1), 10315193323Sed SCC.getOperand(4)); 10316193323Sed AddToWorkList(SETCC.getNode()); 10317263508Sdim return DAG.getSelect(SDLoc(SCC), SCC.getValueType(), 10318263508Sdim SCC.getOperand(2), SCC.getOperand(3), SETCC); 10319193323Sed } 10320193323Sed 10321193323Sed return SCC; 10322193323Sed } 10323193323Sed return SDValue(); 10324193323Sed} 10325193323Sed 10326193323Sed/// SimplifySelectOps - Given a SELECT or a SELECT_CC node, where LHS and RHS 10327193323Sed/// are the two values being selected between, see if we can simplify the 10328193323Sed/// select. Callers of this should assume that TheSelect is deleted if this 10329193323Sed/// returns true. As such, they should return the appropriate thing (e.g. the 10330193323Sed/// node) back to the top-level of the DAG combiner loop to avoid it being 10331193323Sed/// looked at. 10332193323Sedbool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDValue LHS, 10333193323Sed SDValue RHS) { 10334193323Sed 10335218893Sdim // Cannot simplify select with vector condition 10336218893Sdim if (TheSelect->getOperand(0).getValueType().isVector()) return false; 10337218893Sdim 10338193323Sed // If this is a select from two identical things, try to pull the operation 10339193323Sed // through the select. 10340218893Sdim if (LHS.getOpcode() != RHS.getOpcode() || 10341218893Sdim !LHS.hasOneUse() || !RHS.hasOneUse()) 10342218893Sdim return false; 10343218893Sdim 10344218893Sdim // If this is a load and the token chain is identical, replace the select 10345218893Sdim // of two loads with a load through a select of the address to load from. 10346218893Sdim // This triggers in things like "select bool X, 10.0, 123.0" after the FP 10347218893Sdim // constants have been dropped into the constant pool. 10348218893Sdim if (LHS.getOpcode() == ISD::LOAD) { 10349218893Sdim LoadSDNode *LLD = cast<LoadSDNode>(LHS); 10350218893Sdim LoadSDNode *RLD = cast<LoadSDNode>(RHS); 10351218893Sdim 10352218893Sdim // Token chains must be identical. 10353218893Sdim if (LHS.getOperand(0) != RHS.getOperand(0) || 10354193323Sed // Do not let this transformation reduce the number of volatile loads. 10355218893Sdim LLD->isVolatile() || RLD->isVolatile() || 10356218893Sdim // If this is an EXTLOAD, the VT's must match. 10357218893Sdim LLD->getMemoryVT() != RLD->getMemoryVT() || 10358218893Sdim // If this is an EXTLOAD, the kind of extension must match. 10359218893Sdim (LLD->getExtensionType() != RLD->getExtensionType() && 10360218893Sdim // The only exception is if one of the extensions is anyext. 10361218893Sdim LLD->getExtensionType() != ISD::EXTLOAD && 10362218893Sdim RLD->getExtensionType() != ISD::EXTLOAD) || 10363198892Srdivacky // FIXME: this discards src value information. This is 10364198892Srdivacky // over-conservative. It would be beneficial to be able to remember 10365202375Srdivacky // both potential memory locations. Since we are discarding 10366202375Srdivacky // src value info, don't do the transformation if the memory 10367202375Srdivacky // locations are not in the default address space. 10368218893Sdim LLD->getPointerInfo().getAddrSpace() != 0 || 10369249423Sdim RLD->getPointerInfo().getAddrSpace() != 0 || 10370249423Sdim !TLI.isOperationLegalOrCustom(TheSelect->getOpcode(), 10371249423Sdim LLD->getBasePtr().getValueType())) 10372218893Sdim return false; 10373193323Sed 10374218893Sdim // Check that the select condition doesn't reach either load. If so, 10375218893Sdim // folding this will induce a cycle into the DAG. If not, this is safe to 10376218893Sdim // xform, so create a select of the addresses. 10377218893Sdim SDValue Addr; 10378218893Sdim if (TheSelect->getOpcode() == ISD::SELECT) { 10379218893Sdim SDNode *CondNode = TheSelect->getOperand(0).getNode(); 10380218893Sdim if ((LLD->hasAnyUseOfValue(1) && LLD->isPredecessorOf(CondNode)) || 10381218893Sdim (RLD->hasAnyUseOfValue(1) && RLD->isPredecessorOf(CondNode))) 10382218893Sdim return false; 10383243830Sdim // The loads must not depend on one another. 10384243830Sdim if (LLD->isPredecessorOf(RLD) || 10385243830Sdim RLD->isPredecessorOf(LLD)) 10386243830Sdim return false; 10387263508Sdim Addr = DAG.getSelect(SDLoc(TheSelect), 10388263508Sdim LLD->getBasePtr().getValueType(), 10389263508Sdim TheSelect->getOperand(0), LLD->getBasePtr(), 10390263508Sdim RLD->getBasePtr()); 10391218893Sdim } else { // Otherwise SELECT_CC 10392218893Sdim SDNode *CondLHS = TheSelect->getOperand(0).getNode(); 10393218893Sdim SDNode *CondRHS = TheSelect->getOperand(1).getNode(); 10394193323Sed 10395218893Sdim if ((LLD->hasAnyUseOfValue(1) && 10396218893Sdim (LLD->isPredecessorOf(CondLHS) || LLD->isPredecessorOf(CondRHS))) || 10397234353Sdim (RLD->hasAnyUseOfValue(1) && 10398234353Sdim (RLD->isPredecessorOf(CondLHS) || RLD->isPredecessorOf(CondRHS)))) 10399218893Sdim return false; 10400193323Sed 10401263508Sdim Addr = DAG.getNode(ISD::SELECT_CC, SDLoc(TheSelect), 10402218893Sdim LLD->getBasePtr().getValueType(), 10403218893Sdim TheSelect->getOperand(0), 10404218893Sdim TheSelect->getOperand(1), 10405218893Sdim LLD->getBasePtr(), RLD->getBasePtr(), 10406218893Sdim TheSelect->getOperand(4)); 10407193323Sed } 10408218893Sdim 10409218893Sdim SDValue Load; 10410218893Sdim if (LLD->getExtensionType() == ISD::NON_EXTLOAD) { 10411218893Sdim Load = DAG.getLoad(TheSelect->getValueType(0), 10412263508Sdim SDLoc(TheSelect), 10413263508Sdim // FIXME: Discards pointer and TBAA info. 10414218893Sdim LLD->getChain(), Addr, MachinePointerInfo(), 10415218893Sdim LLD->isVolatile(), LLD->isNonTemporal(), 10416234353Sdim LLD->isInvariant(), LLD->getAlignment()); 10417218893Sdim } else { 10418218893Sdim Load = DAG.getExtLoad(LLD->getExtensionType() == ISD::EXTLOAD ? 10419218893Sdim RLD->getExtensionType() : LLD->getExtensionType(), 10420263508Sdim SDLoc(TheSelect), 10421218893Sdim TheSelect->getValueType(0), 10422263508Sdim // FIXME: Discards pointer and TBAA info. 10423218893Sdim LLD->getChain(), Addr, MachinePointerInfo(), 10424218893Sdim LLD->getMemoryVT(), LLD->isVolatile(), 10425218893Sdim LLD->isNonTemporal(), LLD->getAlignment()); 10426218893Sdim } 10427218893Sdim 10428218893Sdim // Users of the select now use the result of the load. 10429218893Sdim CombineTo(TheSelect, Load); 10430218893Sdim 10431218893Sdim // Users of the old loads now use the new load's chain. We know the 10432218893Sdim // old-load value is dead now. 10433218893Sdim CombineTo(LHS.getNode(), Load.getValue(0), Load.getValue(1)); 10434218893Sdim CombineTo(RHS.getNode(), Load.getValue(0), Load.getValue(1)); 10435218893Sdim return true; 10436193323Sed } 10437193323Sed 10438193323Sed return false; 10439193323Sed} 10440193323Sed 10441193323Sed/// SimplifySelectCC - Simplify an expression of the form (N0 cond N1) ? N2 : N3 10442193323Sed/// where 'cond' is the comparison specified by CC. 10443263508SdimSDValue DAGCombiner::SimplifySelectCC(SDLoc DL, SDValue N0, SDValue N1, 10444193323Sed SDValue N2, SDValue N3, 10445193323Sed ISD::CondCode CC, bool NotExtCompare) { 10446193323Sed // (x ? y : y) -> y. 10447193323Sed if (N2 == N3) return N2; 10448218893Sdim 10449198090Srdivacky EVT VT = N2.getValueType(); 10450193323Sed ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode()); 10451193323Sed ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode()); 10452193323Sed ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N3.getNode()); 10453193323Sed 10454193323Sed // Determine if the condition we're dealing with is constant 10455263508Sdim SDValue SCC = SimplifySetCC(getSetCCResultType(N0.getValueType()), 10456193323Sed N0, N1, CC, DL, false); 10457193323Sed if (SCC.getNode()) AddToWorkList(SCC.getNode()); 10458193323Sed ConstantSDNode *SCCC = dyn_cast_or_null<ConstantSDNode>(SCC.getNode()); 10459193323Sed 10460193323Sed // fold select_cc true, x, y -> x 10461193323Sed if (SCCC && !SCCC->isNullValue()) 10462193323Sed return N2; 10463193323Sed // fold select_cc false, x, y -> y 10464193323Sed if (SCCC && SCCC->isNullValue()) 10465193323Sed return N3; 10466193323Sed 10467193323Sed // Check to see if we can simplify the select into an fabs node 10468193323Sed if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1)) { 10469193323Sed // Allow either -0.0 or 0.0 10470193323Sed if (CFP->getValueAPF().isZero()) { 10471193323Sed // select (setg[te] X, +/-0.0), X, fneg(X) -> fabs 10472193323Sed if ((CC == ISD::SETGE || CC == ISD::SETGT) && 10473193323Sed N0 == N2 && N3.getOpcode() == ISD::FNEG && 10474193323Sed N2 == N3.getOperand(0)) 10475193323Sed return DAG.getNode(ISD::FABS, DL, VT, N0); 10476193323Sed 10477193323Sed // select (setl[te] X, +/-0.0), fneg(X), X -> fabs 10478193323Sed if ((CC == ISD::SETLT || CC == ISD::SETLE) && 10479193323Sed N0 == N3 && N2.getOpcode() == ISD::FNEG && 10480193323Sed N2.getOperand(0) == N3) 10481193323Sed return DAG.getNode(ISD::FABS, DL, VT, N3); 10482193323Sed } 10483193323Sed } 10484218893Sdim 10485193323Sed // Turn "(a cond b) ? 1.0f : 2.0f" into "load (tmp + ((a cond b) ? 0 : 4)" 10486193323Sed // where "tmp" is a constant pool entry containing an array with 1.0 and 2.0 10487193323Sed // in it. This is a win when the constant is not otherwise available because 10488193323Sed // it replaces two constant pool loads with one. We only do this if the FP 10489193323Sed // type is known to be legal, because if it isn't, then we are before legalize 10490193323Sed // types an we want the other legalization to happen first (e.g. to avoid 10491193323Sed // messing with soft float) and if the ConstantFP is not legal, because if 10492193323Sed // it is legal, we may not need to store the FP constant in a constant pool. 10493193323Sed if (ConstantFPSDNode *TV = dyn_cast<ConstantFPSDNode>(N2)) 10494193323Sed if (ConstantFPSDNode *FV = dyn_cast<ConstantFPSDNode>(N3)) { 10495193323Sed if (TLI.isTypeLegal(N2.getValueType()) && 10496193323Sed (TLI.getOperationAction(ISD::ConstantFP, N2.getValueType()) != 10497193323Sed TargetLowering::Legal) && 10498193323Sed // If both constants have multiple uses, then we won't need to do an 10499193323Sed // extra load, they are likely around in registers for other users. 10500193323Sed (TV->hasOneUse() || FV->hasOneUse())) { 10501193323Sed Constant *Elts[] = { 10502193323Sed const_cast<ConstantFP*>(FV->getConstantFPValue()), 10503193323Sed const_cast<ConstantFP*>(TV->getConstantFPValue()) 10504193323Sed }; 10505226633Sdim Type *FPTy = Elts[0]->getType(); 10506243830Sdim const DataLayout &TD = *TLI.getDataLayout(); 10507218893Sdim 10508193323Sed // Create a ConstantArray of the two constants. 10509224145Sdim Constant *CA = ConstantArray::get(ArrayType::get(FPTy, 2), Elts); 10510193323Sed SDValue CPIdx = DAG.getConstantPool(CA, TLI.getPointerTy(), 10511193323Sed TD.getPrefTypeAlignment(FPTy)); 10512193323Sed unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); 10513193323Sed 10514193323Sed // Get the offsets to the 0 and 1 element of the array so that we can 10515193323Sed // select between them. 10516193323Sed SDValue Zero = DAG.getIntPtrConstant(0); 10517193323Sed unsigned EltSize = (unsigned)TD.getTypeAllocSize(Elts[0]->getType()); 10518193323Sed SDValue One = DAG.getIntPtrConstant(EltSize); 10519218893Sdim 10520193323Sed SDValue Cond = DAG.getSetCC(DL, 10521263508Sdim getSetCCResultType(N0.getValueType()), 10522193323Sed N0, N1, CC); 10523226633Sdim AddToWorkList(Cond.getNode()); 10524263508Sdim SDValue CstOffset = DAG.getSelect(DL, Zero.getValueType(), 10525263508Sdim Cond, One, Zero); 10526226633Sdim AddToWorkList(CstOffset.getNode()); 10527263508Sdim CPIdx = DAG.getNode(ISD::ADD, DL, CPIdx.getValueType(), CPIdx, 10528193323Sed CstOffset); 10529226633Sdim AddToWorkList(CPIdx.getNode()); 10530193323Sed return DAG.getLoad(TV->getValueType(0), DL, DAG.getEntryNode(), CPIdx, 10531218893Sdim MachinePointerInfo::getConstantPool(), false, 10532234353Sdim false, false, Alignment); 10533193323Sed 10534193323Sed } 10535218893Sdim } 10536193323Sed 10537193323Sed // Check to see if we can perform the "gzip trick", transforming 10538193323Sed // (select_cc setlt X, 0, A, 0) -> (and (sra X, (sub size(X), 1), A) 10539193323Sed if (N1C && N3C && N3C->isNullValue() && CC == ISD::SETLT && 10540193323Sed (N1C->isNullValue() || // (a < 0) ? b : 0 10541193323Sed (N1C->getAPIntValue() == 1 && N0 == N2))) { // (a < 1) ? a : 0 10542198090Srdivacky EVT XType = N0.getValueType(); 10543198090Srdivacky EVT AType = N2.getValueType(); 10544193323Sed if (XType.bitsGE(AType)) { 10545193323Sed // and (sra X, size(X)-1, A) -> "and (srl X, C2), A" iff A is a 10546193323Sed // single-bit constant. 10547193323Sed if (N2C && ((N2C->getAPIntValue() & (N2C->getAPIntValue()-1)) == 0)) { 10548193323Sed unsigned ShCtV = N2C->getAPIntValue().logBase2(); 10549193323Sed ShCtV = XType.getSizeInBits()-ShCtV-1; 10550219077Sdim SDValue ShCt = DAG.getConstant(ShCtV, 10551219077Sdim getShiftAmountTy(N0.getValueType())); 10552263508Sdim SDValue Shift = DAG.getNode(ISD::SRL, SDLoc(N0), 10553193323Sed XType, N0, ShCt); 10554193323Sed AddToWorkList(Shift.getNode()); 10555193323Sed 10556193323Sed if (XType.bitsGT(AType)) { 10557193323Sed Shift = DAG.getNode(ISD::TRUNCATE, DL, AType, Shift); 10558193323Sed AddToWorkList(Shift.getNode()); 10559193323Sed } 10560193323Sed 10561193323Sed return DAG.getNode(ISD::AND, DL, AType, Shift, N2); 10562193323Sed } 10563193323Sed 10564263508Sdim SDValue Shift = DAG.getNode(ISD::SRA, SDLoc(N0), 10565193323Sed XType, N0, 10566193323Sed DAG.getConstant(XType.getSizeInBits()-1, 10567219077Sdim getShiftAmountTy(N0.getValueType()))); 10568193323Sed AddToWorkList(Shift.getNode()); 10569193323Sed 10570193323Sed if (XType.bitsGT(AType)) { 10571193323Sed Shift = DAG.getNode(ISD::TRUNCATE, DL, AType, Shift); 10572193323Sed AddToWorkList(Shift.getNode()); 10573193323Sed } 10574193323Sed 10575193323Sed return DAG.getNode(ISD::AND, DL, AType, Shift, N2); 10576193323Sed } 10577193323Sed } 10578193323Sed 10579218893Sdim // fold (select_cc seteq (and x, y), 0, 0, A) -> (and (shr (shl x)) A) 10580218893Sdim // where y is has a single bit set. 10581218893Sdim // A plaintext description would be, we can turn the SELECT_CC into an AND 10582218893Sdim // when the condition can be materialized as an all-ones register. Any 10583218893Sdim // single bit-test can be materialized as an all-ones register with 10584218893Sdim // shift-left and shift-right-arith. 10585218893Sdim if (CC == ISD::SETEQ && N0->getOpcode() == ISD::AND && 10586218893Sdim N0->getValueType(0) == VT && 10587218893Sdim N1C && N1C->isNullValue() && 10588218893Sdim N2C && N2C->isNullValue()) { 10589218893Sdim SDValue AndLHS = N0->getOperand(0); 10590218893Sdim ConstantSDNode *ConstAndRHS = dyn_cast<ConstantSDNode>(N0->getOperand(1)); 10591218893Sdim if (ConstAndRHS && ConstAndRHS->getAPIntValue().countPopulation() == 1) { 10592218893Sdim // Shift the tested bit over the sign bit. 10593218893Sdim APInt AndMask = ConstAndRHS->getAPIntValue(); 10594218893Sdim SDValue ShlAmt = 10595219077Sdim DAG.getConstant(AndMask.countLeadingZeros(), 10596219077Sdim getShiftAmountTy(AndLHS.getValueType())); 10597263508Sdim SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N0), VT, AndLHS, ShlAmt); 10598218893Sdim 10599218893Sdim // Now arithmetic right shift it all the way over, so the result is either 10600218893Sdim // all-ones, or zero. 10601218893Sdim SDValue ShrAmt = 10602219077Sdim DAG.getConstant(AndMask.getBitWidth()-1, 10603219077Sdim getShiftAmountTy(Shl.getValueType())); 10604263508Sdim SDValue Shr = DAG.getNode(ISD::SRA, SDLoc(N0), VT, Shl, ShrAmt); 10605218893Sdim 10606218893Sdim return DAG.getNode(ISD::AND, DL, VT, Shr, N3); 10607218893Sdim } 10608218893Sdim } 10609218893Sdim 10610193323Sed // fold select C, 16, 0 -> shl C, 4 10611193323Sed if (N2C && N3C && N3C->isNullValue() && N2C->getAPIntValue().isPowerOf2() && 10612226633Sdim TLI.getBooleanContents(N0.getValueType().isVector()) == 10613226633Sdim TargetLowering::ZeroOrOneBooleanContent) { 10614193323Sed 10615193323Sed // If the caller doesn't want us to simplify this into a zext of a compare, 10616193323Sed // don't do it. 10617193323Sed if (NotExtCompare && N2C->getAPIntValue() == 1) 10618193323Sed return SDValue(); 10619193323Sed 10620193323Sed // Get a SetCC of the condition 10621243830Sdim // NOTE: Don't create a SETCC if it's not legal on this target. 10622243830Sdim if (!LegalOperations || 10623243830Sdim TLI.isOperationLegal(ISD::SETCC, 10624263508Sdim LegalTypes ? getSetCCResultType(N0.getValueType()) : MVT::i1)) { 10625243830Sdim SDValue Temp, SCC; 10626243830Sdim // cast from setcc result type to select result type 10627243830Sdim if (LegalTypes) { 10628263508Sdim SCC = DAG.getSetCC(DL, getSetCCResultType(N0.getValueType()), 10629243830Sdim N0, N1, CC); 10630243830Sdim if (N2.getValueType().bitsLT(SCC.getValueType())) 10631263508Sdim Temp = DAG.getZeroExtendInReg(SCC, SDLoc(N2), 10632243830Sdim N2.getValueType()); 10633243830Sdim else 10634263508Sdim Temp = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N2), 10635243830Sdim N2.getValueType(), SCC); 10636243830Sdim } else { 10637263508Sdim SCC = DAG.getSetCC(SDLoc(N0), MVT::i1, N0, N1, CC); 10638263508Sdim Temp = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N2), 10639193323Sed N2.getValueType(), SCC); 10640243830Sdim } 10641193323Sed 10642243830Sdim AddToWorkList(SCC.getNode()); 10643243830Sdim AddToWorkList(Temp.getNode()); 10644193323Sed 10645243830Sdim if (N2C->getAPIntValue() == 1) 10646243830Sdim return Temp; 10647193323Sed 10648243830Sdim // shl setcc result by log2 n2c 10649263508Sdim return DAG.getNode( 10650263508Sdim ISD::SHL, DL, N2.getValueType(), Temp, 10651263508Sdim DAG.getConstant(N2C->getAPIntValue().logBase2(), 10652263508Sdim getShiftAmountTy(Temp.getValueType()))); 10653243830Sdim } 10654193323Sed } 10655193323Sed 10656193323Sed // Check to see if this is the equivalent of setcc 10657193323Sed // FIXME: Turn all of these into setcc if setcc if setcc is legal 10658193323Sed // otherwise, go ahead with the folds. 10659193323Sed if (0 && N3C && N3C->isNullValue() && N2C && (N2C->getAPIntValue() == 1ULL)) { 10660198090Srdivacky EVT XType = N0.getValueType(); 10661193323Sed if (!LegalOperations || 10662263508Sdim TLI.isOperationLegal(ISD::SETCC, getSetCCResultType(XType))) { 10663263508Sdim SDValue Res = DAG.getSetCC(DL, getSetCCResultType(XType), N0, N1, CC); 10664193323Sed if (Res.getValueType() != VT) 10665193323Sed Res = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Res); 10666193323Sed return Res; 10667193323Sed } 10668193323Sed 10669193323Sed // fold (seteq X, 0) -> (srl (ctlz X, log2(size(X)))) 10670193323Sed if (N1C && N1C->isNullValue() && CC == ISD::SETEQ && 10671193323Sed (!LegalOperations || 10672193323Sed TLI.isOperationLegal(ISD::CTLZ, XType))) { 10673263508Sdim SDValue Ctlz = DAG.getNode(ISD::CTLZ, SDLoc(N0), XType, N0); 10674193323Sed return DAG.getNode(ISD::SRL, DL, XType, Ctlz, 10675193323Sed DAG.getConstant(Log2_32(XType.getSizeInBits()), 10676219077Sdim getShiftAmountTy(Ctlz.getValueType()))); 10677193323Sed } 10678193323Sed // fold (setgt X, 0) -> (srl (and (-X, ~X), size(X)-1)) 10679193323Sed if (N1C && N1C->isNullValue() && CC == ISD::SETGT) { 10680263508Sdim SDValue NegN0 = DAG.getNode(ISD::SUB, SDLoc(N0), 10681193323Sed XType, DAG.getConstant(0, XType), N0); 10682263508Sdim SDValue NotN0 = DAG.getNOT(SDLoc(N0), N0, XType); 10683193323Sed return DAG.getNode(ISD::SRL, DL, XType, 10684193323Sed DAG.getNode(ISD::AND, DL, XType, NegN0, NotN0), 10685193323Sed DAG.getConstant(XType.getSizeInBits()-1, 10686219077Sdim getShiftAmountTy(XType))); 10687193323Sed } 10688193323Sed // fold (setgt X, -1) -> (xor (srl (X, size(X)-1), 1)) 10689193323Sed if (N1C && N1C->isAllOnesValue() && CC == ISD::SETGT) { 10690263508Sdim SDValue Sign = DAG.getNode(ISD::SRL, SDLoc(N0), XType, N0, 10691193323Sed DAG.getConstant(XType.getSizeInBits()-1, 10692219077Sdim getShiftAmountTy(N0.getValueType()))); 10693193323Sed return DAG.getNode(ISD::XOR, DL, XType, Sign, DAG.getConstant(1, XType)); 10694193323Sed } 10695193323Sed } 10696193323Sed 10697210299Sed // Check to see if this is an integer abs. 10698210299Sed // select_cc setg[te] X, 0, X, -X -> 10699210299Sed // select_cc setgt X, -1, X, -X -> 10700210299Sed // select_cc setl[te] X, 0, -X, X -> 10701210299Sed // select_cc setlt X, 1, -X, X -> 10702193323Sed // Y = sra (X, size(X)-1); xor (add (X, Y), Y) 10703210299Sed if (N1C) { 10704210299Sed ConstantSDNode *SubC = NULL; 10705210299Sed if (((N1C->isNullValue() && (CC == ISD::SETGT || CC == ISD::SETGE)) || 10706210299Sed (N1C->isAllOnesValue() && CC == ISD::SETGT)) && 10707210299Sed N0 == N2 && N3.getOpcode() == ISD::SUB && N0 == N3.getOperand(1)) 10708210299Sed SubC = dyn_cast<ConstantSDNode>(N3.getOperand(0)); 10709210299Sed else if (((N1C->isNullValue() && (CC == ISD::SETLT || CC == ISD::SETLE)) || 10710210299Sed (N1C->isOne() && CC == ISD::SETLT)) && 10711210299Sed N0 == N3 && N2.getOpcode() == ISD::SUB && N0 == N2.getOperand(1)) 10712210299Sed SubC = dyn_cast<ConstantSDNode>(N2.getOperand(0)); 10713210299Sed 10714198090Srdivacky EVT XType = N0.getValueType(); 10715210299Sed if (SubC && SubC->isNullValue() && XType.isInteger()) { 10716263508Sdim SDValue Shift = DAG.getNode(ISD::SRA, SDLoc(N0), XType, 10717210299Sed N0, 10718210299Sed DAG.getConstant(XType.getSizeInBits()-1, 10719219077Sdim getShiftAmountTy(N0.getValueType()))); 10720263508Sdim SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N0), 10721210299Sed XType, N0, Shift); 10722210299Sed AddToWorkList(Shift.getNode()); 10723210299Sed AddToWorkList(Add.getNode()); 10724210299Sed return DAG.getNode(ISD::XOR, DL, XType, Add, Shift); 10725193323Sed } 10726193323Sed } 10727193323Sed 10728193323Sed return SDValue(); 10729193323Sed} 10730193323Sed 10731193323Sed/// SimplifySetCC - This is a stub for TargetLowering::SimplifySetCC. 10732198090SrdivackySDValue DAGCombiner::SimplifySetCC(EVT VT, SDValue N0, 10733193323Sed SDValue N1, ISD::CondCode Cond, 10734263508Sdim SDLoc DL, bool foldBooleans) { 10735193323Sed TargetLowering::DAGCombinerInfo 10736249423Sdim DagCombineInfo(DAG, Level, false, this); 10737193323Sed return TLI.SimplifySetCC(VT, N0, N1, Cond, foldBooleans, DagCombineInfo, DL); 10738193323Sed} 10739193323Sed 10740193323Sed/// BuildSDIVSequence - Given an ISD::SDIV node expressing a divide by constant, 10741193323Sed/// return a DAG expression to select that will generate the same value by 10742193323Sed/// multiplying by a magic number. See: 10743193323Sed/// <http://the.wall.riscom.net/books/proc/ppc/cwg/code2.html> 10744193323SedSDValue DAGCombiner::BuildSDIV(SDNode *N) { 10745193323Sed std::vector<SDNode*> Built; 10746234353Sdim SDValue S = TLI.BuildSDIV(N, DAG, LegalOperations, &Built); 10747193323Sed 10748193323Sed for (std::vector<SDNode*>::iterator ii = Built.begin(), ee = Built.end(); 10749193323Sed ii != ee; ++ii) 10750193323Sed AddToWorkList(*ii); 10751193323Sed return S; 10752193323Sed} 10753193323Sed 10754193323Sed/// BuildUDIVSequence - Given an ISD::UDIV node expressing a divide by constant, 10755193323Sed/// return a DAG expression to select that will generate the same value by 10756193323Sed/// multiplying by a magic number. See: 10757193323Sed/// <http://the.wall.riscom.net/books/proc/ppc/cwg/code2.html> 10758193323SedSDValue DAGCombiner::BuildUDIV(SDNode *N) { 10759193323Sed std::vector<SDNode*> Built; 10760234353Sdim SDValue S = TLI.BuildUDIV(N, DAG, LegalOperations, &Built); 10761193323Sed 10762193323Sed for (std::vector<SDNode*>::iterator ii = Built.begin(), ee = Built.end(); 10763193323Sed ii != ee; ++ii) 10764193323Sed AddToWorkList(*ii); 10765193323Sed return S; 10766193323Sed} 10767193323Sed 10768198090Srdivacky/// FindBaseOffset - Return true if base is a frame index, which is known not 10769218893Sdim// to alias with anything but itself. Provides base object and offset as 10770218893Sdim// results. 10771198090Srdivackystatic bool FindBaseOffset(SDValue Ptr, SDValue &Base, int64_t &Offset, 10772243830Sdim const GlobalValue *&GV, const void *&CV) { 10773193323Sed // Assume it is a primitive operation. 10774198090Srdivacky Base = Ptr; Offset = 0; GV = 0; CV = 0; 10775193323Sed 10776193323Sed // If it's an adding a simple constant then integrate the offset. 10777193323Sed if (Base.getOpcode() == ISD::ADD) { 10778193323Sed if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Base.getOperand(1))) { 10779193323Sed Base = Base.getOperand(0); 10780193323Sed Offset += C->getZExtValue(); 10781193323Sed } 10782193323Sed } 10783218893Sdim 10784198090Srdivacky // Return the underlying GlobalValue, and update the Offset. Return false 10785198090Srdivacky // for GlobalAddressSDNode since the same GlobalAddress may be represented 10786198090Srdivacky // by multiple nodes with different offsets. 10787198090Srdivacky if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Base)) { 10788198090Srdivacky GV = G->getGlobal(); 10789198090Srdivacky Offset += G->getOffset(); 10790198090Srdivacky return false; 10791198090Srdivacky } 10792193323Sed 10793198090Srdivacky // Return the underlying Constant value, and update the Offset. Return false 10794198090Srdivacky // for ConstantSDNodes since the same constant pool entry may be represented 10795198090Srdivacky // by multiple nodes with different offsets. 10796198090Srdivacky if (ConstantPoolSDNode *C = dyn_cast<ConstantPoolSDNode>(Base)) { 10797243830Sdim CV = C->isMachineConstantPoolEntry() ? (const void *)C->getMachineCPVal() 10798243830Sdim : (const void *)C->getConstVal(); 10799198090Srdivacky Offset += C->getOffset(); 10800198090Srdivacky return false; 10801198090Srdivacky } 10802193323Sed // If it's any of the following then it can't alias with anything but itself. 10803198090Srdivacky return isa<FrameIndexSDNode>(Base); 10804193323Sed} 10805193323Sed 10806193323Sed/// isAlias - Return true if there is any possibility that the two addresses 10807193323Sed/// overlap. 10808263508Sdimbool DAGCombiner::isAlias(SDValue Ptr1, int64_t Size1, bool IsVolatile1, 10809193323Sed const Value *SrcValue1, int SrcValueOffset1, 10810198090Srdivacky unsigned SrcValueAlign1, 10811218893Sdim const MDNode *TBAAInfo1, 10812263508Sdim SDValue Ptr2, int64_t Size2, bool IsVolatile2, 10813198090Srdivacky const Value *SrcValue2, int SrcValueOffset2, 10814218893Sdim unsigned SrcValueAlign2, 10815218893Sdim const MDNode *TBAAInfo2) const { 10816193323Sed // If they are the same then they must be aliases. 10817193323Sed if (Ptr1 == Ptr2) return true; 10818193323Sed 10819263508Sdim // If they are both volatile then they cannot be reordered. 10820263508Sdim if (IsVolatile1 && IsVolatile2) return true; 10821263508Sdim 10822193323Sed // Gather base node and offset information. 10823193323Sed SDValue Base1, Base2; 10824193323Sed int64_t Offset1, Offset2; 10825207618Srdivacky const GlobalValue *GV1, *GV2; 10826243830Sdim const void *CV1, *CV2; 10827198090Srdivacky bool isFrameIndex1 = FindBaseOffset(Ptr1, Base1, Offset1, GV1, CV1); 10828198090Srdivacky bool isFrameIndex2 = FindBaseOffset(Ptr2, Base2, Offset2, GV2, CV2); 10829193323Sed 10830198090Srdivacky // If they have a same base address then check to see if they overlap. 10831198090Srdivacky if (Base1 == Base2 || (GV1 && (GV1 == GV2)) || (CV1 && (CV1 == CV2))) 10832193323Sed return !((Offset1 + Size1) <= Offset2 || (Offset2 + Size2) <= Offset1); 10833193323Sed 10834218893Sdim // It is possible for different frame indices to alias each other, mostly 10835218893Sdim // when tail call optimization reuses return address slots for arguments. 10836218893Sdim // To catch this case, look up the actual index of frame indices to compute 10837218893Sdim // the real alias relationship. 10838218893Sdim if (isFrameIndex1 && isFrameIndex2) { 10839218893Sdim MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); 10840218893Sdim Offset1 += MFI->getObjectOffset(cast<FrameIndexSDNode>(Base1)->getIndex()); 10841218893Sdim Offset2 += MFI->getObjectOffset(cast<FrameIndexSDNode>(Base2)->getIndex()); 10842218893Sdim return !((Offset1 + Size1) <= Offset2 || (Offset2 + Size2) <= Offset1); 10843218893Sdim } 10844218893Sdim 10845218893Sdim // Otherwise, if we know what the bases are, and they aren't identical, then 10846218893Sdim // we know they cannot alias. 10847198090Srdivacky if ((isFrameIndex1 || CV1 || GV1) && (isFrameIndex2 || CV2 || GV2)) 10848198090Srdivacky return false; 10849193323Sed 10850198090Srdivacky // If we know required SrcValue1 and SrcValue2 have relatively large alignment 10851198090Srdivacky // compared to the size and offset of the access, we may be able to prove they 10852198090Srdivacky // do not alias. This check is conservative for now to catch cases created by 10853198090Srdivacky // splitting vector types. 10854198090Srdivacky if ((SrcValueAlign1 == SrcValueAlign2) && 10855198090Srdivacky (SrcValueOffset1 != SrcValueOffset2) && 10856198090Srdivacky (Size1 == Size2) && (SrcValueAlign1 > Size1)) { 10857198090Srdivacky int64_t OffAlign1 = SrcValueOffset1 % SrcValueAlign1; 10858198090Srdivacky int64_t OffAlign2 = SrcValueOffset2 % SrcValueAlign1; 10859218893Sdim 10860198090Srdivacky // There is no overlap between these relatively aligned accesses of similar 10861198090Srdivacky // size, return no alias. 10862198090Srdivacky if ((OffAlign1 + Size1) <= OffAlign2 || (OffAlign2 + Size2) <= OffAlign1) 10863198090Srdivacky return false; 10864198090Srdivacky } 10865218893Sdim 10866263508Sdim bool UseAA = CombinerGlobalAA.getNumOccurrences() > 0 ? CombinerGlobalAA : 10867263508Sdim TLI.getTargetMachine().getSubtarget<TargetSubtargetInfo>().useAA(); 10868263508Sdim if (UseAA && SrcValue1 && SrcValue2) { 10869193323Sed // Use alias analysis information. 10870193323Sed int64_t MinOffset = std::min(SrcValueOffset1, SrcValueOffset2); 10871193323Sed int64_t Overlap1 = Size1 + SrcValueOffset1 - MinOffset; 10872193323Sed int64_t Overlap2 = Size2 + SrcValueOffset2 - MinOffset; 10873193323Sed AliasAnalysis::AliasResult AAResult = 10874218893Sdim AA.alias(AliasAnalysis::Location(SrcValue1, Overlap1, TBAAInfo1), 10875218893Sdim AliasAnalysis::Location(SrcValue2, Overlap2, TBAAInfo2)); 10876193323Sed if (AAResult == AliasAnalysis::NoAlias) 10877193323Sed return false; 10878193323Sed } 10879193323Sed 10880193323Sed // Otherwise we have to assume they alias. 10881193323Sed return true; 10882193323Sed} 10883193323Sed 10884249423Sdimbool DAGCombiner::isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) { 10885249423Sdim SDValue Ptr0, Ptr1; 10886249423Sdim int64_t Size0, Size1; 10887263508Sdim bool IsVolatile0, IsVolatile1; 10888249423Sdim const Value *SrcValue0, *SrcValue1; 10889249423Sdim int SrcValueOffset0, SrcValueOffset1; 10890249423Sdim unsigned SrcValueAlign0, SrcValueAlign1; 10891249423Sdim const MDNode *SrcTBAAInfo0, *SrcTBAAInfo1; 10892263508Sdim FindAliasInfo(Op0, Ptr0, Size0, IsVolatile0, SrcValue0, SrcValueOffset0, 10893249423Sdim SrcValueAlign0, SrcTBAAInfo0); 10894263508Sdim FindAliasInfo(Op1, Ptr1, Size1, IsVolatile1, SrcValue1, SrcValueOffset1, 10895249423Sdim SrcValueAlign1, SrcTBAAInfo1); 10896263508Sdim return isAlias(Ptr0, Size0, IsVolatile0, SrcValue0, SrcValueOffset0, 10897249423Sdim SrcValueAlign0, SrcTBAAInfo0, 10898263508Sdim Ptr1, Size1, IsVolatile1, SrcValue1, SrcValueOffset1, 10899249423Sdim SrcValueAlign1, SrcTBAAInfo1); 10900249423Sdim} 10901249423Sdim 10902193323Sed/// FindAliasInfo - Extracts the relevant alias information from the memory 10903263508Sdim/// node. Returns true if the operand was a nonvolatile load. 10904193323Sedbool DAGCombiner::FindAliasInfo(SDNode *N, 10905263508Sdim SDValue &Ptr, int64_t &Size, bool &IsVolatile, 10906234353Sdim const Value *&SrcValue, 10907234353Sdim int &SrcValueOffset, 10908234353Sdim unsigned &SrcValueAlign, 10909234353Sdim const MDNode *&TBAAInfo) const { 10910234353Sdim LSBaseSDNode *LS = cast<LSBaseSDNode>(N); 10911234353Sdim 10912234353Sdim Ptr = LS->getBasePtr(); 10913234353Sdim Size = LS->getMemoryVT().getSizeInBits() >> 3; 10914263508Sdim IsVolatile = LS->isVolatile(); 10915234353Sdim SrcValue = LS->getSrcValue(); 10916234353Sdim SrcValueOffset = LS->getSrcValueOffset(); 10917234353Sdim SrcValueAlign = LS->getOriginalAlignment(); 10918234353Sdim TBAAInfo = LS->getTBAAInfo(); 10919263508Sdim return isa<LoadSDNode>(LS) && !IsVolatile; 10920193323Sed} 10921193323Sed 10922193323Sed/// GatherAllAliases - Walk up chain skipping non-aliasing memory nodes, 10923193323Sed/// looking for aliasing nodes and adding them to the Aliases vector. 10924193323Sedvoid DAGCombiner::GatherAllAliases(SDNode *N, SDValue OriginalChain, 10925263508Sdim SmallVectorImpl<SDValue> &Aliases) { 10926193323Sed SmallVector<SDValue, 8> Chains; // List of chains to visit. 10927198090Srdivacky SmallPtrSet<SDNode *, 16> Visited; // Visited node set. 10928193323Sed 10929193323Sed // Get alias information for node. 10930193323Sed SDValue Ptr; 10931198090Srdivacky int64_t Size; 10932263508Sdim bool IsVolatile; 10933198090Srdivacky const Value *SrcValue; 10934198090Srdivacky int SrcValueOffset; 10935198090Srdivacky unsigned SrcValueAlign; 10936218893Sdim const MDNode *SrcTBAAInfo; 10937263508Sdim bool IsLoad = FindAliasInfo(N, Ptr, Size, IsVolatile, SrcValue, 10938263508Sdim SrcValueOffset, SrcValueAlign, SrcTBAAInfo); 10939193323Sed 10940193323Sed // Starting off. 10941193323Sed Chains.push_back(OriginalChain); 10942198090Srdivacky unsigned Depth = 0; 10943218893Sdim 10944193323Sed // Look at each chain and determine if it is an alias. If so, add it to the 10945193323Sed // aliases list. If not, then continue up the chain looking for the next 10946193323Sed // candidate. 10947193323Sed while (!Chains.empty()) { 10948193323Sed SDValue Chain = Chains.back(); 10949193323Sed Chains.pop_back(); 10950218893Sdim 10951218893Sdim // For TokenFactor nodes, look at each operand and only continue up the 10952218893Sdim // chain until we find two aliases. If we've seen two aliases, assume we'll 10953198090Srdivacky // find more and revert to original chain since the xform is unlikely to be 10954198090Srdivacky // profitable. 10955218893Sdim // 10956218893Sdim // FIXME: The depth check could be made to return the last non-aliasing 10957198090Srdivacky // chain we found before we hit a tokenfactor rather than the original 10958198090Srdivacky // chain. 10959198090Srdivacky if (Depth > 6 || Aliases.size() == 2) { 10960198090Srdivacky Aliases.clear(); 10961198090Srdivacky Aliases.push_back(OriginalChain); 10962198090Srdivacky break; 10963198090Srdivacky } 10964193323Sed 10965198090Srdivacky // Don't bother if we've been before. 10966198090Srdivacky if (!Visited.insert(Chain.getNode())) 10967198090Srdivacky continue; 10968193323Sed 10969193323Sed switch (Chain.getOpcode()) { 10970193323Sed case ISD::EntryToken: 10971193323Sed // Entry token is ideal chain operand, but handled in FindBetterChain. 10972193323Sed break; 10973193323Sed 10974193323Sed case ISD::LOAD: 10975193323Sed case ISD::STORE: { 10976193323Sed // Get alias information for Chain. 10977193323Sed SDValue OpPtr; 10978198090Srdivacky int64_t OpSize; 10979263508Sdim bool OpIsVolatile; 10980198090Srdivacky const Value *OpSrcValue; 10981198090Srdivacky int OpSrcValueOffset; 10982198090Srdivacky unsigned OpSrcValueAlign; 10983218893Sdim const MDNode *OpSrcTBAAInfo; 10984193323Sed bool IsOpLoad = FindAliasInfo(Chain.getNode(), OpPtr, OpSize, 10985263508Sdim OpIsVolatile, OpSrcValue, OpSrcValueOffset, 10986218893Sdim OpSrcValueAlign, 10987218893Sdim OpSrcTBAAInfo); 10988193323Sed 10989193323Sed // If chain is alias then stop here. 10990193323Sed if (!(IsLoad && IsOpLoad) && 10991263508Sdim isAlias(Ptr, Size, IsVolatile, SrcValue, SrcValueOffset, 10992263508Sdim SrcValueAlign, SrcTBAAInfo, 10993263508Sdim OpPtr, OpSize, OpIsVolatile, OpSrcValue, OpSrcValueOffset, 10994218893Sdim OpSrcValueAlign, OpSrcTBAAInfo)) { 10995193323Sed Aliases.push_back(Chain); 10996193323Sed } else { 10997193323Sed // Look further up the chain. 10998193323Sed Chains.push_back(Chain.getOperand(0)); 10999198090Srdivacky ++Depth; 11000193323Sed } 11001193323Sed break; 11002193323Sed } 11003193323Sed 11004193323Sed case ISD::TokenFactor: 11005198090Srdivacky // We have to check each of the operands of the token factor for "small" 11006198090Srdivacky // token factors, so we queue them up. Adding the operands to the queue 11007198090Srdivacky // (stack) in reverse order maintains the original order and increases the 11008198090Srdivacky // likelihood that getNode will find a matching token factor (CSE.) 11009198090Srdivacky if (Chain.getNumOperands() > 16) { 11010198090Srdivacky Aliases.push_back(Chain); 11011198090Srdivacky break; 11012198090Srdivacky } 11013193323Sed for (unsigned n = Chain.getNumOperands(); n;) 11014193323Sed Chains.push_back(Chain.getOperand(--n)); 11015198090Srdivacky ++Depth; 11016193323Sed break; 11017193323Sed 11018193323Sed default: 11019193323Sed // For all other instructions we will just have to take what we can get. 11020193323Sed Aliases.push_back(Chain); 11021193323Sed break; 11022193323Sed } 11023193323Sed } 11024193323Sed} 11025193323Sed 11026193323Sed/// FindBetterChain - Walk up chain skipping non-aliasing memory nodes, looking 11027193323Sed/// for a better chain (aliasing node.) 11028193323SedSDValue DAGCombiner::FindBetterChain(SDNode *N, SDValue OldChain) { 11029193323Sed SmallVector<SDValue, 8> Aliases; // Ops for replacing token factor. 11030193323Sed 11031193323Sed // Accumulate all the aliases to this node. 11032193323Sed GatherAllAliases(N, OldChain, Aliases); 11033193323Sed 11034223017Sdim // If no operands then chain to entry token. 11035223017Sdim if (Aliases.size() == 0) 11036193323Sed return DAG.getEntryNode(); 11037223017Sdim 11038223017Sdim // If a single operand then chain to it. We don't need to revisit it. 11039223017Sdim if (Aliases.size() == 1) 11040193323Sed return Aliases[0]; 11041218893Sdim 11042193323Sed // Construct a custom tailored token factor. 11043263508Sdim return DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, 11044198090Srdivacky &Aliases[0], Aliases.size()); 11045193323Sed} 11046193323Sed 11047193323Sed// SelectionDAG::Combine - This is the entry point for the file. 11048193323Sed// 11049193323Sedvoid SelectionDAG::Combine(CombineLevel Level, AliasAnalysis &AA, 11050193323Sed CodeGenOpt::Level OptLevel) { 11051193323Sed /// run - This is the main entry point to this class. 11052193323Sed /// 11053193323Sed DAGCombiner(*this, AA, OptLevel).Run(Level); 11054193323Sed} 11055