Lines Matching refs:DL
669 EVT SystemZTargetLowering::getSetCCResultType(const DataLayout &DL,
936 bool SystemZTargetLowering::isLegalAddressingMode(const DataLayout &DL,
1271 static SDValue convertLocVTToValVT(SelectionDAG &DAG, const SDLoc &DL,
1277 Value = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Value,
1280 Value = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Value,
1284 Value = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Value);
1290 Value = DAG.getBuildVector(MVT::v2i64, DL, {Value, DAG.getUNDEF(MVT::i64)});
1291 Value = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Value);
1300 static SDValue convertValVTToLocVT(SelectionDAG &DAG, const SDLoc &DL,
1304 return DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Value);
1306 return DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Value);
1308 return DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Value);
1314 Value = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Value);
1315 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VA.getLocVT(), Value,
1316 DAG.getConstant(0, DL, MVT::i32));
1326 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
1387 ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, LocVT);
1400 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN,
1401 DAG.getIntPtrConstant(4, DL));
1402 ArgValue = DAG.getLoad(LocVT, DL, Chain, FIN,
1409 InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
1418 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue,
1419 DAG.getIntPtrConstant(PartOffset, DL));
1420 InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
1425 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, ArgValue));
1454 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f64);
1455 MemOps[I] = DAG.getStore(ArgValue.getValue(1), DL, ArgValue, FIN,
1459 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
1493 SDLoc &DL = CLI.DL;
1526 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, DL);
1541 DAG.getStore(Chain, DL, ArgValue, SpillSlot,
1550 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot,
1551 DAG.getIntPtrConstant(PartOffset, DL));
1553 DAG.getStore(Chain, DL, PartValue, Address,
1559 ArgValue = convertValVTToLocVT(DAG, DL, VA, ArgValue);
1570 StackPtr = DAG.getCopyFromReg(Chain, DL, SystemZ::R15D, PtrVT);
1574 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
1575 DAG.getIntPtrConstant(Offset, DL));
1579 DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo()));
1585 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
1592 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT);
1593 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee);
1596 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee);
1598 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R1D, Callee, Glue);
1605 Chain = DAG.getCopyToReg(Chain, DL, RegsToPass[I].first,
1634 return DAG.getNode(SystemZISD::SIBCALL, DL, NodeTys, Ops);
1635 Chain = DAG.getNode(SystemZISD::CALL, DL, NodeTys, Ops);
1640 DAG.getConstant(NumBytes, DL, PtrVT, true),
1641 DAG.getConstant(0, DL, PtrVT, true),
1642 Glue, DL);
1655 SDValue RetValue = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(),
1662 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, RetValue));
1693 const SDLoc &DL, SelectionDAG &DAG) const {
1707 return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, Chain);
1724 RetValue = convertValVTToLocVT(DAG, DL, VA, RetValue);
1728 Chain = DAG.getCopyToReg(Chain, DL, Reg, RetValue, Glue);
1738 return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, RetOps);
1992 static void adjustZeroCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
2006 C.Op1 = DAG.getConstant(0, DL, C.Op1.getValueType());
2012 static void adjustSubwordCmp(SelectionDAG &DAG, const SDLoc &DL,
2080 C.Op1 = DAG.getConstant(Value, DL, MVT::i32);
2176 static void adjustForSubtraction(SelectionDAG &DAG, const SDLoc &DL,
2186 C.Op1 = DAG.getConstant(0, DL, N->getValueType(0));
2246 static void adjustICmpTruncate(SelectionDAG &DAG, const SDLoc &DL,
2258 C.Op1 = DAG.getConstant(0, DL, C.Op0.getValueType());
2377 static void adjustForTestUnderMask(SelectionDAG &DAG, const SDLoc &DL,
2460 C.Op1 = DAG.getConstant(MaskVal, DL, C.Op0.getValueType());
2468 static void adjustForRedundantAnd(SelectionDAG &DAG, const SDLoc &DL,
2520 ISD::CondCode Cond, const SDLoc &DL,
2565 adjustForRedundantAnd(DAG, DL, C);
2566 adjustZeroCmp(DAG, DL, C);
2567 adjustSubwordCmp(DAG, DL, C);
2568 adjustForSubtraction(DAG, DL, C);
2570 adjustICmpTruncate(DAG, DL, C);
2578 adjustForTestUnderMask(DAG, DL, C);
2583 static SDValue emitCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
2598 return DAG.getNode(SystemZISD::ICMP, DL, MVT::i32, C.Op0, C.Op1,
2599 DAG.getTargetConstant(C.ICmpType, DL, MVT::i32));
2603 return DAG.getNode(SystemZISD::TM, DL, MVT::i32, C.Op0, C.Op1,
2604 DAG.getTargetConstant(RegisterOnly, DL, MVT::i32));
2608 return DAG.getNode(C.Opcode, DL, VTs, C.Chain, C.Op0, C.Op1);
2610 return DAG.getNode(C.Opcode, DL, MVT::i32, C.Op0, C.Op1);
2616 static void lowerMUL_LOHI32(SelectionDAG &DAG, const SDLoc &DL, unsigned Extend,
2619 Op0 = DAG.getNode(Extend, DL, MVT::i64, Op0);
2620 Op1 = DAG.getNode(Extend, DL, MVT::i64, Op1);
2621 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, Op0, Op1);
2622 Hi = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul,
2623 DAG.getConstant(32, DL, MVT::i64));
2624 Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Hi);
2625 Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mul);
2632 static void lowerGR128Binary(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
2635 SDValue Result = DAG.getNode(Opcode, DL, MVT::Untyped, Op0, Op1);
2637 Even = DAG.getTargetExtractSubreg(SystemZ::even128(Is32Bit), DL, VT, Result);
2638 Odd = DAG.getTargetExtractSubreg(SystemZ::odd128(Is32Bit), DL, VT, Result);
2644 static SDValue emitSETCC(SelectionDAG &DAG, const SDLoc &DL, SDValue CCReg,
2646 SDValue Ops[] = {DAG.getConstant(1, DL, MVT::i32),
2647 DAG.getConstant(0, DL, MVT::i32),
2648 DAG.getTargetConstant(CCValid, DL, MVT::i32),
2649 DAG.getTargetConstant(CCMask, DL, MVT::i32), CCReg};
2650 return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, MVT::i32, Ops);
2726 static SDValue expandV4F32ToV2F64(SelectionDAG &DAG, int Start, const SDLoc &DL,
2729 Op = DAG.getVectorShuffle(MVT::v4f32, DL, Op, DAG.getUNDEF(MVT::v4f32), Mask);
2732 return DAG.getNode(SystemZISD::STRICT_VEXTEND, DL, VTs, Chain, Op);
2734 return DAG.getNode(SystemZISD::VEXTEND, DL, MVT::v2f64, Op);
2740 const SDLoc &DL, EVT VT,
2749 SDValue H0 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp0, Chain);
2750 SDValue L0 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp0, Chain);
2751 SDValue H1 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp1, Chain);
2752 SDValue L1 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp1, Chain);
2755 SDValue HRes = DAG.getNode(Opcode, DL, VTs, Chain, H0, H1);
2756 SDValue LRes = DAG.getNode(Opcode, DL, VTs, Chain, L0, L1);
2757 SDValue Res = DAG.getNode(SystemZISD::PACK, DL, VT, HRes, LRes);
2761 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
2763 return DAG.getMergeValues(Ops, DL);
2765 SDValue HRes = DAG.getNode(Opcode, DL, MVT::v2i64, H0, H1);
2766 SDValue LRes = DAG.getNode(Opcode, DL, MVT::v2i64, L0, L1);
2767 return DAG.getNode(SystemZISD::PACK, DL, VT, HRes, LRes);
2771 return DAG.getNode(Opcode, DL, VTs, Chain, CmpOp0, CmpOp1);
2773 return DAG.getNode(Opcode, DL, VT, CmpOp0, CmpOp1);
2781 const SDLoc &DL, EVT VT,
2802 DL, VT, CmpOp1, CmpOp0, Chain);
2804 DL, VT, CmpOp0, CmpOp1, Chain);
2805 Cmp = DAG.getNode(ISD::OR, DL, VT, LT, GE);
2807 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
2819 DL, VT, CmpOp1, CmpOp0, Chain);
2821 DL, VT, CmpOp0, CmpOp1, Chain);
2822 Cmp = DAG.getNode(ISD::OR, DL, VT, LT, GT);
2824 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
2834 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp0, CmpOp1, Chain);
2838 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp1, CmpOp0, Chain);
2848 DAG.getSplatBuildVector(VT, DL, DAG.getConstant(-1, DL, MVT::i64));
2849 Cmp = DAG.getNode(ISD::XOR, DL, VT, Cmp, Mask);
2853 Cmp = DAG.getMergeValues(Ops, DL);
2863 SDLoc DL(Op);
2866 return lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1);
2868 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
2869 SDValue CCReg = emitCmp(DAG, DL, C);
2870 return emitSETCC(DAG, DL, CCReg, C.CCValid, C.CCMask);
2880 SDLoc DL(Op);
2883 SDValue Res = lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1,
2888 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL, Chain, IsSignaling));
2889 SDValue CCReg = emitCmp(DAG, DL, C);
2891 SDValue Result = emitSETCC(DAG, DL, CCReg, C.CCValid, C.CCMask);
2893 return DAG.getMergeValues(Ops, DL);
2901 SDLoc DL(Op);
2903 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
2904 SDValue CCReg = emitCmp(DAG, DL, C);
2906 SystemZISD::BR_CCMASK, DL, Op.getValueType(), Op.getOperand(0),
2907 DAG.getTargetConstant(C.CCValid, DL, MVT::i32),
2908 DAG.getTargetConstant(C.CCMask, DL, MVT::i32), Dest, CCReg);
2924 static SDValue getAbsolute(SelectionDAG &DAG, const SDLoc &DL, SDValue Op,
2926 Op = DAG.getNode(SystemZISD::IABS, DL, Op.getValueType(), Op);
2928 Op = DAG.getNode(ISD::SUB, DL, Op.getValueType(),
2929 DAG.getConstant(0, DL, Op.getValueType()), Op);
2940 SDLoc DL(Op);
2942 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
2953 return getAbsolute(DAG, DL, TrueOp, C.CCMask & SystemZ::CCMASK_CMP_LT);
2955 return getAbsolute(DAG, DL, FalseOp, C.CCMask & SystemZ::CCMASK_CMP_GT);
2958 SDValue CCReg = emitCmp(DAG, DL, C);
2960 DAG.getTargetConstant(C.CCValid, DL, MVT::i32),
2961 DAG.getTargetConstant(C.CCMask, DL, MVT::i32), CCReg};
2963 return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, Op.getValueType(), Ops);
2968 SDLoc DL(Node);
2979 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor);
2980 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
2987 DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor + Offset);
2988 Result = DAG.getNode(SystemZISD::PCREL_OFFSET, DL, PtrVT, Full, Result);
2994 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT);
2995 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
2998 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, SystemZII::MO_GOT);
2999 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
3000 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result,
3007 Result = DAG.getNode(ISD::ADD, DL, PtrVT, Result,
3008 DAG.getConstant(Offset, DL, PtrVT));
3017 SDLoc DL(Node);
3028 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R12D, GOT, Glue);
3030 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R2D, GOTOffset, Glue);
3036 Ops.push_back(DAG.getTargetGlobalAddress(Node->getGlobal(), DL,
3057 Chain = DAG.getNode(Opcode, DL, NodeTys, Ops);
3061 return DAG.getCopyFromReg(Chain, DL, SystemZ::R2D, PtrVT, Glue);
3064 SDValue SystemZTargetLowering::lowerThreadPointer(const SDLoc &DL,
3070 SDValue TPHi = DAG.getCopyFromReg(Chain, DL, SystemZ::A0, MVT::i32);
3071 TPHi = DAG.getNode(ISD::ANY_EXTEND, DL, PtrVT, TPHi);
3074 SDValue TPLo = DAG.getCopyFromReg(Chain, DL, SystemZ::A1, MVT::i32);
3075 TPLo = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TPLo);
3078 SDValue TPHiShifted = DAG.getNode(ISD::SHL, DL, PtrVT, TPHi,
3079 DAG.getConstant(32, DL, PtrVT));
3080 return DAG.getNode(ISD::OR, DL, PtrVT, TPHiShifted, TPLo);
3087 SDLoc DL(Node);
3096 SDValue TP = lowerThreadPointer(DL, DAG);
3108 PtrVT, DL, DAG.getEntryNode(), Offset,
3123 PtrVT, DL, DAG.getEntryNode(), Offset,
3141 PtrVT, DL, DAG.getEntryNode(), DTPOffset,
3144 Offset = DAG.getNode(ISD::ADD, DL, PtrVT, Offset, DTPOffset);
3150 Offset = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0,
3152 Offset = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Offset);
3154 DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Offset,
3166 PtrVT, DL, DAG.getEntryNode(), Offset,
3173 return DAG.getNode(ISD::ADD, DL, PtrVT, TP, Offset);
3178 SDLoc DL(Node);
3184 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
3190 SDLoc DL(JT);
3195 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
3200 SDLoc DL(CP);
3212 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
3221 SDLoc DL(Op);
3248 SDLoc DL(Op);
3259 return DAG.getCopyFromReg(DAG.getEntryNode(), DL, LinkReg, PtrVT);
3264 SDLoc DL(Op);
3274 SDValue NewLoad = DAG.getLoad(ResVT, DL, LoadN->getChain(),
3284 SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL,
3286 In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL,
3289 In64 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, In);
3290 In64 = DAG.getNode(ISD::SHL, DL, MVT::i64, In64,
3291 DAG.getConstant(32, DL, MVT::i64));
3293 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::f64, In64);
3295 DL, MVT::f32, Out64);
3298 SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::f64);
3299 SDValue In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL,
3301 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::i64, In64);
3303 return DAG.getTargetExtractSubreg(SystemZ::subreg_h32, DL,
3305 SDValue Shift = DAG.getNode(ISD::SRL, DL, MVT::i64, Out64,
3306 DAG.getConstant(32, DL, MVT::i64));
3307 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Shift);
3322 SDLoc DL(Op);
3327 DAG.getConstant(FuncInfo->getVarArgsFirstGPR(), DL, PtrVT),
3328 DAG.getConstant(FuncInfo->getVarArgsFirstFPR(), DL, PtrVT),
3339 FieldAddr = DAG.getNode(ISD::ADD, DL, PtrVT, FieldAddr,
3340 DAG.getIntPtrConstant(Offset, DL));
3341 MemOps[I] = DAG.getStore(Chain, DL, Fields[I], FieldAddr,
3345 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
3355 SDLoc DL(Op);
3357 return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(32, DL),
3373 SDLoc DL(Op);
3388 SDValue OldSP = DAG.getCopyFromReg(Chain, DL, SPReg, MVT::i64);
3393 Backchain = DAG.getLoad(MVT::i64, DL, Chain, OldSP, MachinePointerInfo());
3397 NeededSpace = DAG.getNode(ISD::ADD, DL, MVT::i64, NeededSpace,
3398 DAG.getConstant(ExtraAlignSpace, DL, MVT::i64));
3401 SDValue NewSP = DAG.getNode(ISD::SUB, DL, MVT::i64, OldSP, NeededSpace);
3404 Chain = DAG.getCopyToReg(Chain, DL, SPReg, NewSP);
3409 SDValue ArgAdjust = DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64);
3410 SDValue Result = DAG.getNode(ISD::ADD, DL, MVT::i64, NewSP, ArgAdjust);
3415 DAG.getNode(ISD::ADD, DL, MVT::i64, Result,
3416 DAG.getConstant(ExtraAlignSpace, DL, MVT::i64));
3418 DAG.getNode(ISD::AND, DL, MVT::i64, Result,
3419 DAG.getConstant(~(RequiredAlign - 1), DL, MVT::i64));
3423 Chain = DAG.getStore(Chain, DL, Backchain, NewSP, MachinePointerInfo());
3426 return DAG.getMergeValues(Ops, DL);
3431 SDLoc DL(Op);
3433 return DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64);
3439 SDLoc DL(Op);
3444 lowerMUL_LOHI32(DAG, DL, ISD::SIGN_EXTEND, Op.getOperand(0),
3450 lowerGR128Binary(DAG, DL, VT, SystemZISD::SMUL_LOHI,
3466 SDValue C63 = DAG.getConstant(63, DL, MVT::i64);
3469 SDValue LH = DAG.getNode(ISD::SRA, DL, VT, LL, C63);
3470 SDValue RH = DAG.getNode(ISD::SRA, DL, VT, RL, C63);
3474 lowerGR128Binary(DAG, DL, VT, SystemZISD::UMUL_LOHI,
3476 SDValue NegLLTimesRH = DAG.getNode(ISD::AND, DL, VT, LL, RH);
3477 SDValue NegLHTimesRL = DAG.getNode(ISD::AND, DL, VT, LH, RL);
3478 SDValue NegSum = DAG.getNode(ISD::ADD, DL, VT, NegLLTimesRH, NegLHTimesRL);
3479 Ops[1] = DAG.getNode(ISD::SUB, DL, VT, Ops[1], NegSum);
3481 return DAG.getMergeValues(Ops, DL);
3487 SDLoc DL(Op);
3492 lowerMUL_LOHI32(DAG, DL, ISD::ZERO_EXTEND, Op.getOperand(0),
3498 lowerGR128Binary(DAG, DL, VT, SystemZISD::UMUL_LOHI,
3500 return DAG.getMergeValues(Ops, DL);
3508 SDLoc DL(Op);
3514 Op0 = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op0);
3516 Op1 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Op1);
3521 lowerGR128Binary(DAG, DL, VT, SystemZISD::SDIVREM, Op0, Op1, Ops[1], Ops[0]);
3522 return DAG.getMergeValues(Ops, DL);
3528 SDLoc DL(Op);
3530 // DL(G) returns the remainder in the even register and the
3533 lowerGR128Binary(DAG, DL, VT, SystemZISD::UDIVREM,
3535 return DAG.getMergeValues(Ops, DL);
3587 SDLoc DL(Op);
3588 SDValue Low32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, LowOp);
3589 return DAG.getTargetInsertSubreg(SystemZ::subreg_l32, DL,
3599 SDLoc DL(N);
3629 SDValue Result = DAG.getNode(BaseOp, DL, VTs, LHS, RHS);
3631 SDValue SetCC = emitSETCC(DAG, DL, Result.getValue(1), CCValid, CCMask);
3633 SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC);
3635 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, SetCC);
3664 SDLoc DL(N);
3690 Carry = DAG.getNode(SystemZISD::GET_CCMASK, DL, MVT::i32, Carry,
3691 DAG.getConstant(CCValid, DL, MVT::i32),
3692 DAG.getConstant(CCMask, DL, MVT::i32));
3695 SDValue Result = DAG.getNode(BaseOp, DL, VTs, LHS, RHS, Carry);
3697 SDValue SetCC = emitSETCC(DAG, DL, Result.getValue(1), CCValid, CCMask);
3699 SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC);
3701 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, SetCC);
3707 SDLoc DL(Op);
3712 Op = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Op);
3713 Op = DAG.getNode(SystemZISD::POPCNT, DL, MVT::v16i8, Op);
3718 Op = DAG.getNode(ISD::BITCAST, DL, VT, Op);
3719 SDValue Shift = DAG.getConstant(8, DL, MVT::i32);
3720 SDValue Tmp = DAG.getNode(SystemZISD::VSHL_BY_SCALAR, DL, VT, Op, Shift);
3721 Op = DAG.getNode(ISD::ADD, DL, VT, Op, Tmp);
3722 Op = DAG.getNode(SystemZISD::VSRL_BY_SCALAR, DL, VT, Op, Shift);
3726 SDValue Tmp = DAG.getSplatBuildVector(MVT::v16i8, DL,
3727 DAG.getConstant(0, DL, MVT::i32));
3728 Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp);
3732 SDValue Tmp = DAG.getSplatBuildVector(MVT::v16i8, DL,
3733 DAG.getConstant(0, DL, MVT::i32));
3734 Op = DAG.getNode(SystemZISD::VSUM, DL, MVT::v4i32, Op, Tmp);
3735 Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp);
3748 return DAG.getConstant(0, DL, VT);
3756 Op = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op);
3757 Op = DAG.getNode(SystemZISD::POPCNT, DL, MVT::i64, Op);
3758 Op = DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
3763 SDValue Tmp = DAG.getNode(ISD::SHL, DL, VT, Op, DAG.getConstant(I, DL, VT));
3765 Tmp = DAG.getNode(ISD::AND, DL, VT, Tmp,
3766 DAG.getConstant(((uint64_t)1 << BitSize) - 1, DL, VT));
3767 Op = DAG.getNode(ISD::ADD, DL, VT, Op, Tmp);
3772 Op = DAG.getNode(ISD::SRL, DL, VT, Op,
3773 DAG.getConstant(BitSize - 8, DL, VT));
3780 SDLoc DL(Op);
3790 return SDValue(DAG.getMachineNode(SystemZ::Serialize, DL, MVT::Other,
3796 return DAG.getNode(SystemZISD::MEMBARRIER, DL, MVT::Other, Op.getOperand(0));
3841 SDLoc DL(Node);
3848 Src2 = DAG.getConstant(-Const->getSExtValue(), DL, Src2.getValueType());
3852 SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr,
3853 DAG.getConstant(-4, DL, PtrVT));
3857 SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr,
3858 DAG.getConstant(3, DL, PtrVT));
3859 BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift);
3863 SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT,
3864 DAG.getConstant(0, DL, WideVT), BitShift);
3872 Src2 = DAG.getNode(ISD::SHL, DL, WideVT, Src2,
3873 DAG.getConstant(32 - BitSize, DL, WideVT));
3876 Src2 = DAG.getNode(ISD::OR, DL, WideVT, Src2,
3877 DAG.getConstant(uint32_t(-1) >> BitSize, DL, WideVT));
3882 DAG.getConstant(BitSize, DL, WideVT) };
3883 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, DL, VTList, Ops,
3888 SDValue ResultShift = DAG.getNode(ISD::ADD, DL, WideVT, BitShift,
3889 DAG.getConstant(BitSize, DL, WideVT));
3890 SDValue Result = DAG.getNode(ISD::ROTL, DL, WideVT, AtomicOp, ResultShift);
3893 return DAG.getMergeValues(RetOps, DL);
3908 SDLoc DL(Src2);
3915 NegSrc2 = DAG.getConstant(Value, DL, MemVT);
3918 NegSrc2 = DAG.getNode(ISD::SUB, DL, MemVT, DAG.getConstant(0, DL, MemVT),
3922 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, MemVT,
3942 SDLoc DL(Node);
3952 DL, Tys, Ops, NarrowVT, MMO);
3953 SDValue Success = emitSETCC(DAG, DL, AtomicOp.getValue(1),
3968 SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr,
3969 DAG.getConstant(-4, DL, PtrVT));
3973 SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr,
3974 DAG.getConstant(3, DL, PtrVT));
3975 BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift);
3979 SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT,
3980 DAG.getConstant(0, DL, WideVT), BitShift);
3985 NegBitShift, DAG.getConstant(BitSize, DL, WideVT) };
3986 SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAPW, DL,
3988 SDValue Success = emitSETCC(DAG, DL, AtomicOp.getValue(1),
4042 SDLoc DL(Op);
4045 SDValue OldSP = DAG.getCopyFromReg(Chain, DL, SystemZ::R15D, MVT::i64);
4046 Backchain = DAG.getLoad(MVT::i64, DL, Chain, OldSP, MachinePointerInfo());
4049 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R15D, NewSP);
4052 Chain = DAG.getStore(Chain, DL, Backchain, NewSP, MachinePointerInfo());
4064 SDLoc DL(Op);
4068 SDValue Ops[] = {Op.getOperand(0), DAG.getTargetConstant(Code, DL, MVT::i32),
4070 return DAG.getMemIntrinsicNode(SystemZISD::PREFETCH, DL,
4077 SDLoc DL(CCReg);
4078 SDValue IPM = DAG.getNode(SystemZISD::IPM, DL, MVT::i32, CCReg);
4079 return DAG.getNode(ISD::SRL, DL, MVT::i32, IPM,
4080 DAG.getConstant(SystemZ::IPM_CC, DL, MVT::i32));
4397 static SDValue getPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
4407 Op0 = DAG.getNode(ISD::BITCAST, DL, InVT, Op0);
4408 Op1 = DAG.getNode(ISD::BITCAST, DL, InVT, Op1);
4411 SDValue Op2 = DAG.getTargetConstant(P.Operand, DL, MVT::i32);
4412 Op = DAG.getNode(SystemZISD::PERMUTE_DWORDS, DL, InVT, Op0, Op1, Op2);
4416 Op = DAG.getNode(SystemZISD::PACK, DL, OutVT, Op0, Op1);
4418 Op = DAG.getNode(P.Opcode, DL, InVT, Op0, Op1);
4426 static SDValue getGeneralPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
4430 Ops[I] = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Ops[I]);
4435 return DAG.getNode(SystemZISD::SHL_DOUBLE, DL, MVT::v16i8, Ops[OpNo0],
4437 DAG.getTargetConstant(StartIndex, DL, MVT::i32));
4443 IndexNodes[I] = DAG.getConstant(Bytes[I], DL, MVT::i32);
4446 SDValue Op2 = DAG.getBuildVector(MVT::v16i8, DL, IndexNodes);
4447 return DAG.getNode(SystemZISD::PERMUTE, DL, MVT::v16i8, Ops[0], Ops[1], Op2);
4544 SDValue GeneralShuffle::getNode(SelectionDAG &DAG, const SDLoc &DL) {
4583 Ops[I] = getPermuteNode(DAG, DL, *P, SubOps[0], SubOps[1]);
4595 Ops[I] = getGeneralPermuteNode(DAG, DL, SubOps, NewBytes);
4616 Op = getPermuteNode(DAG, DL, *P, Ops[OpNo0], Ops[OpNo1]);
4618 Op = getGeneralPermuteNode(DAG, DL, &Ops[0], Bytes);
4619 return DAG.getNode(ISD::BITCAST, DL, VT, Op);
4632 static SDValue buildScalarToVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
4639 return DAG.getBuildVector(VT, DL, Ops);
4643 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Value);
4648 static SDValue buildMergeScalars(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
4653 return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op1);
4656 return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op0);
4657 return DAG.getNode(SystemZISD::MERGE_HIGH, DL, VT,
4658 buildScalarToVector(DAG, DL, VT, Op0),
4659 buildScalarToVector(DAG, DL, VT, Op1));
4664 static SDValue joinDwords(SelectionDAG &DAG, const SDLoc &DL, SDValue Op0,
4671 Op0 = Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op1);
4673 Op0 = Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
4675 Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
4676 Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op1);
4678 return DAG.getNode(SystemZISD::JOIN_DWORDS, DL, MVT::v2i64, Op0, Op1);
4744 SystemZTargetLowering::buildVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
4775 return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Single);
4787 return joinDwords(DAG, DL, Elems[0], Elems[1]);
4791 return buildMergeScalars(DAG, DL, VT, Elems[0], Elems[1]);
4801 SDValue Op01 = buildMergeScalars(DAG, DL, VT, Elems[0], Elems[1]);
4802 SDValue Op23 = buildMergeScalars(DAG, DL, VT, Elems[2], Elems[3]);
4811 Op01 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Op01);
4812 Op23 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Op23);
4814 DL, MVT::v2i64, Op01, Op23);
4815 return DAG.getNode(ISD::BITCAST, DL, VT, Op);
4841 Result = DAG.getBuildVector(VT, DL, Constants);
4860 Result = DAG.getNode(SystemZISD::REPLICATE, DL, VT, ReplicatedVal);
4870 Result = DAG.getNode(ISD::BITCAST, DL, VT,
4871 joinDwords(DAG, DL, Elem1, Elem2));
4882 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Result, Elems[I],
4883 DAG.getConstant(I, DL, MVT::i32));
4890 SDLoc DL(Op);
4907 return buildScalarToVector(DAG, DL, VT, Op.getOperand(0));
4914 return buildVector(DAG, DL, VT, Ops);
4920 SDLoc DL(Op);
4932 return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op0.getOperand(Index));
4934 return DAG.getNode(SystemZISD::SPLAT, DL, VT, Op.getOperand(0),
4935 DAG.getTargetConstant(Index, DL, MVT::i32));
4952 SDLoc DL(Op);
4954 return DAG.getNode(ISD::INSERT_VECTOR_ELT, DL,
4956 Op.getOperand(0), DAG.getConstant(0, DL, MVT::i32));
4962 SDLoc DL(Op);
4984 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntVecVT,
4985 DAG.getNode(ISD::BITCAST, DL, IntVecVT, Op0),
4986 DAG.getNode(ISD::BITCAST, DL, IntVT, Op1), Op2);
4987 return DAG.getNode(ISD::BITCAST, DL, VT, Res);
4994 SDLoc DL(Op);
5011 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntVT,
5012 DAG.getNode(ISD::BITCAST, DL, IntVecVT, Op0), Op1);
5013 return DAG.getNode(ISD::BITCAST, DL, VT, Res);
5038 SDLoc DL(Op);
5053 DL, MVT::i32);
5054 return DAG.getNode(ByScalar, DL, VT, Op0, Shift);
5062 SDValue Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Splat);
5063 return DAG.getNode(ByScalar, DL, VT, Op0, Shift);
5079 SDValue Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32,
5081 return DAG.getNode(ByScalar, DL, VT, Op0, Shift);
5216 SDLoc DL(In);
5217 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, In,
5218 DAG.getIntPtrConstant(0, DL));
5219 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, In,
5220 DAG.getIntPtrConstant(1, DL));
5221 SDNode *Pair = DAG.getMachineNode(SystemZ::PAIR128, DL,
5227 SDLoc DL(In);
5229 DL, MVT::i64, In);
5231 DL, MVT::i64, In);
5232 return DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i128, Lo, Hi);
5241 SDLoc DL(N);
5246 DL, Tys, Ops, MVT::i128, MMO);
5252 SDLoc DL(N);
5259 DL, Tys, Ops, MVT::i128, MMO);
5264 Res = SDValue(DAG.getMachineNode(SystemZ::Serialize, DL,
5270 SDLoc DL(N);
5277 DL, Tys, Ops, MVT::i128, MMO);
5278 SDValue Success = emitSETCC(DAG, DL, Res.getValue(1),
5280 Success = DAG.getZExtOrTrunc(Success, DL, N->getValueType(1));
5445 SDValue SystemZTargetLowering::combineExtract(const SDLoc &DL, EVT ResVT,
5500 Op = DAG.getNode(ISD::BITCAST, DL, VT, Op);
5504 Op = DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
5507 Op = DAG.getNode(ISD::BITCAST, DL, ResVT, Op);
5540 Op = DAG.getNode(ISD::BITCAST, DL, VecVT, Op);
5543 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Op,
5544 DAG.getConstant(Index, DL, MVT::i32));
5552 const SDLoc &DL, EVT TruncVT, SDValue Op, DAGCombinerInfo &DCI) const {
5578 return combineExtract(DL, ResVT, VecVT, Vec, NewIndex, DCI, true);
5596 SDLoc DL(N0);
5597 SDValue Ops[] = { DAG.getConstant(TrueOp->getZExtValue(), DL, VT),
5598 DAG.getConstant(FalseOp->getZExtValue(), DL, VT),
5600 SDValue NewSelect = DAG.getNode(SystemZISD::SELECT_CCMASK, DL, VT, Ops);
5604 DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), NewSelect);
5625 SDLoc DL(N0);
5627 DAG.getConstant(-1, DL, VT), DAG.getConstant(0, DL, VT),
5629 return DAG.getNode(ISD::SELECT_CC, DL, VT, Ops);
5725 SDLoc DL(N);
5726 SDValue Extract0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, LdVT,
5727 Replicate, DAG.getConstant(0, DL, MVT::i32));
6829 DebugLoc DL = MI->getDebugLoc();
6830 BuildMI(*SinkMBB, SinkInsertionPoint, DL, TII->get(SystemZ::PHI), DestReg)
6952 DebugLoc DL = MI.getDebugLoc();
6972 BuildMI(*MBB, MI, DL, TII->get(STOCOpcode))
7003 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
7012 BuildMI(MBB, DL, TII->get(StoreOpcode))
7047 DebugLoc DL = MI.getDebugLoc();
7081 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal).add(Base).addImm(Disp).addReg(0);
7093 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)
7097 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal)
7102 BuildMI(MBB, DL, TII->get(BinOpcode), Tmp).addReg(RotatedOldVal).add(Src2);
7105 BuildMI(MBB, DL, TII->get(SystemZ::XILF), RotatedNewVal)
7111 BuildMI(MBB, DL, TII->get(SystemZ::LCGR), Tmp2).addReg(Tmp);
7112 BuildMI(MBB, DL, TII->get(SystemZ::AGHI), RotatedNewVal)
7117 BuildMI(MBB, DL, TII->get(BinOpcode), RotatedNewVal)
7123 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedNewVal)
7127 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal)
7129 BuildMI(MBB, DL, TII->get(CSOpcode), Dest)
7134 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
7165 DebugLoc DL = MI.getDebugLoc();
7201 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal).add(Base).addImm(Disp).addReg(0);
7210 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)
7214 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal)
7216 BuildMI(MBB, DL, TII->get(CompareOpcode))
7218 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
7228 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedAltVal)
7241 BuildMI(MBB, DL, TII->get(SystemZ::PHI), RotatedNewVal)
7245 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal)
7247 BuildMI(MBB, DL, TII->get(CSOpcode), Dest)
7252 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
7281 DebugLoc DL = MI.getDebugLoc();
7311 BuildMI(MBB, DL, TII->get(LOpcode), OrigOldVal)
7332 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)
7335 BuildMI(MBB, DL, TII->get(SystemZ::PHI), CmpVal)
7338 BuildMI(MBB, DL, TII->get(SystemZ::PHI), SwapVal)
7341 BuildMI(MBB, DL, TII->get(SystemZ::RLL), Dest)
7343 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetryCmpVal)
7345 BuildMI(MBB, DL, TII->get(SystemZ::CR))
7347 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
7363 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal)
7365 BuildMI(MBB, DL, TII->get(SystemZ::RLL), StoreVal)
7367 BuildMI(MBB, DL, TII->get(CSOpcode), RetryOldVal)
7372 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
7395 DebugLoc DL = MI.getDebugLoc();
7403 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::IMPLICIT_DEF), Tmp1);
7404 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Tmp2)
7406 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dest)
7423 DebugLoc DL = MI.getDebugLoc();
7429 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::IMPLICIT_DEF), In128);
7434 BuildMI(*MBB, MI, DL, TII->get(SystemZ::LLILL), Zero64)
7436 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), NewIn128)
7440 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dest)
7453 DebugLoc DL = MI.getDebugLoc();
7510 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisDestReg)
7514 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisSrcReg)
7517 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisCountReg)
7521 BuildMI(MBB, DL, TII->get(SystemZ::PFD))
7524 BuildMI(MBB, DL, TII->get(Opcode))
7528 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
7546 BuildMI(MBB, DL, TII->get(SystemZ::LA), NextDestReg)
7549 BuildMI(MBB, DL, TII->get(SystemZ::LA), NextSrcReg)
7551 BuildMI(MBB, DL, TII->get(SystemZ::AGHI), NextCountReg)
7553 BuildMI(MBB, DL, TII->get(SystemZ::CGHI))
7555 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
7593 BuildMI(*MBB, MI, DL, TII->get(Opcode))
7607 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
7633 DebugLoc DL = MI.getDebugLoc();
7664 BuildMI(MBB, DL, TII->get(SystemZ::PHI), This1Reg)
7667 BuildMI(MBB, DL, TII->get(SystemZ::PHI), This2Reg)
7670 BuildMI(MBB, DL, TII->get(TargetOpcode::COPY), SystemZ::R0L).addReg(CharReg);
7671 BuildMI(MBB, DL, TII->get(Opcode))
7674 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
7740 DebugLoc DL = MI.getDebugLoc();
7750 BuildMI(*MBB, MI, DL, TII->get(Opcode), DstReg)