| H A D | X86ISelLowering.cpp | 3289 static SDValue getTargetShuffleNode(unsigned Opc, SDLoc dl, EVT VT, function
3300 static SDValue getTargetShuffleNode(unsigned Opc, SDLoc dl, EVT VT, function
3314 static SDValue getTargetShuffleNode(unsigned Opc, SDLoc dl, EVT VT, function
3327 static SDValue getTargetShuffleNode(unsigned Opc, SDLoc dl, EVT VT, function
6877 assert(Idx < 8 && �); if (Idx < 4) { Locs[i] = std::make_pair(0, NumLo); Mask1[NumLo] = Idx; NumLo++; } else { Locs[i] = std::make_pair(1, NumHi); if (2+NumHi < 4) Mask1[2+NumHi] = Idx; NumHi++; } } } if (NumLo <= 2 && NumHi <= 2) { V1 = DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]); int Mask2[] = { -1, -1, -1, -1 }; for (unsigned i = 0; i != 4; ++i) if (Locs[i].first != -1) { unsigned Idx = (i < 2) ? 0 : 4; Idx += Locs[i].first * 2 + Locs[i].second; Mask2[i] = Idx; } return DAG.getVectorShuffle(VT, dl, V1, V1, &Mask2[0]); } if (NumLo == 3 || NumHi == 3) { if (NumHi == 3) { CommuteVectorShuffleMask(PermMask, 4); std::swap(V1, V2); } unsigned HiIndex; for (HiIndex = 0; HiIndex < 3; ++HiIndex) { int Val = PermMask[HiIndex]; if (Val < 0) continue; if (Val >= 4) break; } Mask1[0] = PermMask[HiIndex]; Mask1[1] = -1; Mask1[2] = PermMask[HiIndex^1]; Mask1[3] = -1; V2 = DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]); if (HiIndex >= 2) { Mask1[0] = PermMask[0]; Mask1[1] = PermMask[1]; Mask1[2] = HiIndex & 1 ? 6 : 4; Mask1[3] = HiIndex & 1 ? 4 : 6; return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]); } Mask1[0] = HiIndex & 1 ? 2 : 0; Mask1[1] = HiIndex & 1 ? 0 : 2; Mask1[2] = PermMask[2]; Mask1[3] = PermMask[3]; if (Mask1[2] >= 0) Mask1[2] += 4; if (Mask1[3] >= 0) Mask1[3] += 4; return DAG.getVectorShuffle(VT, dl, V2, V1, &Mask1[0]); } int LoMask[] = { -1, -1, -1, -1 }; int HiMask[] = { -1, -1, -1, -1 }; int *MaskPtr = LoMask; unsigned MaskIdx = 0; unsigned LoIdx = 0; unsigned HiIdx = 2; for (unsigned i = 0; i != 4; ++i) { if (i == 2) { MaskPtr = HiMask; MaskIdx = 1; LoIdx = 0; HiIdx = 2; } int Idx = PermMask[i]; if (Idx < 0) { Locs[i] = std::make_pair(-1, -1); } else if (Idx < 4) { Locs[i] = std::make_pair(MaskIdx, LoIdx); MaskPtr[LoIdx] = Idx; LoIdx++; } else { Locs[i] = std::make_pair(MaskIdx, HiIdx); MaskPtr[HiIdx] = Idx; HiIdx++; } } SDValue LoShuffle = DAG.getVectorShuffle(VT, dl, V1, V2, &LoMask[0]); SDValue HiShuffle = DAG.getVectorShuffle(VT, dl, V1, V2, &HiMask[0]); int MaskOps[] = { -1, -1, -1, -1 }; for (unsigned i = 0; i != 4; ++i) if (Locs[i].first != -1) MaskOps[i] = Locs[i].first * 4 + Locs[i].second; return DAG.getVectorShuffle(VT, dl, LoShuffle, HiShuffle, &MaskOps[0]); } static bool MayFoldVectorLoad(SDValue V) { while (V.hasOneUse() && V.getOpcode() == ISD::BITCAST) V = V.getOperand(0); if (V.hasOneUse() && V.getOpcode() == ISD::SCALAR_TO_VECTOR) V = V.getOperand(0); if (V.hasOneUse() && V.getOpcode() == ISD::BUILD_VECTOR && V.getNumOperands() == 2 && V.getOperand(1).getOpcode() == ISD::UNDEF) V = V.getOperand(0); return MayFoldLoad(V); } static SDValue getMOVDDup(SDValue &Op, SDLoc &dl, SDValue V1, SelectionDAG &DAG) { MVT VT = Op.getSimpleValueType(); V1 = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, V1); return DAG.getNode(ISD::BITCAST, dl, VT, getTargetShuffleNode(X86ISD::MOVDDUP, dl, MVT::v2f64, V1, DAG)); } static SDValue getMOVLowToHigh(SDValue &Op, SDLoc &dl, SelectionDAG &DAG, bool HasSSE2) { SDValue V1 = Op.getOperand(0); SDValue V2 = Op.getOperand(1); MVT VT = Op.getSimpleValueType(); assert(VT != MVT::v2i64 && �); if (HasSSE2 && VT == MVT::v2f64) return getTargetShuffleNode(X86ISD::MOVLHPD, dl, VT, V1, V2, DAG); return DAG.getNode(ISD::BITCAST, dl, VT, getTargetShuffleNode(X86ISD::MOVLHPS, dl, MVT::v4f32, DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, V1), DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, V2), DAG)); } static SDValue getMOVHighToLow(SDValue &Op, SDLoc &dl, SelectionDAG &DAG) { SDValue V1 = Op.getOperand(0); SDValue V2 = Op.getOperand(1); MVT VT = Op.getSimpleValueType(); assert((VT == MVT::v4i32 || VT == MVT::v4f32) && �); if (V2.getOpcode() == ISD::UNDEF) V2 = V1; return getTargetShuffleNode(X86ISD::MOVHLPS, dl, VT, V1, V2, DAG); } static SDValue getMOVLP(SDValue &Op, SDLoc &dl, SelectionDAG &DAG, bool HasSSE2) { SDValue V1 = Op.getOperand(0); SDValue V2 = Op.getOperand(1); MVT VT = Op.getSimpleValueType(); unsigned NumElems = VT.getVectorNumElements(); bool CanFoldLoad = false; if (MayFoldVectorLoad(V2)) CanFoldLoad = true; else if (MayFoldVectorLoad(V1) && MayFoldIntoStore(Op)) CanFoldLoad = true; ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op); if (CanFoldLoad) { if (HasSSE2 && NumElems == 2) return getTargetShuffleNode(X86ISD::MOVLPD, dl, VT, V1, V2, DAG); if (NumElems == 4) if (SVOp->getMaskElt(1) != -1) return getTargetShuffleNode(X86ISD::MOVLPS, dl, VT, V1, V2, DAG); } if (HasSSE2) { if (NumElems == 2 || !isMOVLMask(SVOp->getMask(), VT)) return getTargetShuffleNode(X86ISD::MOVSD, dl, VT, V1, V2, DAG); return getTargetShuffleNode(X86ISD::MOVSS, dl, VT, V1, V2, DAG); } assert(VT != MVT::v4i32 && �); return getTargetShuffleNode(X86ISD::SHUFP, dl, VT, V2, V1, getShuffleSHUFImmediate(SVOp), DAG); } static SDValue LowerVectorIntExtend(SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) { if (!Subtarget->hasSSE41()) return SDValue(); MVT VT = Op.getSimpleValueType(); if (!Subtarget->hasInt256() && VT.is256BitVector()) return SDValue(); ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op); SDLoc DL(Op); SDValue V1 = Op.getOperand(0); SDValue V2 = Op.getOperand(1); unsigned NumElems = VT.getVectorNumElements(); if (V2.getOpcode() != ISD::UNDEF || !VT.isInteger() || VT.getVectorElementType() == MVT::i64) return SDValue(); unsigned Shift = 1; while ((1U << Shift) < NumElems) { if (SVOp->getMaskElt(1U << Shift) == 1) break; Shift += 1; if (Shift > 3) return SDValue(); } unsigned Mask = (1U << Shift) - 1; for (unsigned i = 0; i != NumElems; ++i) { int EltIdx = SVOp->getMaskElt(i); if ((i & Mask) != 0 && EltIdx != -1) return SDValue(); if ((i & Mask) == 0 && (unsigned)EltIdx != (i >> Shift)) return SDValue(); } unsigned NBits = VT.getVectorElementType().getSizeInBits() << Shift; MVT NeVT = MVT::getIntegerVT(NBits); MVT NVT = MVT::getVectorVT(NeVT, NumElems >> Shift); if (!DAG.getTargetLoweringInfo().isTypeLegal(NVT)) return SDValue(); unsigned SignificantBits = NVT.getSizeInBits() >> Shift; if (V1.getOpcode() == ISD::BITCAST && V1.getOperand(0).getOpcode() == ISD::SCALAR_TO_VECTOR && V1.getOperand(0).getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT && V1.getOperand(0).getOperand(0) .getSimpleValueType().getSizeInBits() == SignificantBits) { SDValue V = V1.getOperand(0).getOperand(0).getOperand(0); ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(V1.getOperand(0).getOperand(0).getOperand(1)); if (CIdx && CIdx->getZExtValue() == 0 && (!ISD::isNormalLoad(V.getNode()) || !V.hasOneUse())) { MVT FullVT = V.getSimpleValueType(); MVT V1VT = V1.getSimpleValueType(); if (FullVT.getSizeInBits() > V1VT.getSizeInBits()) argument
[all...] |