X86ShuffleDecodeConstantPool.cpp revision 321369 
1//===-- X86ShuffleDecodeConstantPool.cpp - X86 shuffle decode -------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// Define several functions to decode x86 specific shuffle semantics using
11// constants from the constant pool.
12//
13//===----------------------------------------------------------------------===//
14
15#include "X86ShuffleDecodeConstantPool.h"
16#include "Utils/X86ShuffleDecode.h"
17#include "llvm/ADT/APInt.h"
18#include "llvm/CodeGen/MachineValueType.h"
19#include "llvm/IR/Constants.h"
20
21//===----------------------------------------------------------------------===//
22//  Vector Mask Decoding
23//===----------------------------------------------------------------------===//
24
25namespace llvm {
26
27static bool extractConstantMask(const Constant *C, unsigned MaskEltSizeInBits,
28                                APInt &UndefElts,
29                                SmallVectorImpl<uint64_t> &RawMask) {
30  // It is not an error for shuffle masks to not be a vector of
31  // MaskEltSizeInBits because the constant pool uniques constants by their
32  // bit representation.
33  // e.g. the following take up the same space in the constant pool:
34  //   i128 -170141183420855150465331762880109871104
35  //
36  //   <2 x i64> <i64 -9223372034707292160, i64 -9223372034707292160>
37  //
38  //   <4 x i32> <i32 -2147483648, i32 -2147483648,
39  //              i32 -2147483648, i32 -2147483648>
40  Type *CstTy = C->getType();
41  if (!CstTy->isVectorTy())
42    return false;
43
44  Type *CstEltTy = CstTy->getVectorElementType();
45  if (!CstEltTy->isIntegerTy())
46    return false;
47
48  unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits();
49  unsigned CstEltSizeInBits = CstTy->getScalarSizeInBits();
50  unsigned NumCstElts = CstTy->getVectorNumElements();
51
52  assert((CstSizeInBits % MaskEltSizeInBits) == 0 &&
53         "Unaligned shuffle mask size");
54
55  unsigned NumMaskElts = CstSizeInBits / MaskEltSizeInBits;
56  UndefElts = APInt(NumMaskElts, 0);
57  RawMask.resize(NumMaskElts, 0);
58
59  // Fast path - if the constants match the mask size then copy direct.
60  if (MaskEltSizeInBits == CstEltSizeInBits) {
61    assert(NumCstElts == NumMaskElts && "Unaligned shuffle mask size");
62    for (unsigned i = 0; i != NumMaskElts; ++i) {
63      Constant *COp = C->getAggregateElement(i);
64      if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
65        return false;
66
67      if (isa<UndefValue>(COp)) {
68        UndefElts.setBit(i);
69        RawMask[i] = 0;
70        continue;
71      }
72
73      auto *Elt = cast<ConstantInt>(COp);
74      RawMask[i] = Elt->getValue().getZExtValue();
75    }
76    return true;
77  }
78
79  // Extract all the undef/constant element data and pack into single bitsets.
80  APInt UndefBits(CstSizeInBits, 0);
81  APInt MaskBits(CstSizeInBits, 0);
82  for (unsigned i = 0; i != NumCstElts; ++i) {
83    Constant *COp = C->getAggregateElement(i);
84    if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
85      return false;
86
87    unsigned BitOffset = i * CstEltSizeInBits;
88
89    if (isa<UndefValue>(COp)) {
90      UndefBits.setBits(BitOffset, BitOffset + CstEltSizeInBits);
91      continue;
92    }
93
94    MaskBits.insertBits(cast<ConstantInt>(COp)->getValue(), BitOffset);
95  }
96
97  // Now extract the undef/constant bit data into the raw shuffle masks.
98  for (unsigned i = 0; i != NumMaskElts; ++i) {
99    unsigned BitOffset = i * MaskEltSizeInBits;
100    APInt EltUndef = UndefBits.extractBits(MaskEltSizeInBits, BitOffset);
101
102    // Only treat the element as UNDEF if all bits are UNDEF, otherwise
103    // treat it as zero.
104    if (EltUndef.isAllOnesValue()) {
105      UndefElts.setBit(i);
106      RawMask[i] = 0;
107      continue;
108    }
109
110    APInt EltBits = MaskBits.extractBits(MaskEltSizeInBits, BitOffset);
111    RawMask[i] = EltBits.getZExtValue();
112  }
113
114  return true;
115}
116
117void DecodePSHUFBMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask) {
118  Type *MaskTy = C->getType();
119  unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
120  (void)MaskTySize;
121  assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) &&
122         "Unexpected vector size.");
123
124  // The shuffle mask requires a byte vector.
125  APInt UndefElts;
126  SmallVector<uint64_t, 64> RawMask;
127  if (!extractConstantMask(C, 8, UndefElts, RawMask))
128    return;
129
130  unsigned NumElts = RawMask.size();
131  assert((NumElts == 16 || NumElts == 32 || NumElts == 64) &&
132         "Unexpected number of vector elements.");
133
134  for (unsigned i = 0; i != NumElts; ++i) {
135    if (UndefElts[i]) {
136      ShuffleMask.push_back(SM_SentinelUndef);
137      continue;
138    }
139
140    uint64_t Element = RawMask[i];
141    // If the high bit (7) of the byte is set, the element is zeroed.
142    if (Element & (1 << 7))
143      ShuffleMask.push_back(SM_SentinelZero);
144    else {
145      // For AVX vectors with 32 bytes the base of the shuffle is the 16-byte
146      // lane of the vector we're inside.
147      unsigned Base = i & ~0xf;
148
149      // Only the least significant 4 bits of the byte are used.
150      int Index = Base + (Element & 0xf);
151      ShuffleMask.push_back(Index);
152    }
153  }
154}
155
156void DecodeVPERMILPMask(const Constant *C, unsigned ElSize,
157                        SmallVectorImpl<int> &ShuffleMask) {
158  Type *MaskTy = C->getType();
159  unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
160  (void)MaskTySize;
161  assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) &&
162         "Unexpected vector size.");
163  assert((ElSize == 32 || ElSize == 64) && "Unexpected vector element size.");
164
165  // The shuffle mask requires elements the same size as the target.
166  APInt UndefElts;
167  SmallVector<uint64_t, 16> RawMask;
168  if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
169    return;
170
171  unsigned NumElts = RawMask.size();
172  unsigned NumEltsPerLane = 128 / ElSize;
173  assert((NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) &&
174         "Unexpected number of vector elements.");
175
176  for (unsigned i = 0; i != NumElts; ++i) {
177    if (UndefElts[i]) {
178      ShuffleMask.push_back(SM_SentinelUndef);
179      continue;
180    }
181
182    int Index = i & ~(NumEltsPerLane - 1);
183    uint64_t Element = RawMask[i];
184    if (ElSize == 64)
185      Index += (Element >> 1) & 0x1;
186    else
187      Index += Element & 0x3;
188
189    ShuffleMask.push_back(Index);
190  }
191}
192
193void DecodeVPERMIL2PMask(const Constant *C, unsigned M2Z, unsigned ElSize,
194                         SmallVectorImpl<int> &ShuffleMask) {
195  Type *MaskTy = C->getType();
196  unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
197  (void)MaskTySize;
198  assert((MaskTySize == 128 || MaskTySize == 256) && "Unexpected vector size.");
199
200  // The shuffle mask requires elements the same size as the target.
201  APInt UndefElts;
202  SmallVector<uint64_t, 8> RawMask;
203  if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
204    return;
205
206  unsigned NumElts = RawMask.size();
207  unsigned NumEltsPerLane = 128 / ElSize;
208  assert((NumElts == 2 || NumElts == 4 || NumElts == 8) &&
209         "Unexpected number of vector elements.");
210
211  for (unsigned i = 0; i != NumElts; ++i) {
212    if (UndefElts[i]) {
213      ShuffleMask.push_back(SM_SentinelUndef);
214      continue;
215    }
216
217    // VPERMIL2 Operation.
218    // Bits[3] - Match Bit.
219    // Bits[2:1] - (Per Lane) PD Shuffle Mask.
220    // Bits[2:0] - (Per Lane) PS Shuffle Mask.
221    uint64_t Selector = RawMask[i];
222    unsigned MatchBit = (Selector >> 3) & 0x1;
223
224    // M2Z[0:1]     MatchBit
225    //   0Xb           X        Source selected by Selector index.
226    //   10b           0        Source selected by Selector index.
227    //   10b           1        Zero.
228    //   11b           0        Zero.
229    //   11b           1        Source selected by Selector index.
230    if ((M2Z & 0x2) != 0u && MatchBit != (M2Z & 0x1)) {
231      ShuffleMask.push_back(SM_SentinelZero);
232      continue;
233    }
234
235    int Index = i & ~(NumEltsPerLane - 1);
236    if (ElSize == 64)
237      Index += (Selector >> 1) & 0x1;
238    else
239      Index += Selector & 0x3;
240
241    int Src = (Selector >> 2) & 0x1;
242    Index += Src * NumElts;
243    ShuffleMask.push_back(Index);
244  }
245}
246
247void DecodeVPPERMMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask) {
248  assert(C->getType()->getPrimitiveSizeInBits() == 128 &&
249         "Unexpected vector size.");
250
251  // The shuffle mask requires a byte vector.
252  APInt UndefElts;
253  SmallVector<uint64_t, 16> RawMask;
254  if (!extractConstantMask(C, 8, UndefElts, RawMask))
255    return;
256
257  unsigned NumElts = RawMask.size();
258  assert(NumElts == 16 && "Unexpected number of vector elements.");
259
260  for (unsigned i = 0; i != NumElts; ++i) {
261    if (UndefElts[i]) {
262      ShuffleMask.push_back(SM_SentinelUndef);
263      continue;
264    }
265
266    // VPPERM Operation
267    // Bits[4:0] - Byte Index (0 - 31)
268    // Bits[7:5] - Permute Operation
269    //
270    // Permute Operation:
271    // 0 - Source byte (no logical operation).
272    // 1 - Invert source byte.
273    // 2 - Bit reverse of source byte.
274    // 3 - Bit reverse of inverted source byte.
275    // 4 - 00h (zero - fill).
276    // 5 - FFh (ones - fill).
277    // 6 - Most significant bit of source byte replicated in all bit positions.
278    // 7 - Invert most significant bit of source byte and replicate in all bit
279    // positions.
280    uint64_t Element = RawMask[i];
281    uint64_t Index = Element & 0x1F;
282    uint64_t PermuteOp = (Element >> 5) & 0x7;
283
284    if (PermuteOp == 4) {
285      ShuffleMask.push_back(SM_SentinelZero);
286      continue;
287    }
288    if (PermuteOp != 0) {
289      ShuffleMask.clear();
290      return;
291    }
292    ShuffleMask.push_back((int)Index);
293  }
294}
295
296void DecodeVPERMVMask(const Constant *C, unsigned ElSize,
297                      SmallVectorImpl<int> &ShuffleMask) {
298  Type *MaskTy = C->getType();
299  unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
300  (void)MaskTySize;
301  assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) &&
302         "Unexpected vector size.");
303  assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
304         "Unexpected vector element size.");
305
306  // The shuffle mask requires elements the same size as the target.
307  APInt UndefElts;
308  SmallVector<uint64_t, 64> RawMask;
309  if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
310    return;
311
312  unsigned NumElts = RawMask.size();
313
314  for (unsigned i = 0; i != NumElts; ++i) {
315    if (UndefElts[i]) {
316      ShuffleMask.push_back(SM_SentinelUndef);
317      continue;
318    }
319    int Index = RawMask[i] & (NumElts - 1);
320    ShuffleMask.push_back(Index);
321  }
322}
323
324void DecodeVPERMV3Mask(const Constant *C, unsigned ElSize,
325                       SmallVectorImpl<int> &ShuffleMask) {
326  Type *MaskTy = C->getType();
327  unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
328  (void)MaskTySize;
329  assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) &&
330         "Unexpected vector size.");
331  assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
332         "Unexpected vector element size.");
333
334  // The shuffle mask requires elements the same size as the target.
335  APInt UndefElts;
336  SmallVector<uint64_t, 64> RawMask;
337  if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
338    return;
339
340  unsigned NumElts = RawMask.size();
341
342  for (unsigned i = 0; i != NumElts; ++i) {
343    if (UndefElts[i]) {
344      ShuffleMask.push_back(SM_SentinelUndef);
345      continue;
346    }
347    int Index = RawMask[i] & (NumElts*2 - 1);
348    ShuffleMask.push_back(Index);
349  }
350}
351} // llvm namespace
352