IRBuilder.cpp revision 360784 
1//===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the IRBuilder class, which is used as a convenient way
10// to create LLVM instructions with a consistent and simplified interface.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/IR/IRBuilder.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/None.h"
17#include "llvm/IR/Constant.h"
18#include "llvm/IR/Constants.h"
19#include "llvm/IR/DerivedTypes.h"
20#include "llvm/IR/Function.h"
21#include "llvm/IR/GlobalValue.h"
22#include "llvm/IR/GlobalVariable.h"
23#include "llvm/IR/IntrinsicInst.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/LLVMContext.h"
26#include "llvm/IR/Operator.h"
27#include "llvm/IR/Statepoint.h"
28#include "llvm/IR/Type.h"
29#include "llvm/IR/Value.h"
30#include "llvm/Support/Casting.h"
31#include "llvm/Support/MathExtras.h"
32#include <cassert>
33#include <cstdint>
34#include <vector>
35
36using namespace llvm;
37
38/// CreateGlobalString - Make a new global variable with an initializer that
39/// has array of i8 type filled in with the nul terminated string value
40/// specified.  If Name is specified, it is the name of the global variable
41/// created.
42GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
43                                                  const Twine &Name,
44                                                  unsigned AddressSpace) {
45  Constant *StrConstant = ConstantDataArray::getString(Context, Str);
46  Module &M = *BB->getParent()->getParent();
47  auto *GV = new GlobalVariable(M, StrConstant->getType(), true,
48                                GlobalValue::PrivateLinkage, StrConstant, Name,
49                                nullptr, GlobalVariable::NotThreadLocal,
50                                AddressSpace);
51  GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
52  GV->setAlignment(Align::None());
53  return GV;
54}
55
56Type *IRBuilderBase::getCurrentFunctionReturnType() const {
57  assert(BB && BB->getParent() && "No current function!");
58  return BB->getParent()->getReturnType();
59}
60
61Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) {
62  auto *PT = cast<PointerType>(Ptr->getType());
63  if (PT->getElementType()->isIntegerTy(8))
64    return Ptr;
65
66  // Otherwise, we need to insert a bitcast.
67  PT = getInt8PtrTy(PT->getAddressSpace());
68  BitCastInst *BCI = new BitCastInst(Ptr, PT, "");
69  BB->getInstList().insert(InsertPt, BCI);
70  SetInstDebugLocation(BCI);
71  return BCI;
72}
73
74static CallInst *createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
75                                  IRBuilderBase *Builder,
76                                  const Twine &Name = "",
77                                  Instruction *FMFSource = nullptr) {
78  CallInst *CI = CallInst::Create(Callee, Ops, Name);
79  if (FMFSource)
80    CI->copyFastMathFlags(FMFSource);
81  Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),CI);
82  Builder->SetInstDebugLocation(CI);
83  return CI;
84}
85
86static InvokeInst *createInvokeHelper(Function *Invokee, BasicBlock *NormalDest,
87                                      BasicBlock *UnwindDest,
88                                      ArrayRef<Value *> Ops,
89                                      IRBuilderBase *Builder,
90                                      const Twine &Name = "") {
91  InvokeInst *II =
92      InvokeInst::Create(Invokee, NormalDest, UnwindDest, Ops, Name);
93  Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),
94                                                  II);
95  Builder->SetInstDebugLocation(II);
96  return II;
97}
98
99CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
100                                      MaybeAlign Align, bool isVolatile,
101                                      MDNode *TBAATag, MDNode *ScopeTag,
102                                      MDNode *NoAliasTag) {
103  Ptr = getCastedInt8PtrValue(Ptr);
104  Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
105  Type *Tys[] = { Ptr->getType(), Size->getType() };
106  Module *M = BB->getParent()->getParent();
107  Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
108
109  CallInst *CI = createCallHelper(TheFn, Ops, this);
110
111  if (Align)
112    cast<MemSetInst>(CI)->setDestAlignment(Align->value());
113
114  // Set the TBAA info if present.
115  if (TBAATag)
116    CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
117
118  if (ScopeTag)
119    CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
120
121  if (NoAliasTag)
122    CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
123
124  return CI;
125}
126
127CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
128    Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize,
129    MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) {
130
131  Ptr = getCastedInt8PtrValue(Ptr);
132  Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
133  Type *Tys[] = {Ptr->getType(), Size->getType()};
134  Module *M = BB->getParent()->getParent();
135  Function *TheFn = Intrinsic::getDeclaration(
136      M, Intrinsic::memset_element_unordered_atomic, Tys);
137
138  CallInst *CI = createCallHelper(TheFn, Ops, this);
139
140  cast<AtomicMemSetInst>(CI)->setDestAlignment(Alignment);
141
142  // Set the TBAA info if present.
143  if (TBAATag)
144    CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
145
146  if (ScopeTag)
147    CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
148
149  if (NoAliasTag)
150    CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
151
152  return CI;
153}
154
155CallInst *IRBuilderBase::CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src,
156                                      unsigned SrcAlign, Value *Size,
157                                      bool isVolatile, MDNode *TBAATag,
158                                      MDNode *TBAAStructTag, MDNode *ScopeTag,
159                                      MDNode *NoAliasTag) {
160  return CreateMemCpy(Dst, MaybeAlign(DstAlign), Src, MaybeAlign(SrcAlign),
161                      Size, isVolatile, TBAATag, TBAAStructTag, ScopeTag,
162                      NoAliasTag);
163}
164
165CallInst *IRBuilderBase::CreateMemCpy(Value *Dst, MaybeAlign DstAlign,
166                                      Value *Src, MaybeAlign SrcAlign,
167                                      Value *Size, bool isVolatile,
168                                      MDNode *TBAATag, MDNode *TBAAStructTag,
169                                      MDNode *ScopeTag, MDNode *NoAliasTag) {
170  Dst = getCastedInt8PtrValue(Dst);
171  Src = getCastedInt8PtrValue(Src);
172
173  Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
174  Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
175  Module *M = BB->getParent()->getParent();
176  Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy, Tys);
177
178  CallInst *CI = createCallHelper(TheFn, Ops, this);
179
180  auto* MCI = cast<MemCpyInst>(CI);
181  if (DstAlign)
182    MCI->setDestAlignment(*DstAlign);
183  if (SrcAlign)
184    MCI->setSourceAlignment(*SrcAlign);
185
186  // Set the TBAA info if present.
187  if (TBAATag)
188    CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
189
190  // Set the TBAA Struct info if present.
191  if (TBAAStructTag)
192    CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
193
194  if (ScopeTag)
195    CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
196
197  if (NoAliasTag)
198    CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
199
200  return CI;
201}
202
203CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
204    Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size,
205    uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
206    MDNode *ScopeTag, MDNode *NoAliasTag) {
207  assert(DstAlign >= ElementSize &&
208         "Pointer alignment must be at least element size");
209  assert(SrcAlign >= ElementSize &&
210         "Pointer alignment must be at least element size");
211  Dst = getCastedInt8PtrValue(Dst);
212  Src = getCastedInt8PtrValue(Src);
213
214  Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
215  Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
216  Module *M = BB->getParent()->getParent();
217  Function *TheFn = Intrinsic::getDeclaration(
218      M, Intrinsic::memcpy_element_unordered_atomic, Tys);
219
220  CallInst *CI = createCallHelper(TheFn, Ops, this);
221
222  // Set the alignment of the pointer args.
223  auto *AMCI = cast<AtomicMemCpyInst>(CI);
224  AMCI->setDestAlignment(DstAlign);
225  AMCI->setSourceAlignment(SrcAlign);
226
227  // Set the TBAA info if present.
228  if (TBAATag)
229    CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
230
231  // Set the TBAA Struct info if present.
232  if (TBAAStructTag)
233    CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
234
235  if (ScopeTag)
236    CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
237
238  if (NoAliasTag)
239    CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
240
241  return CI;
242}
243
244CallInst *IRBuilderBase::CreateMemMove(Value *Dst, MaybeAlign DstAlign,
245                                       Value *Src, MaybeAlign SrcAlign,
246                                       Value *Size, bool isVolatile,
247                                       MDNode *TBAATag, MDNode *ScopeTag,
248                                       MDNode *NoAliasTag) {
249  Dst = getCastedInt8PtrValue(Dst);
250  Src = getCastedInt8PtrValue(Src);
251
252  Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
253  Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
254  Module *M = BB->getParent()->getParent();
255  Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
256
257  CallInst *CI = createCallHelper(TheFn, Ops, this);
258
259  auto *MMI = cast<MemMoveInst>(CI);
260  if (DstAlign)
261    MMI->setDestAlignment(*DstAlign);
262  if (SrcAlign)
263    MMI->setSourceAlignment(*SrcAlign);
264
265  // Set the TBAA info if present.
266  if (TBAATag)
267    CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
268
269  if (ScopeTag)
270    CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
271
272  if (NoAliasTag)
273    CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
274
275  return CI;
276}
277
278CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
279    Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size,
280    uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
281    MDNode *ScopeTag, MDNode *NoAliasTag) {
282  assert(DstAlign >= ElementSize &&
283         "Pointer alignment must be at least element size");
284  assert(SrcAlign >= ElementSize &&
285         "Pointer alignment must be at least element size");
286  Dst = getCastedInt8PtrValue(Dst);
287  Src = getCastedInt8PtrValue(Src);
288
289  Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
290  Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
291  Module *M = BB->getParent()->getParent();
292  Function *TheFn = Intrinsic::getDeclaration(
293      M, Intrinsic::memmove_element_unordered_atomic, Tys);
294
295  CallInst *CI = createCallHelper(TheFn, Ops, this);
296
297  // Set the alignment of the pointer args.
298  CI->addParamAttr(
299      0, Attribute::getWithAlignment(CI->getContext(), Align(DstAlign)));
300  CI->addParamAttr(
301      1, Attribute::getWithAlignment(CI->getContext(), Align(SrcAlign)));
302
303  // Set the TBAA info if present.
304  if (TBAATag)
305    CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
306
307  // Set the TBAA Struct info if present.
308  if (TBAAStructTag)
309    CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
310
311  if (ScopeTag)
312    CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
313
314  if (NoAliasTag)
315    CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
316
317  return CI;
318}
319
320static CallInst *getReductionIntrinsic(IRBuilderBase *Builder, Intrinsic::ID ID,
321                                    Value *Src) {
322  Module *M = Builder->GetInsertBlock()->getParent()->getParent();
323  Value *Ops[] = {Src};
324  Type *Tys[] = { Src->getType() };
325  auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
326  return createCallHelper(Decl, Ops, Builder);
327}
328
329CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
330  Module *M = GetInsertBlock()->getParent()->getParent();
331  Value *Ops[] = {Acc, Src};
332  Type *Tys[] = {Acc->getType(), Src->getType()};
333  auto Decl = Intrinsic::getDeclaration(
334      M, Intrinsic::experimental_vector_reduce_v2_fadd, Tys);
335  return createCallHelper(Decl, Ops, this);
336}
337
338CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
339  Module *M = GetInsertBlock()->getParent()->getParent();
340  Value *Ops[] = {Acc, Src};
341  Type *Tys[] = {Acc->getType(), Src->getType()};
342  auto Decl = Intrinsic::getDeclaration(
343      M, Intrinsic::experimental_vector_reduce_v2_fmul, Tys);
344  return createCallHelper(Decl, Ops, this);
345}
346
347CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {
348  return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_add,
349                               Src);
350}
351
352CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {
353  return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_mul,
354                               Src);
355}
356
357CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {
358  return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_and,
359                               Src);
360}
361
362CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {
363  return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_or,
364                               Src);
365}
366
367CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {
368  return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_xor,
369                               Src);
370}
371
372CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
373  auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smax
374                     : Intrinsic::experimental_vector_reduce_umax;
375  return getReductionIntrinsic(this, ID, Src);
376}
377
378CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
379  auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smin
380                     : Intrinsic::experimental_vector_reduce_umin;
381  return getReductionIntrinsic(this, ID, Src);
382}
383
384CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src, bool NoNaN) {
385  auto Rdx = getReductionIntrinsic(
386      this, Intrinsic::experimental_vector_reduce_fmax, Src);
387  if (NoNaN) {
388    FastMathFlags FMF;
389    FMF.setNoNaNs();
390    Rdx->setFastMathFlags(FMF);
391  }
392  return Rdx;
393}
394
395CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src, bool NoNaN) {
396  auto Rdx = getReductionIntrinsic(
397      this, Intrinsic::experimental_vector_reduce_fmin, Src);
398  if (NoNaN) {
399    FastMathFlags FMF;
400    FMF.setNoNaNs();
401    Rdx->setFastMathFlags(FMF);
402  }
403  return Rdx;
404}
405
406CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
407  assert(isa<PointerType>(Ptr->getType()) &&
408         "lifetime.start only applies to pointers.");
409  Ptr = getCastedInt8PtrValue(Ptr);
410  if (!Size)
411    Size = getInt64(-1);
412  else
413    assert(Size->getType() == getInt64Ty() &&
414           "lifetime.start requires the size to be an i64");
415  Value *Ops[] = { Size, Ptr };
416  Module *M = BB->getParent()->getParent();
417  Function *TheFn =
418      Intrinsic::getDeclaration(M, Intrinsic::lifetime_start, {Ptr->getType()});
419  return createCallHelper(TheFn, Ops, this);
420}
421
422CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
423  assert(isa<PointerType>(Ptr->getType()) &&
424         "lifetime.end only applies to pointers.");
425  Ptr = getCastedInt8PtrValue(Ptr);
426  if (!Size)
427    Size = getInt64(-1);
428  else
429    assert(Size->getType() == getInt64Ty() &&
430           "lifetime.end requires the size to be an i64");
431  Value *Ops[] = { Size, Ptr };
432  Module *M = BB->getParent()->getParent();
433  Function *TheFn =
434      Intrinsic::getDeclaration(M, Intrinsic::lifetime_end, {Ptr->getType()});
435  return createCallHelper(TheFn, Ops, this);
436}
437
438CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
439
440  assert(isa<PointerType>(Ptr->getType()) &&
441         "invariant.start only applies to pointers.");
442  Ptr = getCastedInt8PtrValue(Ptr);
443  if (!Size)
444    Size = getInt64(-1);
445  else
446    assert(Size->getType() == getInt64Ty() &&
447           "invariant.start requires the size to be an i64");
448
449  Value *Ops[] = {Size, Ptr};
450  // Fill in the single overloaded type: memory object type.
451  Type *ObjectPtr[1] = {Ptr->getType()};
452  Module *M = BB->getParent()->getParent();
453  Function *TheFn =
454      Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
455  return createCallHelper(TheFn, Ops, this);
456}
457
458CallInst *IRBuilderBase::CreateAssumption(Value *Cond) {
459  assert(Cond->getType() == getInt1Ty() &&
460         "an assumption condition must be of type i1");
461
462  Value *Ops[] = { Cond };
463  Module *M = BB->getParent()->getParent();
464  Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
465  return createCallHelper(FnAssume, Ops, this);
466}
467
468/// Create a call to a Masked Load intrinsic.
469/// \p Ptr      - base pointer for the load
470/// \p Align    - alignment of the source location
471/// \p Mask     - vector of booleans which indicates what vector lanes should
472///               be accessed in memory
473/// \p PassThru - pass-through value that is used to fill the masked-off lanes
474///               of the result
475/// \p Name     - name of the result variable
476CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, unsigned Align,
477                                          Value *Mask, Value *PassThru,
478                                          const Twine &Name) {
479  auto *PtrTy = cast<PointerType>(Ptr->getType());
480  Type *DataTy = PtrTy->getElementType();
481  assert(DataTy->isVectorTy() && "Ptr should point to a vector");
482  assert(Mask && "Mask should not be all-ones (null)");
483  if (!PassThru)
484    PassThru = UndefValue::get(DataTy);
485  Type *OverloadedTypes[] = { DataTy, PtrTy };
486  Value *Ops[] = { Ptr, getInt32(Align), Mask,  PassThru};
487  return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
488                               OverloadedTypes, Name);
489}
490
491/// Create a call to a Masked Store intrinsic.
492/// \p Val   - data to be stored,
493/// \p Ptr   - base pointer for the store
494/// \p Align - alignment of the destination location
495/// \p Mask  - vector of booleans which indicates what vector lanes should
496///            be accessed in memory
497CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
498                                           unsigned Align, Value *Mask) {
499  auto *PtrTy = cast<PointerType>(Ptr->getType());
500  Type *DataTy = PtrTy->getElementType();
501  assert(DataTy->isVectorTy() && "Ptr should point to a vector");
502  assert(Mask && "Mask should not be all-ones (null)");
503  Type *OverloadedTypes[] = { DataTy, PtrTy };
504  Value *Ops[] = { Val, Ptr, getInt32(Align), Mask };
505  return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
506}
507
508/// Create a call to a Masked intrinsic, with given intrinsic Id,
509/// an array of operands - Ops, and an array of overloaded types -
510/// OverloadedTypes.
511CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
512                                               ArrayRef<Value *> Ops,
513                                               ArrayRef<Type *> OverloadedTypes,
514                                               const Twine &Name) {
515  Module *M = BB->getParent()->getParent();
516  Function *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
517  return createCallHelper(TheFn, Ops, this, Name);
518}
519
520/// Create a call to a Masked Gather intrinsic.
521/// \p Ptrs     - vector of pointers for loading
522/// \p Align    - alignment for one element
523/// \p Mask     - vector of booleans which indicates what vector lanes should
524///               be accessed in memory
525/// \p PassThru - pass-through value that is used to fill the masked-off lanes
526///               of the result
527/// \p Name     - name of the result variable
528CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, unsigned Align,
529                                            Value *Mask,  Value *PassThru,
530                                            const Twine& Name) {
531  auto PtrsTy = cast<VectorType>(Ptrs->getType());
532  auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
533  unsigned NumElts = PtrsTy->getVectorNumElements();
534  Type *DataTy = VectorType::get(PtrTy->getElementType(), NumElts);
535
536  if (!Mask)
537    Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
538                                     NumElts));
539
540  if (!PassThru)
541    PassThru = UndefValue::get(DataTy);
542
543  Type *OverloadedTypes[] = {DataTy, PtrsTy};
544  Value * Ops[] = {Ptrs, getInt32(Align), Mask, PassThru};
545
546  // We specify only one type when we create this intrinsic. Types of other
547  // arguments are derived from this type.
548  return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes,
549                               Name);
550}
551
552/// Create a call to a Masked Scatter intrinsic.
553/// \p Data  - data to be stored,
554/// \p Ptrs  - the vector of pointers, where the \p Data elements should be
555///            stored
556/// \p Align - alignment for one element
557/// \p Mask  - vector of booleans which indicates what vector lanes should
558///            be accessed in memory
559CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
560                                             unsigned Align, Value *Mask) {
561  auto PtrsTy = cast<VectorType>(Ptrs->getType());
562  auto DataTy = cast<VectorType>(Data->getType());
563  unsigned NumElts = PtrsTy->getVectorNumElements();
564
565#ifndef NDEBUG
566  auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
567  assert(NumElts == DataTy->getVectorNumElements() &&
568         PtrTy->getElementType() == DataTy->getElementType() &&
569         "Incompatible pointer and data types");
570#endif
571
572  if (!Mask)
573    Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
574                                     NumElts));
575
576  Type *OverloadedTypes[] = {DataTy, PtrsTy};
577  Value * Ops[] = {Data, Ptrs, getInt32(Align), Mask};
578
579  // We specify only one type when we create this intrinsic. Types of other
580  // arguments are derived from this type.
581  return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
582}
583
584template <typename T0, typename T1, typename T2, typename T3>
585static std::vector<Value *>
586getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
587                  Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
588                  ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs,
589                  ArrayRef<T3> GCArgs) {
590  std::vector<Value *> Args;
591  Args.push_back(B.getInt64(ID));
592  Args.push_back(B.getInt32(NumPatchBytes));
593  Args.push_back(ActualCallee);
594  Args.push_back(B.getInt32(CallArgs.size()));
595  Args.push_back(B.getInt32(Flags));
596  Args.insert(Args.end(), CallArgs.begin(), CallArgs.end());
597  Args.push_back(B.getInt32(TransitionArgs.size()));
598  Args.insert(Args.end(), TransitionArgs.begin(), TransitionArgs.end());
599  Args.push_back(B.getInt32(DeoptArgs.size()));
600  Args.insert(Args.end(), DeoptArgs.begin(), DeoptArgs.end());
601  Args.insert(Args.end(), GCArgs.begin(), GCArgs.end());
602
603  return Args;
604}
605
606template <typename T0, typename T1, typename T2, typename T3>
607static CallInst *CreateGCStatepointCallCommon(
608    IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
609    Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
610    ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs,
611    const Twine &Name) {
612  // Extract out the type of the callee.
613  auto *FuncPtrType = cast<PointerType>(ActualCallee->getType());
614  assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
615         "actual callee must be a callable value");
616
617  Module *M = Builder->GetInsertBlock()->getParent()->getParent();
618  // Fill in the one generic type'd argument (the function is also vararg)
619  Type *ArgTypes[] = { FuncPtrType };
620  Function *FnStatepoint =
621    Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
622                              ArgTypes);
623
624  std::vector<Value *> Args =
625      getStatepointArgs(*Builder, ID, NumPatchBytes, ActualCallee, Flags,
626                        CallArgs, TransitionArgs, DeoptArgs, GCArgs);
627  return createCallHelper(FnStatepoint, Args, Builder, Name);
628}
629
630CallInst *IRBuilderBase::CreateGCStatepointCall(
631    uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
632    ArrayRef<Value *> CallArgs, ArrayRef<Value *> DeoptArgs,
633    ArrayRef<Value *> GCArgs, const Twine &Name) {
634  return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
635      this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
636      CallArgs, None /* No Transition Args */, DeoptArgs, GCArgs, Name);
637}
638
639CallInst *IRBuilderBase::CreateGCStatepointCall(
640    uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags,
641    ArrayRef<Use> CallArgs, ArrayRef<Use> TransitionArgs,
642    ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
643  return CreateGCStatepointCallCommon<Use, Use, Use, Value *>(
644      this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
645      DeoptArgs, GCArgs, Name);
646}
647
648CallInst *IRBuilderBase::CreateGCStatepointCall(
649    uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
650    ArrayRef<Use> CallArgs, ArrayRef<Value *> DeoptArgs,
651    ArrayRef<Value *> GCArgs, const Twine &Name) {
652  return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
653      this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
654      CallArgs, None, DeoptArgs, GCArgs, Name);
655}
656
657template <typename T0, typename T1, typename T2, typename T3>
658static InvokeInst *CreateGCStatepointInvokeCommon(
659    IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
660    Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest,
661    uint32_t Flags, ArrayRef<T0> InvokeArgs, ArrayRef<T1> TransitionArgs,
662    ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs, const Twine &Name) {
663  // Extract out the type of the callee.
664  auto *FuncPtrType = cast<PointerType>(ActualInvokee->getType());
665  assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
666         "actual callee must be a callable value");
667
668  Module *M = Builder->GetInsertBlock()->getParent()->getParent();
669  // Fill in the one generic type'd argument (the function is also vararg)
670  Function *FnStatepoint = Intrinsic::getDeclaration(
671      M, Intrinsic::experimental_gc_statepoint, {FuncPtrType});
672
673  std::vector<Value *> Args =
674      getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee, Flags,
675                        InvokeArgs, TransitionArgs, DeoptArgs, GCArgs);
676  return createInvokeHelper(FnStatepoint, NormalDest, UnwindDest, Args, Builder,
677                            Name);
678}
679
680InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
681    uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
682    BasicBlock *NormalDest, BasicBlock *UnwindDest,
683    ArrayRef<Value *> InvokeArgs, ArrayRef<Value *> DeoptArgs,
684    ArrayRef<Value *> GCArgs, const Twine &Name) {
685  return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
686      this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
687      uint32_t(StatepointFlags::None), InvokeArgs, None /* No Transition Args*/,
688      DeoptArgs, GCArgs, Name);
689}
690
691InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
692    uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
693    BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
694    ArrayRef<Use> InvokeArgs, ArrayRef<Use> TransitionArgs,
695    ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
696  return CreateGCStatepointInvokeCommon<Use, Use, Use, Value *>(
697      this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
698      InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
699}
700
701InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
702    uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
703    BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
704    ArrayRef<Value *> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
705  return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
706      this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
707      uint32_t(StatepointFlags::None), InvokeArgs, None, DeoptArgs, GCArgs,
708      Name);
709}
710
711CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
712                                       Type *ResultType,
713                                       const Twine &Name) {
714 Intrinsic::ID ID = Intrinsic::experimental_gc_result;
715 Module *M = BB->getParent()->getParent();
716 Type *Types[] = {ResultType};
717 Function *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
718
719 Value *Args[] = {Statepoint};
720 return createCallHelper(FnGCResult, Args, this, Name);
721}
722
723CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
724                                         int BaseOffset,
725                                         int DerivedOffset,
726                                         Type *ResultType,
727                                         const Twine &Name) {
728 Module *M = BB->getParent()->getParent();
729 Type *Types[] = {ResultType};
730 Function *FnGCRelocate =
731     Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
732
733 Value *Args[] = {Statepoint,
734                  getInt32(BaseOffset),
735                  getInt32(DerivedOffset)};
736 return createCallHelper(FnGCRelocate, Args, this, Name);
737}
738
739CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
740                                              Instruction *FMFSource,
741                                              const Twine &Name) {
742  Module *M = BB->getModule();
743  Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()});
744  return createCallHelper(Fn, {V}, this, Name, FMFSource);
745}
746
747CallInst *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
748                                               Value *RHS,
749                                               Instruction *FMFSource,
750                                               const Twine &Name) {
751  Module *M = BB->getModule();
752  Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
753  return createCallHelper(Fn, {LHS, RHS}, this, Name, FMFSource);
754}
755
756CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
757                                         ArrayRef<Type *> Types,
758                                         ArrayRef<Value *> Args,
759                                         Instruction *FMFSource,
760                                         const Twine &Name) {
761  Module *M = BB->getModule();
762  Function *Fn = Intrinsic::getDeclaration(M, ID, Types);
763  return createCallHelper(Fn, Args, this, Name, FMFSource);
764}
765