1//===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the IRBuilder class, which is used as a convenient way
10// to create LLVM instructions with a consistent and simplified interface.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/IR/IRBuilder.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/IR/Constant.h"
17#include "llvm/IR/Constants.h"
18#include "llvm/IR/DebugInfoMetadata.h"
19#include "llvm/IR/DerivedTypes.h"
20#include "llvm/IR/Function.h"
21#include "llvm/IR/GlobalValue.h"
22#include "llvm/IR/GlobalVariable.h"
23#include "llvm/IR/IntrinsicInst.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/LLVMContext.h"
26#include "llvm/IR/NoFolder.h"
27#include "llvm/IR/Operator.h"
28#include "llvm/IR/Statepoint.h"
29#include "llvm/IR/Type.h"
30#include "llvm/IR/Value.h"
31#include "llvm/Support/Casting.h"
32#include <cassert>
33#include <cstdint>
34#include <optional>
35#include <vector>
36
37using namespace llvm;
38
39/// CreateGlobalString - Make a new global variable with an initializer that
40/// has array of i8 type filled in with the nul terminated string value
41/// specified.  If Name is specified, it is the name of the global variable
42/// created.
43GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
44                                                  const Twine &Name,
45                                                  unsigned AddressSpace,
46                                                  Module *M) {
47  Constant *StrConstant = ConstantDataArray::getString(Context, Str);
48  if (!M)
49    M = BB->getParent()->getParent();
50  auto *GV = new GlobalVariable(
51      *M, StrConstant->getType(), true, GlobalValue::PrivateLinkage,
52      StrConstant, Name, nullptr, GlobalVariable::NotThreadLocal, AddressSpace);
53  GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
54  GV->setAlignment(Align(1));
55  return GV;
56}
57
58Type *IRBuilderBase::getCurrentFunctionReturnType() const {
59  assert(BB && BB->getParent() && "No current function!");
60  return BB->getParent()->getReturnType();
61}
62
63Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) {
64  auto *PT = cast<PointerType>(Ptr->getType());
65  if (PT->isOpaqueOrPointeeTypeMatches(getInt8Ty()))
66    return Ptr;
67
68  // Otherwise, we need to insert a bitcast.
69  return CreateBitCast(Ptr, getInt8PtrTy(PT->getAddressSpace()));
70}
71
72DebugLoc IRBuilderBase::getCurrentDebugLocation() const {
73  for (auto &KV : MetadataToCopy)
74    if (KV.first == LLVMContext::MD_dbg)
75      return {cast<DILocation>(KV.second)};
76
77  return {};
78}
79void IRBuilderBase::SetInstDebugLocation(Instruction *I) const {
80  for (const auto &KV : MetadataToCopy)
81    if (KV.first == LLVMContext::MD_dbg) {
82      I->setDebugLoc(DebugLoc(KV.second));
83      return;
84    }
85}
86
87CallInst *
88IRBuilderBase::createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
89                                const Twine &Name, Instruction *FMFSource,
90                                ArrayRef<OperandBundleDef> OpBundles) {
91  CallInst *CI = CreateCall(Callee, Ops, OpBundles, Name);
92  if (FMFSource)
93    CI->copyFastMathFlags(FMFSource);
94  return CI;
95}
96
97Value *IRBuilderBase::CreateVScale(Constant *Scaling, const Twine &Name) {
98  assert(isa<ConstantInt>(Scaling) && "Expected constant integer");
99  if (cast<ConstantInt>(Scaling)->isZero())
100    return Scaling;
101  Module *M = GetInsertBlock()->getParent()->getParent();
102  Function *TheFn =
103      Intrinsic::getDeclaration(M, Intrinsic::vscale, {Scaling->getType()});
104  CallInst *CI = CreateCall(TheFn, {}, {}, Name);
105  return cast<ConstantInt>(Scaling)->getSExtValue() == 1
106             ? CI
107             : CreateMul(CI, Scaling);
108}
109
110Value *IRBuilderBase::CreateStepVector(Type *DstType, const Twine &Name) {
111  Type *STy = DstType->getScalarType();
112  if (isa<ScalableVectorType>(DstType)) {
113    Type *StepVecType = DstType;
114    // TODO: We expect this special case (element type < 8 bits) to be
115    // temporary - once the intrinsic properly supports < 8 bits this code
116    // can be removed.
117    if (STy->getScalarSizeInBits() < 8)
118      StepVecType =
119          VectorType::get(getInt8Ty(), cast<ScalableVectorType>(DstType));
120    Value *Res = CreateIntrinsic(Intrinsic::experimental_stepvector,
121                                 {StepVecType}, {}, nullptr, Name);
122    if (StepVecType != DstType)
123      Res = CreateTrunc(Res, DstType);
124    return Res;
125  }
126
127  unsigned NumEls = cast<FixedVectorType>(DstType)->getNumElements();
128
129  // Create a vector of consecutive numbers from zero to VF.
130  SmallVector<Constant *, 8> Indices;
131  for (unsigned i = 0; i < NumEls; ++i)
132    Indices.push_back(ConstantInt::get(STy, i));
133
134  // Add the consecutive indices to the vector value.
135  return ConstantVector::get(Indices);
136}
137
138CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
139                                      MaybeAlign Align, bool isVolatile,
140                                      MDNode *TBAATag, MDNode *ScopeTag,
141                                      MDNode *NoAliasTag) {
142  Ptr = getCastedInt8PtrValue(Ptr);
143  Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
144  Type *Tys[] = { Ptr->getType(), Size->getType() };
145  Module *M = BB->getParent()->getParent();
146  Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
147
148  CallInst *CI = CreateCall(TheFn, Ops);
149
150  if (Align)
151    cast<MemSetInst>(CI)->setDestAlignment(*Align);
152
153  // Set the TBAA info if present.
154  if (TBAATag)
155    CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
156
157  if (ScopeTag)
158    CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
159
160  if (NoAliasTag)
161    CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
162
163  return CI;
164}
165
166CallInst *IRBuilderBase::CreateMemSetInline(Value *Dst, MaybeAlign DstAlign,
167                                            Value *Val, Value *Size,
168                                            bool IsVolatile, MDNode *TBAATag,
169                                            MDNode *ScopeTag,
170                                            MDNode *NoAliasTag) {
171  Dst = getCastedInt8PtrValue(Dst);
172  Value *Ops[] = {Dst, Val, Size, getInt1(IsVolatile)};
173  Type *Tys[] = {Dst->getType(), Size->getType()};
174  Module *M = BB->getParent()->getParent();
175  Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset_inline, Tys);
176
177  CallInst *CI = CreateCall(TheFn, Ops);
178
179  if (DstAlign)
180    cast<MemSetInlineInst>(CI)->setDestAlignment(*DstAlign);
181
182  // Set the TBAA info if present.
183  if (TBAATag)
184    CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
185
186  if (ScopeTag)
187    CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
188
189  if (NoAliasTag)
190    CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
191
192  return CI;
193}
194
195CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
196    Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize,
197    MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) {
198
199  Ptr = getCastedInt8PtrValue(Ptr);
200  Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
201  Type *Tys[] = {Ptr->getType(), Size->getType()};
202  Module *M = BB->getParent()->getParent();
203  Function *TheFn = Intrinsic::getDeclaration(
204      M, Intrinsic::memset_element_unordered_atomic, Tys);
205
206  CallInst *CI = CreateCall(TheFn, Ops);
207
208  cast<AtomicMemSetInst>(CI)->setDestAlignment(Alignment);
209
210  // Set the TBAA info if present.
211  if (TBAATag)
212    CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
213
214  if (ScopeTag)
215    CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
216
217  if (NoAliasTag)
218    CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
219
220  return CI;
221}
222
223CallInst *IRBuilderBase::CreateMemTransferInst(
224    Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign, Value *Src,
225    MaybeAlign SrcAlign, Value *Size, bool isVolatile, MDNode *TBAATag,
226    MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) {
227  Dst = getCastedInt8PtrValue(Dst);
228  Src = getCastedInt8PtrValue(Src);
229
230  Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
231  Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
232  Module *M = BB->getParent()->getParent();
233  Function *TheFn = Intrinsic::getDeclaration(M, IntrID, Tys);
234
235  CallInst *CI = CreateCall(TheFn, Ops);
236
237  auto* MCI = cast<MemTransferInst>(CI);
238  if (DstAlign)
239    MCI->setDestAlignment(*DstAlign);
240  if (SrcAlign)
241    MCI->setSourceAlignment(*SrcAlign);
242
243  // Set the TBAA info if present.
244  if (TBAATag)
245    CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
246
247  // Set the TBAA Struct info if present.
248  if (TBAAStructTag)
249    CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
250
251  if (ScopeTag)
252    CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
253
254  if (NoAliasTag)
255    CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
256
257  return CI;
258}
259
260CallInst *IRBuilderBase::CreateMemCpyInline(
261    Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign,
262    Value *Size, bool IsVolatile, MDNode *TBAATag, MDNode *TBAAStructTag,
263    MDNode *ScopeTag, MDNode *NoAliasTag) {
264  Dst = getCastedInt8PtrValue(Dst);
265  Src = getCastedInt8PtrValue(Src);
266
267  Value *Ops[] = {Dst, Src, Size, getInt1(IsVolatile)};
268  Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
269  Function *F = BB->getParent();
270  Module *M = F->getParent();
271  Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy_inline, Tys);
272
273  CallInst *CI = CreateCall(TheFn, Ops);
274
275  auto *MCI = cast<MemCpyInlineInst>(CI);
276  if (DstAlign)
277    MCI->setDestAlignment(*DstAlign);
278  if (SrcAlign)
279    MCI->setSourceAlignment(*SrcAlign);
280
281  // Set the TBAA info if present.
282  if (TBAATag)
283    MCI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
284
285  // Set the TBAA Struct info if present.
286  if (TBAAStructTag)
287    MCI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
288
289  if (ScopeTag)
290    MCI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
291
292  if (NoAliasTag)
293    MCI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
294
295  return CI;
296}
297
298CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
299    Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
300    uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
301    MDNode *ScopeTag, MDNode *NoAliasTag) {
302  assert(DstAlign >= ElementSize &&
303         "Pointer alignment must be at least element size");
304  assert(SrcAlign >= ElementSize &&
305         "Pointer alignment must be at least element size");
306  Dst = getCastedInt8PtrValue(Dst);
307  Src = getCastedInt8PtrValue(Src);
308
309  Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
310  Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
311  Module *M = BB->getParent()->getParent();
312  Function *TheFn = Intrinsic::getDeclaration(
313      M, Intrinsic::memcpy_element_unordered_atomic, Tys);
314
315  CallInst *CI = CreateCall(TheFn, Ops);
316
317  // Set the alignment of the pointer args.
318  auto *AMCI = cast<AtomicMemCpyInst>(CI);
319  AMCI->setDestAlignment(DstAlign);
320  AMCI->setSourceAlignment(SrcAlign);
321
322  // Set the TBAA info if present.
323  if (TBAATag)
324    CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
325
326  // Set the TBAA Struct info if present.
327  if (TBAAStructTag)
328    CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
329
330  if (ScopeTag)
331    CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
332
333  if (NoAliasTag)
334    CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
335
336  return CI;
337}
338
339CallInst *IRBuilderBase::CreateMemMove(Value *Dst, MaybeAlign DstAlign,
340                                       Value *Src, MaybeAlign SrcAlign,
341                                       Value *Size, bool isVolatile,
342                                       MDNode *TBAATag, MDNode *ScopeTag,
343                                       MDNode *NoAliasTag) {
344  Dst = getCastedInt8PtrValue(Dst);
345  Src = getCastedInt8PtrValue(Src);
346
347  Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
348  Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
349  Module *M = BB->getParent()->getParent();
350  Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
351
352  CallInst *CI = CreateCall(TheFn, Ops);
353
354  auto *MMI = cast<MemMoveInst>(CI);
355  if (DstAlign)
356    MMI->setDestAlignment(*DstAlign);
357  if (SrcAlign)
358    MMI->setSourceAlignment(*SrcAlign);
359
360  // Set the TBAA info if present.
361  if (TBAATag)
362    CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
363
364  if (ScopeTag)
365    CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
366
367  if (NoAliasTag)
368    CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
369
370  return CI;
371}
372
373CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
374    Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
375    uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
376    MDNode *ScopeTag, MDNode *NoAliasTag) {
377  assert(DstAlign >= ElementSize &&
378         "Pointer alignment must be at least element size");
379  assert(SrcAlign >= ElementSize &&
380         "Pointer alignment must be at least element size");
381  Dst = getCastedInt8PtrValue(Dst);
382  Src = getCastedInt8PtrValue(Src);
383
384  Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
385  Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
386  Module *M = BB->getParent()->getParent();
387  Function *TheFn = Intrinsic::getDeclaration(
388      M, Intrinsic::memmove_element_unordered_atomic, Tys);
389
390  CallInst *CI = CreateCall(TheFn, Ops);
391
392  // Set the alignment of the pointer args.
393  CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign));
394  CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign));
395
396  // Set the TBAA info if present.
397  if (TBAATag)
398    CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
399
400  // Set the TBAA Struct info if present.
401  if (TBAAStructTag)
402    CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
403
404  if (ScopeTag)
405    CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
406
407  if (NoAliasTag)
408    CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
409
410  return CI;
411}
412
413CallInst *IRBuilderBase::getReductionIntrinsic(Intrinsic::ID ID, Value *Src) {
414  Module *M = GetInsertBlock()->getParent()->getParent();
415  Value *Ops[] = {Src};
416  Type *Tys[] = { Src->getType() };
417  auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
418  return CreateCall(Decl, Ops);
419}
420
421CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
422  Module *M = GetInsertBlock()->getParent()->getParent();
423  Value *Ops[] = {Acc, Src};
424  auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fadd,
425                                        {Src->getType()});
426  return CreateCall(Decl, Ops);
427}
428
429CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
430  Module *M = GetInsertBlock()->getParent()->getParent();
431  Value *Ops[] = {Acc, Src};
432  auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fmul,
433                                        {Src->getType()});
434  return CreateCall(Decl, Ops);
435}
436
437CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {
438  return getReductionIntrinsic(Intrinsic::vector_reduce_add, Src);
439}
440
441CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {
442  return getReductionIntrinsic(Intrinsic::vector_reduce_mul, Src);
443}
444
445CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {
446  return getReductionIntrinsic(Intrinsic::vector_reduce_and, Src);
447}
448
449CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {
450  return getReductionIntrinsic(Intrinsic::vector_reduce_or, Src);
451}
452
453CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {
454  return getReductionIntrinsic(Intrinsic::vector_reduce_xor, Src);
455}
456
457CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
458  auto ID =
459      IsSigned ? Intrinsic::vector_reduce_smax : Intrinsic::vector_reduce_umax;
460  return getReductionIntrinsic(ID, Src);
461}
462
463CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
464  auto ID =
465      IsSigned ? Intrinsic::vector_reduce_smin : Intrinsic::vector_reduce_umin;
466  return getReductionIntrinsic(ID, Src);
467}
468
469CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src) {
470  return getReductionIntrinsic(Intrinsic::vector_reduce_fmax, Src);
471}
472
473CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src) {
474  return getReductionIntrinsic(Intrinsic::vector_reduce_fmin, Src);
475}
476
477CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
478  assert(isa<PointerType>(Ptr->getType()) &&
479         "lifetime.start only applies to pointers.");
480  Ptr = getCastedInt8PtrValue(Ptr);
481  if (!Size)
482    Size = getInt64(-1);
483  else
484    assert(Size->getType() == getInt64Ty() &&
485           "lifetime.start requires the size to be an i64");
486  Value *Ops[] = { Size, Ptr };
487  Module *M = BB->getParent()->getParent();
488  Function *TheFn =
489      Intrinsic::getDeclaration(M, Intrinsic::lifetime_start, {Ptr->getType()});
490  return CreateCall(TheFn, Ops);
491}
492
493CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
494  assert(isa<PointerType>(Ptr->getType()) &&
495         "lifetime.end only applies to pointers.");
496  Ptr = getCastedInt8PtrValue(Ptr);
497  if (!Size)
498    Size = getInt64(-1);
499  else
500    assert(Size->getType() == getInt64Ty() &&
501           "lifetime.end requires the size to be an i64");
502  Value *Ops[] = { Size, Ptr };
503  Module *M = BB->getParent()->getParent();
504  Function *TheFn =
505      Intrinsic::getDeclaration(M, Intrinsic::lifetime_end, {Ptr->getType()});
506  return CreateCall(TheFn, Ops);
507}
508
509CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
510
511  assert(isa<PointerType>(Ptr->getType()) &&
512         "invariant.start only applies to pointers.");
513  Ptr = getCastedInt8PtrValue(Ptr);
514  if (!Size)
515    Size = getInt64(-1);
516  else
517    assert(Size->getType() == getInt64Ty() &&
518           "invariant.start requires the size to be an i64");
519
520  Value *Ops[] = {Size, Ptr};
521  // Fill in the single overloaded type: memory object type.
522  Type *ObjectPtr[1] = {Ptr->getType()};
523  Module *M = BB->getParent()->getParent();
524  Function *TheFn =
525      Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
526  return CreateCall(TheFn, Ops);
527}
528
529static MaybeAlign getAlign(Value *Ptr) {
530  if (auto *O = dyn_cast<GlobalObject>(Ptr))
531    return O->getAlign();
532  if (auto *A = dyn_cast<GlobalAlias>(Ptr))
533    return A->getAliaseeObject()->getAlign();
534  return {};
535}
536
537CallInst *IRBuilderBase::CreateThreadLocalAddress(Value *Ptr) {
538#ifndef NDEBUG
539  // Handle specially for constexpr cast. This is possible when
540  // opaque pointers not enabled since constant could be sinked
541  // directly by the design of llvm. This could be eliminated
542  // after we eliminate the abuse of constexpr.
543  auto *V = Ptr;
544  if (auto *CE = dyn_cast<ConstantExpr>(V))
545    if (CE->isCast())
546      V = CE->getOperand(0);
547
548  assert(isa<GlobalValue>(V) && cast<GlobalValue>(V)->isThreadLocal() &&
549         "threadlocal_address only applies to thread local variables.");
550#endif
551  CallInst *CI = CreateIntrinsic(llvm::Intrinsic::threadlocal_address,
552                                 {Ptr->getType()}, {Ptr});
553  if (MaybeAlign A = getAlign(Ptr)) {
554    CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), *A));
555    CI->addRetAttr(Attribute::getWithAlignment(CI->getContext(), *A));
556  }
557  return CI;
558}
559
560CallInst *
561IRBuilderBase::CreateAssumption(Value *Cond,
562                                ArrayRef<OperandBundleDef> OpBundles) {
563  assert(Cond->getType() == getInt1Ty() &&
564         "an assumption condition must be of type i1");
565
566  Value *Ops[] = { Cond };
567  Module *M = BB->getParent()->getParent();
568  Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
569  return CreateCall(FnAssume, Ops, OpBundles);
570}
571
572Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) {
573  Module *M = BB->getModule();
574  auto *FnIntrinsic = Intrinsic::getDeclaration(
575      M, Intrinsic::experimental_noalias_scope_decl, {});
576  return CreateCall(FnIntrinsic, {Scope});
577}
578
579/// Create a call to a Masked Load intrinsic.
580/// \p Ty        - vector type to load
581/// \p Ptr       - base pointer for the load
582/// \p Alignment - alignment of the source location
583/// \p Mask      - vector of booleans which indicates what vector lanes should
584///                be accessed in memory
585/// \p PassThru  - pass-through value that is used to fill the masked-off lanes
586///                of the result
587/// \p Name      - name of the result variable
588CallInst *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
589                                          Value *Mask, Value *PassThru,
590                                          const Twine &Name) {
591  auto *PtrTy = cast<PointerType>(Ptr->getType());
592  assert(Ty->isVectorTy() && "Type should be vector");
593  assert(PtrTy->isOpaqueOrPointeeTypeMatches(Ty) && "Wrong element type");
594  assert(Mask && "Mask should not be all-ones (null)");
595  if (!PassThru)
596    PassThru = PoisonValue::get(Ty);
597  Type *OverloadedTypes[] = { Ty, PtrTy };
598  Value *Ops[] = {Ptr, getInt32(Alignment.value()), Mask, PassThru};
599  return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
600                               OverloadedTypes, Name);
601}
602
603/// Create a call to a Masked Store intrinsic.
604/// \p Val       - data to be stored,
605/// \p Ptr       - base pointer for the store
606/// \p Alignment - alignment of the destination location
607/// \p Mask      - vector of booleans which indicates what vector lanes should
608///                be accessed in memory
609CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
610                                           Align Alignment, Value *Mask) {
611  auto *PtrTy = cast<PointerType>(Ptr->getType());
612  Type *DataTy = Val->getType();
613  assert(DataTy->isVectorTy() && "Val should be a vector");
614  assert(PtrTy->isOpaqueOrPointeeTypeMatches(DataTy) && "Wrong element type");
615  assert(Mask && "Mask should not be all-ones (null)");
616  Type *OverloadedTypes[] = { DataTy, PtrTy };
617  Value *Ops[] = {Val, Ptr, getInt32(Alignment.value()), Mask};
618  return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
619}
620
621/// Create a call to a Masked intrinsic, with given intrinsic Id,
622/// an array of operands - Ops, and an array of overloaded types -
623/// OverloadedTypes.
624CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
625                                               ArrayRef<Value *> Ops,
626                                               ArrayRef<Type *> OverloadedTypes,
627                                               const Twine &Name) {
628  Module *M = BB->getParent()->getParent();
629  Function *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
630  return CreateCall(TheFn, Ops, {}, Name);
631}
632
633/// Create a call to a Masked Gather intrinsic.
634/// \p Ty       - vector type to gather
635/// \p Ptrs     - vector of pointers for loading
636/// \p Align    - alignment for one element
637/// \p Mask     - vector of booleans which indicates what vector lanes should
638///               be accessed in memory
639/// \p PassThru - pass-through value that is used to fill the masked-off lanes
640///               of the result
641/// \p Name     - name of the result variable
642CallInst *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs,
643                                            Align Alignment, Value *Mask,
644                                            Value *PassThru,
645                                            const Twine &Name) {
646  auto *VecTy = cast<VectorType>(Ty);
647  ElementCount NumElts = VecTy->getElementCount();
648  auto *PtrsTy = cast<VectorType>(Ptrs->getType());
649  assert(cast<PointerType>(PtrsTy->getElementType())
650             ->isOpaqueOrPointeeTypeMatches(
651                 cast<VectorType>(Ty)->getElementType()) &&
652         "Element type mismatch");
653  assert(NumElts == PtrsTy->getElementCount() && "Element count mismatch");
654
655  if (!Mask)
656    Mask = Constant::getAllOnesValue(
657        VectorType::get(Type::getInt1Ty(Context), NumElts));
658
659  if (!PassThru)
660    PassThru = PoisonValue::get(Ty);
661
662  Type *OverloadedTypes[] = {Ty, PtrsTy};
663  Value *Ops[] = {Ptrs, getInt32(Alignment.value()), Mask, PassThru};
664
665  // We specify only one type when we create this intrinsic. Types of other
666  // arguments are derived from this type.
667  return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes,
668                               Name);
669}
670
671/// Create a call to a Masked Scatter intrinsic.
672/// \p Data  - data to be stored,
673/// \p Ptrs  - the vector of pointers, where the \p Data elements should be
674///            stored
675/// \p Align - alignment for one element
676/// \p Mask  - vector of booleans which indicates what vector lanes should
677///            be accessed in memory
678CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
679                                             Align Alignment, Value *Mask) {
680  auto *PtrsTy = cast<VectorType>(Ptrs->getType());
681  auto *DataTy = cast<VectorType>(Data->getType());
682  ElementCount NumElts = PtrsTy->getElementCount();
683
684#ifndef NDEBUG
685  auto *PtrTy = cast<PointerType>(PtrsTy->getElementType());
686  assert(NumElts == DataTy->getElementCount() &&
687         PtrTy->isOpaqueOrPointeeTypeMatches(DataTy->getElementType()) &&
688         "Incompatible pointer and data types");
689#endif
690
691  if (!Mask)
692    Mask = Constant::getAllOnesValue(
693        VectorType::get(Type::getInt1Ty(Context), NumElts));
694
695  Type *OverloadedTypes[] = {DataTy, PtrsTy};
696  Value *Ops[] = {Data, Ptrs, getInt32(Alignment.value()), Mask};
697
698  // We specify only one type when we create this intrinsic. Types of other
699  // arguments are derived from this type.
700  return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
701}
702
703/// Create a call to Masked Expand Load intrinsic
704/// \p Ty        - vector type to load
705/// \p Ptr       - base pointer for the load
706/// \p Mask      - vector of booleans which indicates what vector lanes should
707///                be accessed in memory
708/// \p PassThru  - pass-through value that is used to fill the masked-off lanes
709///                of the result
710/// \p Name      - name of the result variable
711CallInst *IRBuilderBase::CreateMaskedExpandLoad(Type *Ty, Value *Ptr,
712                                                Value *Mask, Value *PassThru,
713                                                const Twine &Name) {
714  auto *PtrTy = cast<PointerType>(Ptr->getType());
715  assert(Ty->isVectorTy() && "Type should be vector");
716  assert(PtrTy->isOpaqueOrPointeeTypeMatches(
717             cast<FixedVectorType>(Ty)->getElementType()) &&
718         "Wrong element type");
719  (void)PtrTy;
720  assert(Mask && "Mask should not be all-ones (null)");
721  if (!PassThru)
722    PassThru = PoisonValue::get(Ty);
723  Type *OverloadedTypes[] = {Ty};
724  Value *Ops[] = {Ptr, Mask, PassThru};
725  return CreateMaskedIntrinsic(Intrinsic::masked_expandload, Ops,
726                               OverloadedTypes, Name);
727}
728
729/// Create a call to Masked Compress Store intrinsic
730/// \p Val       - data to be stored,
731/// \p Ptr       - base pointer for the store
732/// \p Mask      - vector of booleans which indicates what vector lanes should
733///                be accessed in memory
734CallInst *IRBuilderBase::CreateMaskedCompressStore(Value *Val, Value *Ptr,
735                                                   Value *Mask) {
736  auto *PtrTy = cast<PointerType>(Ptr->getType());
737  Type *DataTy = Val->getType();
738  assert(DataTy->isVectorTy() && "Val should be a vector");
739  assert(PtrTy->isOpaqueOrPointeeTypeMatches(
740             cast<FixedVectorType>(DataTy)->getElementType()) &&
741         "Wrong element type");
742  (void)PtrTy;
743  assert(Mask && "Mask should not be all-ones (null)");
744  Type *OverloadedTypes[] = {DataTy};
745  Value *Ops[] = {Val, Ptr, Mask};
746  return CreateMaskedIntrinsic(Intrinsic::masked_compressstore, Ops,
747                               OverloadedTypes);
748}
749
750template <typename T0>
751static std::vector<Value *>
752getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
753                  Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs) {
754  std::vector<Value *> Args;
755  Args.push_back(B.getInt64(ID));
756  Args.push_back(B.getInt32(NumPatchBytes));
757  Args.push_back(ActualCallee);
758  Args.push_back(B.getInt32(CallArgs.size()));
759  Args.push_back(B.getInt32(Flags));
760  llvm::append_range(Args, CallArgs);
761  // GC Transition and Deopt args are now always handled via operand bundle.
762  // They will be removed from the signature of gc.statepoint shortly.
763  Args.push_back(B.getInt32(0));
764  Args.push_back(B.getInt32(0));
765  // GC args are now encoded in the gc-live operand bundle
766  return Args;
767}
768
769template<typename T1, typename T2, typename T3>
770static std::vector<OperandBundleDef>
771getStatepointBundles(std::optional<ArrayRef<T1>> TransitionArgs,
772                     std::optional<ArrayRef<T2>> DeoptArgs,
773                     ArrayRef<T3> GCArgs) {
774  std::vector<OperandBundleDef> Rval;
775  if (DeoptArgs) {
776    SmallVector<Value*, 16> DeoptValues;
777    llvm::append_range(DeoptValues, *DeoptArgs);
778    Rval.emplace_back("deopt", DeoptValues);
779  }
780  if (TransitionArgs) {
781    SmallVector<Value*, 16> TransitionValues;
782    llvm::append_range(TransitionValues, *TransitionArgs);
783    Rval.emplace_back("gc-transition", TransitionValues);
784  }
785  if (GCArgs.size()) {
786    SmallVector<Value*, 16> LiveValues;
787    llvm::append_range(LiveValues, GCArgs);
788    Rval.emplace_back("gc-live", LiveValues);
789  }
790  return Rval;
791}
792
793template <typename T0, typename T1, typename T2, typename T3>
794static CallInst *CreateGCStatepointCallCommon(
795    IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
796    FunctionCallee ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
797    std::optional<ArrayRef<T1>> TransitionArgs,
798    std::optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,
799    const Twine &Name) {
800  Module *M = Builder->GetInsertBlock()->getParent()->getParent();
801  // Fill in the one generic type'd argument (the function is also vararg)
802  Function *FnStatepoint =
803      Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
804                                {ActualCallee.getCallee()->getType()});
805
806  std::vector<Value *> Args = getStatepointArgs(
807      *Builder, ID, NumPatchBytes, ActualCallee.getCallee(), Flags, CallArgs);
808
809  CallInst *CI = Builder->CreateCall(
810      FnStatepoint, Args,
811      getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name);
812  CI->addParamAttr(2,
813                   Attribute::get(Builder->getContext(), Attribute::ElementType,
814                                  ActualCallee.getFunctionType()));
815  return CI;
816}
817
818CallInst *IRBuilderBase::CreateGCStatepointCall(
819    uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
820    ArrayRef<Value *> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
821    ArrayRef<Value *> GCArgs, const Twine &Name) {
822  return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
823      this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
824      CallArgs, std::nullopt /* No Transition Args */, DeoptArgs, GCArgs, Name);
825}
826
827CallInst *IRBuilderBase::CreateGCStatepointCall(
828    uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
829    uint32_t Flags, ArrayRef<Value *> CallArgs,
830    std::optional<ArrayRef<Use>> TransitionArgs,
831    std::optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
832    const Twine &Name) {
833  return CreateGCStatepointCallCommon<Value *, Use, Use, Value *>(
834      this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
835      DeoptArgs, GCArgs, Name);
836}
837
838CallInst *IRBuilderBase::CreateGCStatepointCall(
839    uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
840    ArrayRef<Use> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
841    ArrayRef<Value *> GCArgs, const Twine &Name) {
842  return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
843      this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
844      CallArgs, std::nullopt, DeoptArgs, GCArgs, Name);
845}
846
847template <typename T0, typename T1, typename T2, typename T3>
848static InvokeInst *CreateGCStatepointInvokeCommon(
849    IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
850    FunctionCallee ActualInvokee, BasicBlock *NormalDest,
851    BasicBlock *UnwindDest, uint32_t Flags, ArrayRef<T0> InvokeArgs,
852    std::optional<ArrayRef<T1>> TransitionArgs,
853    std::optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,
854    const Twine &Name) {
855  Module *M = Builder->GetInsertBlock()->getParent()->getParent();
856  // Fill in the one generic type'd argument (the function is also vararg)
857  Function *FnStatepoint =
858      Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
859                                {ActualInvokee.getCallee()->getType()});
860
861  std::vector<Value *> Args =
862      getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee.getCallee(),
863                        Flags, InvokeArgs);
864
865  InvokeInst *II = Builder->CreateInvoke(
866      FnStatepoint, NormalDest, UnwindDest, Args,
867      getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name);
868  II->addParamAttr(2,
869                   Attribute::get(Builder->getContext(), Attribute::ElementType,
870                                  ActualInvokee.getFunctionType()));
871  return II;
872}
873
874InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
875    uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
876    BasicBlock *NormalDest, BasicBlock *UnwindDest,
877    ArrayRef<Value *> InvokeArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
878    ArrayRef<Value *> GCArgs, const Twine &Name) {
879  return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
880      this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
881      uint32_t(StatepointFlags::None), InvokeArgs,
882      std::nullopt /* No Transition Args*/, DeoptArgs, GCArgs, Name);
883}
884
885InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
886    uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
887    BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
888    ArrayRef<Value *> InvokeArgs, std::optional<ArrayRef<Use>> TransitionArgs,
889    std::optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
890    const Twine &Name) {
891  return CreateGCStatepointInvokeCommon<Value *, Use, Use, Value *>(
892      this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
893      InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
894}
895
896InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
897    uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
898    BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
899    std::optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs,
900    const Twine &Name) {
901  return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
902      this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
903      uint32_t(StatepointFlags::None), InvokeArgs, std::nullopt, DeoptArgs,
904      GCArgs, Name);
905}
906
907CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
908                                        Type *ResultType, const Twine &Name) {
909  Intrinsic::ID ID = Intrinsic::experimental_gc_result;
910  Module *M = BB->getParent()->getParent();
911  Type *Types[] = {ResultType};
912  Function *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
913
914  Value *Args[] = {Statepoint};
915  return CreateCall(FnGCResult, Args, {}, Name);
916}
917
918CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
919                                          int BaseOffset, int DerivedOffset,
920                                          Type *ResultType, const Twine &Name) {
921  Module *M = BB->getParent()->getParent();
922  Type *Types[] = {ResultType};
923  Function *FnGCRelocate =
924      Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
925
926  Value *Args[] = {Statepoint, getInt32(BaseOffset), getInt32(DerivedOffset)};
927  return CreateCall(FnGCRelocate, Args, {}, Name);
928}
929
930CallInst *IRBuilderBase::CreateGCGetPointerBase(Value *DerivedPtr,
931                                                const Twine &Name) {
932  Module *M = BB->getParent()->getParent();
933  Type *PtrTy = DerivedPtr->getType();
934  Function *FnGCFindBase = Intrinsic::getDeclaration(
935      M, Intrinsic::experimental_gc_get_pointer_base, {PtrTy, PtrTy});
936  return CreateCall(FnGCFindBase, {DerivedPtr}, {}, Name);
937}
938
939CallInst *IRBuilderBase::CreateGCGetPointerOffset(Value *DerivedPtr,
940                                                  const Twine &Name) {
941  Module *M = BB->getParent()->getParent();
942  Type *PtrTy = DerivedPtr->getType();
943  Function *FnGCGetOffset = Intrinsic::getDeclaration(
944      M, Intrinsic::experimental_gc_get_pointer_offset, {PtrTy});
945  return CreateCall(FnGCGetOffset, {DerivedPtr}, {}, Name);
946}
947
948CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
949                                              Instruction *FMFSource,
950                                              const Twine &Name) {
951  Module *M = BB->getModule();
952  Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()});
953  return createCallHelper(Fn, {V}, Name, FMFSource);
954}
955
956CallInst *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
957                                               Value *RHS,
958                                               Instruction *FMFSource,
959                                               const Twine &Name) {
960  Module *M = BB->getModule();
961  Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
962  return createCallHelper(Fn, {LHS, RHS}, Name, FMFSource);
963}
964
965CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
966                                         ArrayRef<Type *> Types,
967                                         ArrayRef<Value *> Args,
968                                         Instruction *FMFSource,
969                                         const Twine &Name) {
970  Module *M = BB->getModule();
971  Function *Fn = Intrinsic::getDeclaration(M, ID, Types);
972  return createCallHelper(Fn, Args, Name, FMFSource);
973}
974
975CallInst *IRBuilderBase::CreateIntrinsic(Type *RetTy, Intrinsic::ID ID,
976                                         ArrayRef<Value *> Args,
977                                         Instruction *FMFSource,
978                                         const Twine &Name) {
979  Module *M = BB->getModule();
980
981  SmallVector<Intrinsic::IITDescriptor> Table;
982  Intrinsic::getIntrinsicInfoTableEntries(ID, Table);
983  ArrayRef<Intrinsic::IITDescriptor> TableRef(Table);
984
985  SmallVector<Type *> ArgTys;
986  ArgTys.reserve(Args.size());
987  for (auto &I : Args)
988    ArgTys.push_back(I->getType());
989  FunctionType *FTy = FunctionType::get(RetTy, ArgTys, false);
990  SmallVector<Type *> OverloadTys;
991  Intrinsic::MatchIntrinsicTypesResult Res =
992      matchIntrinsicSignature(FTy, TableRef, OverloadTys);
993  (void)Res;
994  assert(Res == Intrinsic::MatchIntrinsicTypes_Match && TableRef.empty() &&
995         "Wrong types for intrinsic!");
996  // TODO: Handle varargs intrinsics.
997
998  Function *Fn = Intrinsic::getDeclaration(M, ID, OverloadTys);
999  return createCallHelper(Fn, Args, Name, FMFSource);
1000}
1001
1002CallInst *IRBuilderBase::CreateConstrainedFPBinOp(
1003    Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource,
1004    const Twine &Name, MDNode *FPMathTag,
1005    std::optional<RoundingMode> Rounding,
1006    std::optional<fp::ExceptionBehavior> Except) {
1007  Value *RoundingV = getConstrainedFPRounding(Rounding);
1008  Value *ExceptV = getConstrainedFPExcept(Except);
1009
1010  FastMathFlags UseFMF = FMF;
1011  if (FMFSource)
1012    UseFMF = FMFSource->getFastMathFlags();
1013
1014  CallInst *C = CreateIntrinsic(ID, {L->getType()},
1015                                {L, R, RoundingV, ExceptV}, nullptr, Name);
1016  setConstrainedFPCallAttr(C);
1017  setFPAttrs(C, FPMathTag, UseFMF);
1018  return C;
1019}
1020
1021Value *IRBuilderBase::CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops,
1022                                   const Twine &Name, MDNode *FPMathTag) {
1023  if (Instruction::isBinaryOp(Opc)) {
1024    assert(Ops.size() == 2 && "Invalid number of operands!");
1025    return CreateBinOp(static_cast<Instruction::BinaryOps>(Opc),
1026                       Ops[0], Ops[1], Name, FPMathTag);
1027  }
1028  if (Instruction::isUnaryOp(Opc)) {
1029    assert(Ops.size() == 1 && "Invalid number of operands!");
1030    return CreateUnOp(static_cast<Instruction::UnaryOps>(Opc),
1031                      Ops[0], Name, FPMathTag);
1032  }
1033  llvm_unreachable("Unexpected opcode!");
1034}
1035
1036CallInst *IRBuilderBase::CreateConstrainedFPCast(
1037    Intrinsic::ID ID, Value *V, Type *DestTy,
1038    Instruction *FMFSource, const Twine &Name, MDNode *FPMathTag,
1039    std::optional<RoundingMode> Rounding,
1040    std::optional<fp::ExceptionBehavior> Except) {
1041  Value *ExceptV = getConstrainedFPExcept(Except);
1042
1043  FastMathFlags UseFMF = FMF;
1044  if (FMFSource)
1045    UseFMF = FMFSource->getFastMathFlags();
1046
1047  CallInst *C;
1048  bool HasRoundingMD = false;
1049  switch (ID) {
1050  default:
1051    break;
1052#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)        \
1053  case Intrinsic::INTRINSIC:                                \
1054    HasRoundingMD = ROUND_MODE;                             \
1055    break;
1056#include "llvm/IR/ConstrainedOps.def"
1057  }
1058  if (HasRoundingMD) {
1059    Value *RoundingV = getConstrainedFPRounding(Rounding);
1060    C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, RoundingV, ExceptV},
1061                        nullptr, Name);
1062  } else
1063    C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, ExceptV}, nullptr,
1064                        Name);
1065
1066  setConstrainedFPCallAttr(C);
1067
1068  if (isa<FPMathOperator>(C))
1069    setFPAttrs(C, FPMathTag, UseFMF);
1070  return C;
1071}
1072
1073Value *IRBuilderBase::CreateFCmpHelper(
1074    CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name,
1075    MDNode *FPMathTag, bool IsSignaling) {
1076  if (IsFPConstrained) {
1077    auto ID = IsSignaling ? Intrinsic::experimental_constrained_fcmps
1078                          : Intrinsic::experimental_constrained_fcmp;
1079    return CreateConstrainedFPCmp(ID, P, LHS, RHS, Name);
1080  }
1081
1082  if (auto *LC = dyn_cast<Constant>(LHS))
1083    if (auto *RC = dyn_cast<Constant>(RHS))
1084      return Insert(Folder.CreateFCmp(P, LC, RC), Name);
1085  return Insert(setFPAttrs(new FCmpInst(P, LHS, RHS), FPMathTag, FMF), Name);
1086}
1087
1088CallInst *IRBuilderBase::CreateConstrainedFPCmp(
1089    Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R,
1090    const Twine &Name, std::optional<fp::ExceptionBehavior> Except) {
1091  Value *PredicateV = getConstrainedFPPredicate(P);
1092  Value *ExceptV = getConstrainedFPExcept(Except);
1093
1094  CallInst *C = CreateIntrinsic(ID, {L->getType()},
1095                                {L, R, PredicateV, ExceptV}, nullptr, Name);
1096  setConstrainedFPCallAttr(C);
1097  return C;
1098}
1099
1100CallInst *IRBuilderBase::CreateConstrainedFPCall(
1101    Function *Callee, ArrayRef<Value *> Args, const Twine &Name,
1102    std::optional<RoundingMode> Rounding,
1103    std::optional<fp::ExceptionBehavior> Except) {
1104  llvm::SmallVector<Value *, 6> UseArgs;
1105
1106  append_range(UseArgs, Args);
1107  bool HasRoundingMD = false;
1108  switch (Callee->getIntrinsicID()) {
1109  default:
1110    break;
1111#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)        \
1112  case Intrinsic::INTRINSIC:                                \
1113    HasRoundingMD = ROUND_MODE;                             \
1114    break;
1115#include "llvm/IR/ConstrainedOps.def"
1116  }
1117  if (HasRoundingMD)
1118    UseArgs.push_back(getConstrainedFPRounding(Rounding));
1119  UseArgs.push_back(getConstrainedFPExcept(Except));
1120
1121  CallInst *C = CreateCall(Callee, UseArgs, Name);
1122  setConstrainedFPCallAttr(C);
1123  return C;
1124}
1125
1126Value *IRBuilderBase::CreateSelect(Value *C, Value *True, Value *False,
1127                                   const Twine &Name, Instruction *MDFrom) {
1128  if (auto *V = Folder.FoldSelect(C, True, False))
1129    return V;
1130
1131  SelectInst *Sel = SelectInst::Create(C, True, False);
1132  if (MDFrom) {
1133    MDNode *Prof = MDFrom->getMetadata(LLVMContext::MD_prof);
1134    MDNode *Unpred = MDFrom->getMetadata(LLVMContext::MD_unpredictable);
1135    Sel = addBranchMetadata(Sel, Prof, Unpred);
1136  }
1137  if (isa<FPMathOperator>(Sel))
1138    setFPAttrs(Sel, nullptr /* MDNode* */, FMF);
1139  return Insert(Sel, Name);
1140}
1141
1142Value *IRBuilderBase::CreatePtrDiff(Type *ElemTy, Value *LHS, Value *RHS,
1143                                    const Twine &Name) {
1144  assert(LHS->getType() == RHS->getType() &&
1145         "Pointer subtraction operand types must match!");
1146  assert(cast<PointerType>(LHS->getType())
1147             ->isOpaqueOrPointeeTypeMatches(ElemTy) &&
1148         "Pointer type must match element type");
1149  Value *LHS_int = CreatePtrToInt(LHS, Type::getInt64Ty(Context));
1150  Value *RHS_int = CreatePtrToInt(RHS, Type::getInt64Ty(Context));
1151  Value *Difference = CreateSub(LHS_int, RHS_int);
1152  return CreateExactSDiv(Difference, ConstantExpr::getSizeOf(ElemTy),
1153                         Name);
1154}
1155
1156Value *IRBuilderBase::CreateLaunderInvariantGroup(Value *Ptr) {
1157  assert(isa<PointerType>(Ptr->getType()) &&
1158         "launder.invariant.group only applies to pointers.");
1159  // FIXME: we could potentially avoid casts to/from i8*.
1160  auto *PtrType = Ptr->getType();
1161  auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace());
1162  if (PtrType != Int8PtrTy)
1163    Ptr = CreateBitCast(Ptr, Int8PtrTy);
1164  Module *M = BB->getParent()->getParent();
1165  Function *FnLaunderInvariantGroup = Intrinsic::getDeclaration(
1166      M, Intrinsic::launder_invariant_group, {Int8PtrTy});
1167
1168  assert(FnLaunderInvariantGroup->getReturnType() == Int8PtrTy &&
1169         FnLaunderInvariantGroup->getFunctionType()->getParamType(0) ==
1170             Int8PtrTy &&
1171         "LaunderInvariantGroup should take and return the same type");
1172
1173  CallInst *Fn = CreateCall(FnLaunderInvariantGroup, {Ptr});
1174
1175  if (PtrType != Int8PtrTy)
1176    return CreateBitCast(Fn, PtrType);
1177  return Fn;
1178}
1179
1180Value *IRBuilderBase::CreateStripInvariantGroup(Value *Ptr) {
1181  assert(isa<PointerType>(Ptr->getType()) &&
1182         "strip.invariant.group only applies to pointers.");
1183
1184  // FIXME: we could potentially avoid casts to/from i8*.
1185  auto *PtrType = Ptr->getType();
1186  auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace());
1187  if (PtrType != Int8PtrTy)
1188    Ptr = CreateBitCast(Ptr, Int8PtrTy);
1189  Module *M = BB->getParent()->getParent();
1190  Function *FnStripInvariantGroup = Intrinsic::getDeclaration(
1191      M, Intrinsic::strip_invariant_group, {Int8PtrTy});
1192
1193  assert(FnStripInvariantGroup->getReturnType() == Int8PtrTy &&
1194         FnStripInvariantGroup->getFunctionType()->getParamType(0) ==
1195             Int8PtrTy &&
1196         "StripInvariantGroup should take and return the same type");
1197
1198  CallInst *Fn = CreateCall(FnStripInvariantGroup, {Ptr});
1199
1200  if (PtrType != Int8PtrTy)
1201    return CreateBitCast(Fn, PtrType);
1202  return Fn;
1203}
1204
1205Value *IRBuilderBase::CreateVectorReverse(Value *V, const Twine &Name) {
1206  auto *Ty = cast<VectorType>(V->getType());
1207  if (isa<ScalableVectorType>(Ty)) {
1208    Module *M = BB->getParent()->getParent();
1209    Function *F = Intrinsic::getDeclaration(
1210        M, Intrinsic::experimental_vector_reverse, Ty);
1211    return Insert(CallInst::Create(F, V), Name);
1212  }
1213  // Keep the original behaviour for fixed vector
1214  SmallVector<int, 8> ShuffleMask;
1215  int NumElts = Ty->getElementCount().getKnownMinValue();
1216  for (int i = 0; i < NumElts; ++i)
1217    ShuffleMask.push_back(NumElts - i - 1);
1218  return CreateShuffleVector(V, ShuffleMask, Name);
1219}
1220
1221Value *IRBuilderBase::CreateVectorSplice(Value *V1, Value *V2, int64_t Imm,
1222                                         const Twine &Name) {
1223  assert(isa<VectorType>(V1->getType()) && "Unexpected type");
1224  assert(V1->getType() == V2->getType() &&
1225         "Splice expects matching operand types!");
1226
1227  if (auto *VTy = dyn_cast<ScalableVectorType>(V1->getType())) {
1228    Module *M = BB->getParent()->getParent();
1229    Function *F = Intrinsic::getDeclaration(
1230        M, Intrinsic::experimental_vector_splice, VTy);
1231
1232    Value *Ops[] = {V1, V2, getInt32(Imm)};
1233    return Insert(CallInst::Create(F, Ops), Name);
1234  }
1235
1236  unsigned NumElts = cast<FixedVectorType>(V1->getType())->getNumElements();
1237  assert(((-Imm <= NumElts) || (Imm < NumElts)) &&
1238         "Invalid immediate for vector splice!");
1239
1240  // Keep the original behaviour for fixed vector
1241  unsigned Idx = (NumElts + Imm) % NumElts;
1242  SmallVector<int, 8> Mask;
1243  for (unsigned I = 0; I < NumElts; ++I)
1244    Mask.push_back(Idx + I);
1245
1246  return CreateShuffleVector(V1, V2, Mask);
1247}
1248
1249Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V,
1250                                        const Twine &Name) {
1251  auto EC = ElementCount::getFixed(NumElts);
1252  return CreateVectorSplat(EC, V, Name);
1253}
1254
1255Value *IRBuilderBase::CreateVectorSplat(ElementCount EC, Value *V,
1256                                        const Twine &Name) {
1257  assert(EC.isNonZero() && "Cannot splat to an empty vector!");
1258
1259  // First insert it into a poison vector so we can shuffle it.
1260  Value *Poison = PoisonValue::get(VectorType::get(V->getType(), EC));
1261  V = CreateInsertElement(Poison, V, getInt64(0), Name + ".splatinsert");
1262
1263  // Shuffle the value across the desired number of elements.
1264  SmallVector<int, 16> Zeros;
1265  Zeros.resize(EC.getKnownMinValue());
1266  return CreateShuffleVector(V, Zeros, Name + ".splat");
1267}
1268
1269Value *IRBuilderBase::CreateExtractInteger(
1270    const DataLayout &DL, Value *From, IntegerType *ExtractedTy,
1271    uint64_t Offset, const Twine &Name) {
1272  auto *IntTy = cast<IntegerType>(From->getType());
1273  assert(DL.getTypeStoreSize(ExtractedTy) + Offset <=
1274             DL.getTypeStoreSize(IntTy) &&
1275         "Element extends past full value");
1276  uint64_t ShAmt = 8 * Offset;
1277  Value *V = From;
1278  if (DL.isBigEndian())
1279    ShAmt = 8 * (DL.getTypeStoreSize(IntTy) -
1280                 DL.getTypeStoreSize(ExtractedTy) - Offset);
1281  if (ShAmt) {
1282    V = CreateLShr(V, ShAmt, Name + ".shift");
1283  }
1284  assert(ExtractedTy->getBitWidth() <= IntTy->getBitWidth() &&
1285         "Cannot extract to a larger integer!");
1286  if (ExtractedTy != IntTy) {
1287    V = CreateTrunc(V, ExtractedTy, Name + ".trunc");
1288  }
1289  return V;
1290}
1291
1292Value *IRBuilderBase::CreatePreserveArrayAccessIndex(
1293    Type *ElTy, Value *Base, unsigned Dimension, unsigned LastIndex,
1294    MDNode *DbgInfo) {
1295  auto *BaseType = Base->getType();
1296  assert(isa<PointerType>(BaseType) &&
1297         "Invalid Base ptr type for preserve.array.access.index.");
1298  assert(cast<PointerType>(BaseType)->isOpaqueOrPointeeTypeMatches(ElTy) &&
1299         "Pointer element type mismatch");
1300
1301  Value *LastIndexV = getInt32(LastIndex);
1302  Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
1303  SmallVector<Value *, 4> IdxList(Dimension, Zero);
1304  IdxList.push_back(LastIndexV);
1305
1306  Type *ResultType =
1307      GetElementPtrInst::getGEPReturnType(ElTy, Base, IdxList);
1308
1309  Module *M = BB->getParent()->getParent();
1310  Function *FnPreserveArrayAccessIndex = Intrinsic::getDeclaration(
1311      M, Intrinsic::preserve_array_access_index, {ResultType, BaseType});
1312
1313  Value *DimV = getInt32(Dimension);
1314  CallInst *Fn =
1315      CreateCall(FnPreserveArrayAccessIndex, {Base, DimV, LastIndexV});
1316  Fn->addParamAttr(
1317      0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy));
1318  if (DbgInfo)
1319    Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1320
1321  return Fn;
1322}
1323
1324Value *IRBuilderBase::CreatePreserveUnionAccessIndex(
1325    Value *Base, unsigned FieldIndex, MDNode *DbgInfo) {
1326  assert(isa<PointerType>(Base->getType()) &&
1327         "Invalid Base ptr type for preserve.union.access.index.");
1328  auto *BaseType = Base->getType();
1329
1330  Module *M = BB->getParent()->getParent();
1331  Function *FnPreserveUnionAccessIndex = Intrinsic::getDeclaration(
1332      M, Intrinsic::preserve_union_access_index, {BaseType, BaseType});
1333
1334  Value *DIIndex = getInt32(FieldIndex);
1335  CallInst *Fn =
1336      CreateCall(FnPreserveUnionAccessIndex, {Base, DIIndex});
1337  if (DbgInfo)
1338    Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1339
1340  return Fn;
1341}
1342
1343Value *IRBuilderBase::CreatePreserveStructAccessIndex(
1344    Type *ElTy, Value *Base, unsigned Index, unsigned FieldIndex,
1345    MDNode *DbgInfo) {
1346  auto *BaseType = Base->getType();
1347  assert(isa<PointerType>(BaseType) &&
1348         "Invalid Base ptr type for preserve.struct.access.index.");
1349  assert(cast<PointerType>(BaseType)->isOpaqueOrPointeeTypeMatches(ElTy) &&
1350         "Pointer element type mismatch");
1351
1352  Value *GEPIndex = getInt32(Index);
1353  Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
1354  Type *ResultType =
1355      GetElementPtrInst::getGEPReturnType(ElTy, Base, {Zero, GEPIndex});
1356
1357  Module *M = BB->getParent()->getParent();
1358  Function *FnPreserveStructAccessIndex = Intrinsic::getDeclaration(
1359      M, Intrinsic::preserve_struct_access_index, {ResultType, BaseType});
1360
1361  Value *DIIndex = getInt32(FieldIndex);
1362  CallInst *Fn = CreateCall(FnPreserveStructAccessIndex,
1363                            {Base, GEPIndex, DIIndex});
1364  Fn->addParamAttr(
1365      0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy));
1366  if (DbgInfo)
1367    Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1368
1369  return Fn;
1370}
1371
1372CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,
1373                                                         Value *PtrValue,
1374                                                         Value *AlignValue,
1375                                                         Value *OffsetValue) {
1376  SmallVector<Value *, 4> Vals({PtrValue, AlignValue});
1377  if (OffsetValue)
1378    Vals.push_back(OffsetValue);
1379  OperandBundleDefT<Value *> AlignOpB("align", Vals);
1380  return CreateAssumption(ConstantInt::getTrue(getContext()), {AlignOpB});
1381}
1382
1383CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
1384                                                   Value *PtrValue,
1385                                                   unsigned Alignment,
1386                                                   Value *OffsetValue) {
1387  assert(isa<PointerType>(PtrValue->getType()) &&
1388         "trying to create an alignment assumption on a non-pointer?");
1389  assert(Alignment != 0 && "Invalid Alignment");
1390  auto *PtrTy = cast<PointerType>(PtrValue->getType());
1391  Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace());
1392  Value *AlignValue = ConstantInt::get(IntPtrTy, Alignment);
1393  return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue, OffsetValue);
1394}
1395
1396CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
1397                                                   Value *PtrValue,
1398                                                   Value *Alignment,
1399                                                   Value *OffsetValue) {
1400  assert(isa<PointerType>(PtrValue->getType()) &&
1401         "trying to create an alignment assumption on a non-pointer?");
1402  return CreateAlignmentAssumptionHelper(DL, PtrValue, Alignment, OffsetValue);
1403}
1404
1405IRBuilderDefaultInserter::~IRBuilderDefaultInserter() = default;
1406IRBuilderCallbackInserter::~IRBuilderCallbackInserter() = default;
1407IRBuilderFolder::~IRBuilderFolder() = default;
1408void ConstantFolder::anchor() {}
1409void NoFolder::anchor() {}
1410