1//===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file implements the IRBuilder class, which is used as a convenient way 10// to create LLVM instructions with a consistent and simplified interface. 11// 12//===----------------------------------------------------------------------===// 13 14#include "llvm/IR/IRBuilder.h" 15#include "llvm/ADT/ArrayRef.h" 16#include "llvm/ADT/None.h" 17#include "llvm/IR/Constant.h" 18#include "llvm/IR/Constants.h" 19#include "llvm/IR/DerivedTypes.h" 20#include "llvm/IR/Function.h" 21#include "llvm/IR/GlobalValue.h" 22#include "llvm/IR/GlobalVariable.h" 23#include "llvm/IR/IntrinsicInst.h" 24#include "llvm/IR/Intrinsics.h" 25#include "llvm/IR/LLVMContext.h" 26#include "llvm/IR/NoFolder.h" 27#include "llvm/IR/Operator.h" 28#include "llvm/IR/Statepoint.h" 29#include "llvm/IR/Type.h" 30#include "llvm/IR/Value.h" 31#include "llvm/Support/Casting.h" 32#include "llvm/Support/MathExtras.h" 33#include <cassert> 34#include <cstdint> 35#include <vector> 36 37using namespace llvm; 38 39/// CreateGlobalString - Make a new global variable with an initializer that 40/// has array of i8 type filled in with the nul terminated string value 41/// specified. If Name is specified, it is the name of the global variable 42/// created. 43GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str, 44 const Twine &Name, 45 unsigned AddressSpace, 46 Module *M) { 47 Constant *StrConstant = ConstantDataArray::getString(Context, Str); 48 if (!M) 49 M = BB->getParent()->getParent(); 50 auto *GV = new GlobalVariable( 51 *M, StrConstant->getType(), true, GlobalValue::PrivateLinkage, 52 StrConstant, Name, nullptr, GlobalVariable::NotThreadLocal, AddressSpace); 53 GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); 54 GV->setAlignment(Align(1)); 55 return GV; 56} 57 58Type *IRBuilderBase::getCurrentFunctionReturnType() const { 59 assert(BB && BB->getParent() && "No current function!"); 60 return BB->getParent()->getReturnType(); 61} 62 63Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) { 64 auto *PT = cast<PointerType>(Ptr->getType()); 65 if (PT->getElementType()->isIntegerTy(8)) 66 return Ptr; 67 68 // Otherwise, we need to insert a bitcast. 69 return CreateBitCast(Ptr, getInt8PtrTy(PT->getAddressSpace())); 70} 71 72static CallInst *createCallHelper(Function *Callee, ArrayRef<Value *> Ops, 73 IRBuilderBase *Builder, 74 const Twine &Name = "", 75 Instruction *FMFSource = nullptr) { 76 CallInst *CI = Builder->CreateCall(Callee, Ops, Name); 77 if (FMFSource) 78 CI->copyFastMathFlags(FMFSource); 79 return CI; 80} 81 82CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size, 83 MaybeAlign Align, bool isVolatile, 84 MDNode *TBAATag, MDNode *ScopeTag, 85 MDNode *NoAliasTag) { 86 Ptr = getCastedInt8PtrValue(Ptr); 87 Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)}; 88 Type *Tys[] = { Ptr->getType(), Size->getType() }; 89 Module *M = BB->getParent()->getParent(); 90 Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys); 91 92 CallInst *CI = createCallHelper(TheFn, Ops, this); 93 94 if (Align) 95 cast<MemSetInst>(CI)->setDestAlignment(Align->value()); 96 97 // Set the TBAA info if present. 98 if (TBAATag) 99 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag); 100 101 if (ScopeTag) 102 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag); 103 104 if (NoAliasTag) 105 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag); 106 107 return CI; 108} 109 110CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet( 111 Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize, 112 MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) { 113 114 Ptr = getCastedInt8PtrValue(Ptr); 115 Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)}; 116 Type *Tys[] = {Ptr->getType(), Size->getType()}; 117 Module *M = BB->getParent()->getParent(); 118 Function *TheFn = Intrinsic::getDeclaration( 119 M, Intrinsic::memset_element_unordered_atomic, Tys); 120 121 CallInst *CI = createCallHelper(TheFn, Ops, this); 122 123 cast<AtomicMemSetInst>(CI)->setDestAlignment(Alignment); 124 125 // Set the TBAA info if present. 126 if (TBAATag) 127 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag); 128 129 if (ScopeTag) 130 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag); 131 132 if (NoAliasTag) 133 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag); 134 135 return CI; 136} 137 138CallInst *IRBuilderBase::CreateMemCpy(Value *Dst, MaybeAlign DstAlign, 139 Value *Src, MaybeAlign SrcAlign, 140 Value *Size, bool isVolatile, 141 MDNode *TBAATag, MDNode *TBAAStructTag, 142 MDNode *ScopeTag, MDNode *NoAliasTag) { 143 Dst = getCastedInt8PtrValue(Dst); 144 Src = getCastedInt8PtrValue(Src); 145 146 Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)}; 147 Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() }; 148 Module *M = BB->getParent()->getParent(); 149 Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy, Tys); 150 151 CallInst *CI = createCallHelper(TheFn, Ops, this); 152 153 auto* MCI = cast<MemCpyInst>(CI); 154 if (DstAlign) 155 MCI->setDestAlignment(*DstAlign); 156 if (SrcAlign) 157 MCI->setSourceAlignment(*SrcAlign); 158 159 // Set the TBAA info if present. 160 if (TBAATag) 161 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag); 162 163 // Set the TBAA Struct info if present. 164 if (TBAAStructTag) 165 CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag); 166 167 if (ScopeTag) 168 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag); 169 170 if (NoAliasTag) 171 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag); 172 173 return CI; 174} 175 176CallInst *IRBuilderBase::CreateMemCpyInline(Value *Dst, MaybeAlign DstAlign, 177 Value *Src, MaybeAlign SrcAlign, 178 Value *Size) { 179 Dst = getCastedInt8PtrValue(Dst); 180 Src = getCastedInt8PtrValue(Src); 181 Value *IsVolatile = getInt1(false); 182 183 Value *Ops[] = {Dst, Src, Size, IsVolatile}; 184 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()}; 185 Function *F = BB->getParent(); 186 Module *M = F->getParent(); 187 Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy_inline, Tys); 188 189 CallInst *CI = createCallHelper(TheFn, Ops, this); 190 191 auto *MCI = cast<MemCpyInlineInst>(CI); 192 if (DstAlign) 193 MCI->setDestAlignment(*DstAlign); 194 if (SrcAlign) 195 MCI->setSourceAlignment(*SrcAlign); 196 197 return CI; 198} 199 200CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy( 201 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size, 202 uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag, 203 MDNode *ScopeTag, MDNode *NoAliasTag) { 204 assert(DstAlign >= ElementSize && 205 "Pointer alignment must be at least element size"); 206 assert(SrcAlign >= ElementSize && 207 "Pointer alignment must be at least element size"); 208 Dst = getCastedInt8PtrValue(Dst); 209 Src = getCastedInt8PtrValue(Src); 210 211 Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)}; 212 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()}; 213 Module *M = BB->getParent()->getParent(); 214 Function *TheFn = Intrinsic::getDeclaration( 215 M, Intrinsic::memcpy_element_unordered_atomic, Tys); 216 217 CallInst *CI = createCallHelper(TheFn, Ops, this); 218 219 // Set the alignment of the pointer args. 220 auto *AMCI = cast<AtomicMemCpyInst>(CI); 221 AMCI->setDestAlignment(DstAlign); 222 AMCI->setSourceAlignment(SrcAlign); 223 224 // Set the TBAA info if present. 225 if (TBAATag) 226 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag); 227 228 // Set the TBAA Struct info if present. 229 if (TBAAStructTag) 230 CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag); 231 232 if (ScopeTag) 233 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag); 234 235 if (NoAliasTag) 236 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag); 237 238 return CI; 239} 240 241CallInst *IRBuilderBase::CreateMemMove(Value *Dst, MaybeAlign DstAlign, 242 Value *Src, MaybeAlign SrcAlign, 243 Value *Size, bool isVolatile, 244 MDNode *TBAATag, MDNode *ScopeTag, 245 MDNode *NoAliasTag) { 246 Dst = getCastedInt8PtrValue(Dst); 247 Src = getCastedInt8PtrValue(Src); 248 249 Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)}; 250 Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() }; 251 Module *M = BB->getParent()->getParent(); 252 Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys); 253 254 CallInst *CI = createCallHelper(TheFn, Ops, this); 255 256 auto *MMI = cast<MemMoveInst>(CI); 257 if (DstAlign) 258 MMI->setDestAlignment(*DstAlign); 259 if (SrcAlign) 260 MMI->setSourceAlignment(*SrcAlign); 261 262 // Set the TBAA info if present. 263 if (TBAATag) 264 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag); 265 266 if (ScopeTag) 267 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag); 268 269 if (NoAliasTag) 270 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag); 271 272 return CI; 273} 274 275CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove( 276 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size, 277 uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag, 278 MDNode *ScopeTag, MDNode *NoAliasTag) { 279 assert(DstAlign >= ElementSize && 280 "Pointer alignment must be at least element size"); 281 assert(SrcAlign >= ElementSize && 282 "Pointer alignment must be at least element size"); 283 Dst = getCastedInt8PtrValue(Dst); 284 Src = getCastedInt8PtrValue(Src); 285 286 Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)}; 287 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()}; 288 Module *M = BB->getParent()->getParent(); 289 Function *TheFn = Intrinsic::getDeclaration( 290 M, Intrinsic::memmove_element_unordered_atomic, Tys); 291 292 CallInst *CI = createCallHelper(TheFn, Ops, this); 293 294 // Set the alignment of the pointer args. 295 CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign)); 296 CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign)); 297 298 // Set the TBAA info if present. 299 if (TBAATag) 300 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag); 301 302 // Set the TBAA Struct info if present. 303 if (TBAAStructTag) 304 CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag); 305 306 if (ScopeTag) 307 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag); 308 309 if (NoAliasTag) 310 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag); 311 312 return CI; 313} 314 315static CallInst *getReductionIntrinsic(IRBuilderBase *Builder, Intrinsic::ID ID, 316 Value *Src) { 317 Module *M = Builder->GetInsertBlock()->getParent()->getParent(); 318 Value *Ops[] = {Src}; 319 Type *Tys[] = { Src->getType() }; 320 auto Decl = Intrinsic::getDeclaration(M, ID, Tys); 321 return createCallHelper(Decl, Ops, Builder); 322} 323 324CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) { 325 Module *M = GetInsertBlock()->getParent()->getParent(); 326 Value *Ops[] = {Acc, Src}; 327 Type *Tys[] = {Acc->getType(), Src->getType()}; 328 auto Decl = Intrinsic::getDeclaration( 329 M, Intrinsic::experimental_vector_reduce_v2_fadd, Tys); 330 return createCallHelper(Decl, Ops, this); 331} 332 333CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) { 334 Module *M = GetInsertBlock()->getParent()->getParent(); 335 Value *Ops[] = {Acc, Src}; 336 Type *Tys[] = {Acc->getType(), Src->getType()}; 337 auto Decl = Intrinsic::getDeclaration( 338 M, Intrinsic::experimental_vector_reduce_v2_fmul, Tys); 339 return createCallHelper(Decl, Ops, this); 340} 341 342CallInst *IRBuilderBase::CreateAddReduce(Value *Src) { 343 return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_add, 344 Src); 345} 346 347CallInst *IRBuilderBase::CreateMulReduce(Value *Src) { 348 return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_mul, 349 Src); 350} 351 352CallInst *IRBuilderBase::CreateAndReduce(Value *Src) { 353 return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_and, 354 Src); 355} 356 357CallInst *IRBuilderBase::CreateOrReduce(Value *Src) { 358 return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_or, 359 Src); 360} 361 362CallInst *IRBuilderBase::CreateXorReduce(Value *Src) { 363 return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_xor, 364 Src); 365} 366 367CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) { 368 auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smax 369 : Intrinsic::experimental_vector_reduce_umax; 370 return getReductionIntrinsic(this, ID, Src); 371} 372 373CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) { 374 auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smin 375 : Intrinsic::experimental_vector_reduce_umin; 376 return getReductionIntrinsic(this, ID, Src); 377} 378 379CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src, bool NoNaN) { 380 auto Rdx = getReductionIntrinsic( 381 this, Intrinsic::experimental_vector_reduce_fmax, Src); 382 if (NoNaN) { 383 FastMathFlags FMF; 384 FMF.setNoNaNs(); 385 Rdx->setFastMathFlags(FMF); 386 } 387 return Rdx; 388} 389 390CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src, bool NoNaN) { 391 auto Rdx = getReductionIntrinsic( 392 this, Intrinsic::experimental_vector_reduce_fmin, Src); 393 if (NoNaN) { 394 FastMathFlags FMF; 395 FMF.setNoNaNs(); 396 Rdx->setFastMathFlags(FMF); 397 } 398 return Rdx; 399} 400 401CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) { 402 assert(isa<PointerType>(Ptr->getType()) && 403 "lifetime.start only applies to pointers."); 404 Ptr = getCastedInt8PtrValue(Ptr); 405 if (!Size) 406 Size = getInt64(-1); 407 else 408 assert(Size->getType() == getInt64Ty() && 409 "lifetime.start requires the size to be an i64"); 410 Value *Ops[] = { Size, Ptr }; 411 Module *M = BB->getParent()->getParent(); 412 Function *TheFn = 413 Intrinsic::getDeclaration(M, Intrinsic::lifetime_start, {Ptr->getType()}); 414 return createCallHelper(TheFn, Ops, this); 415} 416 417CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) { 418 assert(isa<PointerType>(Ptr->getType()) && 419 "lifetime.end only applies to pointers."); 420 Ptr = getCastedInt8PtrValue(Ptr); 421 if (!Size) 422 Size = getInt64(-1); 423 else 424 assert(Size->getType() == getInt64Ty() && 425 "lifetime.end requires the size to be an i64"); 426 Value *Ops[] = { Size, Ptr }; 427 Module *M = BB->getParent()->getParent(); 428 Function *TheFn = 429 Intrinsic::getDeclaration(M, Intrinsic::lifetime_end, {Ptr->getType()}); 430 return createCallHelper(TheFn, Ops, this); 431} 432 433CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) { 434 435 assert(isa<PointerType>(Ptr->getType()) && 436 "invariant.start only applies to pointers."); 437 Ptr = getCastedInt8PtrValue(Ptr); 438 if (!Size) 439 Size = getInt64(-1); 440 else 441 assert(Size->getType() == getInt64Ty() && 442 "invariant.start requires the size to be an i64"); 443 444 Value *Ops[] = {Size, Ptr}; 445 // Fill in the single overloaded type: memory object type. 446 Type *ObjectPtr[1] = {Ptr->getType()}; 447 Module *M = BB->getParent()->getParent(); 448 Function *TheFn = 449 Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr); 450 return createCallHelper(TheFn, Ops, this); 451} 452 453CallInst *IRBuilderBase::CreateAssumption(Value *Cond) { 454 assert(Cond->getType() == getInt1Ty() && 455 "an assumption condition must be of type i1"); 456 457 Value *Ops[] = { Cond }; 458 Module *M = BB->getParent()->getParent(); 459 Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume); 460 return createCallHelper(FnAssume, Ops, this); 461} 462 463/// Create a call to a Masked Load intrinsic. 464/// \p Ptr - base pointer for the load 465/// \p Alignment - alignment of the source location 466/// \p Mask - vector of booleans which indicates what vector lanes should 467/// be accessed in memory 468/// \p PassThru - pass-through value that is used to fill the masked-off lanes 469/// of the result 470/// \p Name - name of the result variable 471CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, Align Alignment, 472 Value *Mask, Value *PassThru, 473 const Twine &Name) { 474 auto *PtrTy = cast<PointerType>(Ptr->getType()); 475 Type *DataTy = PtrTy->getElementType(); 476 assert(DataTy->isVectorTy() && "Ptr should point to a vector"); 477 assert(Mask && "Mask should not be all-ones (null)"); 478 if (!PassThru) 479 PassThru = UndefValue::get(DataTy); 480 Type *OverloadedTypes[] = { DataTy, PtrTy }; 481 Value *Ops[] = {Ptr, getInt32(Alignment.value()), Mask, PassThru}; 482 return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops, 483 OverloadedTypes, Name); 484} 485 486/// Create a call to a Masked Store intrinsic. 487/// \p Val - data to be stored, 488/// \p Ptr - base pointer for the store 489/// \p Alignment - alignment of the destination location 490/// \p Mask - vector of booleans which indicates what vector lanes should 491/// be accessed in memory 492CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr, 493 Align Alignment, Value *Mask) { 494 auto *PtrTy = cast<PointerType>(Ptr->getType()); 495 Type *DataTy = PtrTy->getElementType(); 496 assert(DataTy->isVectorTy() && "Ptr should point to a vector"); 497 assert(Mask && "Mask should not be all-ones (null)"); 498 Type *OverloadedTypes[] = { DataTy, PtrTy }; 499 Value *Ops[] = {Val, Ptr, getInt32(Alignment.value()), Mask}; 500 return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes); 501} 502 503/// Create a call to a Masked intrinsic, with given intrinsic Id, 504/// an array of operands - Ops, and an array of overloaded types - 505/// OverloadedTypes. 506CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id, 507 ArrayRef<Value *> Ops, 508 ArrayRef<Type *> OverloadedTypes, 509 const Twine &Name) { 510 Module *M = BB->getParent()->getParent(); 511 Function *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes); 512 return createCallHelper(TheFn, Ops, this, Name); 513} 514 515/// Create a call to a Masked Gather intrinsic. 516/// \p Ptrs - vector of pointers for loading 517/// \p Align - alignment for one element 518/// \p Mask - vector of booleans which indicates what vector lanes should 519/// be accessed in memory 520/// \p PassThru - pass-through value that is used to fill the masked-off lanes 521/// of the result 522/// \p Name - name of the result variable 523CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, Align Alignment, 524 Value *Mask, Value *PassThru, 525 const Twine &Name) { 526 auto PtrsTy = cast<VectorType>(Ptrs->getType()); 527 auto PtrTy = cast<PointerType>(PtrsTy->getElementType()); 528 unsigned NumElts = PtrsTy->getNumElements(); 529 auto *DataTy = FixedVectorType::get(PtrTy->getElementType(), NumElts); 530 531 if (!Mask) 532 Mask = Constant::getAllOnesValue( 533 FixedVectorType::get(Type::getInt1Ty(Context), NumElts)); 534 535 if (!PassThru) 536 PassThru = UndefValue::get(DataTy); 537 538 Type *OverloadedTypes[] = {DataTy, PtrsTy}; 539 Value *Ops[] = {Ptrs, getInt32(Alignment.value()), Mask, PassThru}; 540 541 // We specify only one type when we create this intrinsic. Types of other 542 // arguments are derived from this type. 543 return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes, 544 Name); 545} 546 547/// Create a call to a Masked Scatter intrinsic. 548/// \p Data - data to be stored, 549/// \p Ptrs - the vector of pointers, where the \p Data elements should be 550/// stored 551/// \p Align - alignment for one element 552/// \p Mask - vector of booleans which indicates what vector lanes should 553/// be accessed in memory 554CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs, 555 Align Alignment, Value *Mask) { 556 auto PtrsTy = cast<VectorType>(Ptrs->getType()); 557 auto DataTy = cast<VectorType>(Data->getType()); 558 unsigned NumElts = PtrsTy->getNumElements(); 559 560#ifndef NDEBUG 561 auto PtrTy = cast<PointerType>(PtrsTy->getElementType()); 562 assert(NumElts == DataTy->getNumElements() && 563 PtrTy->getElementType() == DataTy->getElementType() && 564 "Incompatible pointer and data types"); 565#endif 566 567 if (!Mask) 568 Mask = Constant::getAllOnesValue( 569 FixedVectorType::get(Type::getInt1Ty(Context), NumElts)); 570 571 Type *OverloadedTypes[] = {DataTy, PtrsTy}; 572 Value *Ops[] = {Data, Ptrs, getInt32(Alignment.value()), Mask}; 573 574 // We specify only one type when we create this intrinsic. Types of other 575 // arguments are derived from this type. 576 return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes); 577} 578 579template <typename T0> 580static std::vector<Value *> 581getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes, 582 Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs) { 583 std::vector<Value *> Args; 584 Args.push_back(B.getInt64(ID)); 585 Args.push_back(B.getInt32(NumPatchBytes)); 586 Args.push_back(ActualCallee); 587 Args.push_back(B.getInt32(CallArgs.size())); 588 Args.push_back(B.getInt32(Flags)); 589 Args.insert(Args.end(), CallArgs.begin(), CallArgs.end()); 590 // GC Transition and Deopt args are now always handled via operand bundle. 591 // They will be removed from the signature of gc.statepoint shortly. 592 Args.push_back(B.getInt32(0)); 593 Args.push_back(B.getInt32(0)); 594 // GC args are now encoded in the gc-live operand bundle 595 return Args; 596} 597 598template<typename T1, typename T2, typename T3> 599static std::vector<OperandBundleDef> 600getStatepointBundles(Optional<ArrayRef<T1>> TransitionArgs, 601 Optional<ArrayRef<T2>> DeoptArgs, 602 ArrayRef<T3> GCArgs) { 603 std::vector<OperandBundleDef> Rval; 604 if (DeoptArgs) { 605 SmallVector<Value*, 16> DeoptValues; 606 DeoptValues.insert(DeoptValues.end(), DeoptArgs->begin(), DeoptArgs->end()); 607 Rval.emplace_back("deopt", DeoptValues); 608 } 609 if (TransitionArgs) { 610 SmallVector<Value*, 16> TransitionValues; 611 TransitionValues.insert(TransitionValues.end(), 612 TransitionArgs->begin(), TransitionArgs->end()); 613 Rval.emplace_back("gc-transition", TransitionValues); 614 } 615 if (GCArgs.size()) { 616 SmallVector<Value*, 16> LiveValues; 617 LiveValues.insert(LiveValues.end(), GCArgs.begin(), GCArgs.end()); 618 Rval.emplace_back("gc-live", LiveValues); 619 } 620 return Rval; 621} 622 623template <typename T0, typename T1, typename T2, typename T3> 624static CallInst *CreateGCStatepointCallCommon( 625 IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes, 626 Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs, 627 Optional<ArrayRef<T1>> TransitionArgs, 628 Optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs, 629 const Twine &Name) { 630 // Extract out the type of the callee. 631 auto *FuncPtrType = cast<PointerType>(ActualCallee->getType()); 632 assert(isa<FunctionType>(FuncPtrType->getElementType()) && 633 "actual callee must be a callable value"); 634 635 Module *M = Builder->GetInsertBlock()->getParent()->getParent(); 636 // Fill in the one generic type'd argument (the function is also vararg) 637 Type *ArgTypes[] = { FuncPtrType }; 638 Function *FnStatepoint = 639 Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint, 640 ArgTypes); 641 642 std::vector<Value *> Args = 643 getStatepointArgs(*Builder, ID, NumPatchBytes, ActualCallee, Flags, 644 CallArgs); 645 646 return Builder->CreateCall(FnStatepoint, Args, 647 getStatepointBundles(TransitionArgs, DeoptArgs, 648 GCArgs), 649 Name); 650} 651 652CallInst *IRBuilderBase::CreateGCStatepointCall( 653 uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, 654 ArrayRef<Value *> CallArgs, Optional<ArrayRef<Value *>> DeoptArgs, 655 ArrayRef<Value *> GCArgs, const Twine &Name) { 656 return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>( 657 this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None), 658 CallArgs, None /* No Transition Args */, DeoptArgs, GCArgs, Name); 659} 660 661CallInst *IRBuilderBase::CreateGCStatepointCall( 662 uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags, 663 ArrayRef<Use> CallArgs, Optional<ArrayRef<Use>> TransitionArgs, 664 Optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs, 665 const Twine &Name) { 666 return CreateGCStatepointCallCommon<Use, Use, Use, Value *>( 667 this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs, 668 DeoptArgs, GCArgs, Name); 669} 670 671CallInst *IRBuilderBase::CreateGCStatepointCall( 672 uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, 673 ArrayRef<Use> CallArgs, Optional<ArrayRef<Value *>> DeoptArgs, 674 ArrayRef<Value *> GCArgs, const Twine &Name) { 675 return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>( 676 this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None), 677 CallArgs, None, DeoptArgs, GCArgs, Name); 678} 679 680template <typename T0, typename T1, typename T2, typename T3> 681static InvokeInst *CreateGCStatepointInvokeCommon( 682 IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes, 683 Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest, 684 uint32_t Flags, ArrayRef<T0> InvokeArgs, 685 Optional<ArrayRef<T1>> TransitionArgs, Optional<ArrayRef<T2>> DeoptArgs, 686 ArrayRef<T3> GCArgs, const Twine &Name) { 687 // Extract out the type of the callee. 688 auto *FuncPtrType = cast<PointerType>(ActualInvokee->getType()); 689 assert(isa<FunctionType>(FuncPtrType->getElementType()) && 690 "actual callee must be a callable value"); 691 692 Module *M = Builder->GetInsertBlock()->getParent()->getParent(); 693 // Fill in the one generic type'd argument (the function is also vararg) 694 Function *FnStatepoint = Intrinsic::getDeclaration( 695 M, Intrinsic::experimental_gc_statepoint, {FuncPtrType}); 696 697 std::vector<Value *> Args = 698 getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee, Flags, 699 InvokeArgs); 700 701 return Builder->CreateInvoke(FnStatepoint, NormalDest, UnwindDest, Args, 702 getStatepointBundles(TransitionArgs, DeoptArgs, 703 GCArgs), 704 Name); 705} 706 707InvokeInst *IRBuilderBase::CreateGCStatepointInvoke( 708 uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee, 709 BasicBlock *NormalDest, BasicBlock *UnwindDest, 710 ArrayRef<Value *> InvokeArgs, Optional<ArrayRef<Value *>> DeoptArgs, 711 ArrayRef<Value *> GCArgs, const Twine &Name) { 712 return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>( 713 this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, 714 uint32_t(StatepointFlags::None), InvokeArgs, None /* No Transition Args*/, 715 DeoptArgs, GCArgs, Name); 716} 717 718InvokeInst *IRBuilderBase::CreateGCStatepointInvoke( 719 uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee, 720 BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags, 721 ArrayRef<Use> InvokeArgs, Optional<ArrayRef<Use>> TransitionArgs, 722 Optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) { 723 return CreateGCStatepointInvokeCommon<Use, Use, Use, Value *>( 724 this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags, 725 InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name); 726} 727 728InvokeInst *IRBuilderBase::CreateGCStatepointInvoke( 729 uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee, 730 BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs, 731 Optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) { 732 return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>( 733 this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, 734 uint32_t(StatepointFlags::None), InvokeArgs, None, DeoptArgs, GCArgs, 735 Name); 736} 737 738CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint, 739 Type *ResultType, 740 const Twine &Name) { 741 Intrinsic::ID ID = Intrinsic::experimental_gc_result; 742 Module *M = BB->getParent()->getParent(); 743 Type *Types[] = {ResultType}; 744 Function *FnGCResult = Intrinsic::getDeclaration(M, ID, Types); 745 746 Value *Args[] = {Statepoint}; 747 return createCallHelper(FnGCResult, Args, this, Name); 748} 749 750CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint, 751 int BaseOffset, 752 int DerivedOffset, 753 Type *ResultType, 754 const Twine &Name) { 755 Module *M = BB->getParent()->getParent(); 756 Type *Types[] = {ResultType}; 757 Function *FnGCRelocate = 758 Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types); 759 760 Value *Args[] = {Statepoint, 761 getInt32(BaseOffset), 762 getInt32(DerivedOffset)}; 763 return createCallHelper(FnGCRelocate, Args, this, Name); 764} 765 766CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V, 767 Instruction *FMFSource, 768 const Twine &Name) { 769 Module *M = BB->getModule(); 770 Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()}); 771 return createCallHelper(Fn, {V}, this, Name, FMFSource); 772} 773 774CallInst *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, 775 Value *RHS, 776 Instruction *FMFSource, 777 const Twine &Name) { 778 Module *M = BB->getModule(); 779 Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() }); 780 return createCallHelper(Fn, {LHS, RHS}, this, Name, FMFSource); 781} 782 783CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID, 784 ArrayRef<Type *> Types, 785 ArrayRef<Value *> Args, 786 Instruction *FMFSource, 787 const Twine &Name) { 788 Module *M = BB->getModule(); 789 Function *Fn = Intrinsic::getDeclaration(M, ID, Types); 790 return createCallHelper(Fn, Args, this, Name, FMFSource); 791} 792 793CallInst *IRBuilderBase::CreateConstrainedFPBinOp( 794 Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource, 795 const Twine &Name, MDNode *FPMathTag, 796 Optional<RoundingMode> Rounding, 797 Optional<fp::ExceptionBehavior> Except) { 798 Value *RoundingV = getConstrainedFPRounding(Rounding); 799 Value *ExceptV = getConstrainedFPExcept(Except); 800 801 FastMathFlags UseFMF = FMF; 802 if (FMFSource) 803 UseFMF = FMFSource->getFastMathFlags(); 804 805 CallInst *C = CreateIntrinsic(ID, {L->getType()}, 806 {L, R, RoundingV, ExceptV}, nullptr, Name); 807 setConstrainedFPCallAttr(C); 808 setFPAttrs(C, FPMathTag, UseFMF); 809 return C; 810} 811 812Value *IRBuilderBase::CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops, 813 const Twine &Name, MDNode *FPMathTag) { 814 if (Instruction::isBinaryOp(Opc)) { 815 assert(Ops.size() == 2 && "Invalid number of operands!"); 816 return CreateBinOp(static_cast<Instruction::BinaryOps>(Opc), 817 Ops[0], Ops[1], Name, FPMathTag); 818 } 819 if (Instruction::isUnaryOp(Opc)) { 820 assert(Ops.size() == 1 && "Invalid number of operands!"); 821 return CreateUnOp(static_cast<Instruction::UnaryOps>(Opc), 822 Ops[0], Name, FPMathTag); 823 } 824 llvm_unreachable("Unexpected opcode!"); 825} 826 827CallInst *IRBuilderBase::CreateConstrainedFPCast( 828 Intrinsic::ID ID, Value *V, Type *DestTy, 829 Instruction *FMFSource, const Twine &Name, MDNode *FPMathTag, 830 Optional<RoundingMode> Rounding, 831 Optional<fp::ExceptionBehavior> Except) { 832 Value *ExceptV = getConstrainedFPExcept(Except); 833 834 FastMathFlags UseFMF = FMF; 835 if (FMFSource) 836 UseFMF = FMFSource->getFastMathFlags(); 837 838 CallInst *C; 839 bool HasRoundingMD = false; 840 switch (ID) { 841 default: 842 break; 843#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \ 844 case Intrinsic::INTRINSIC: \ 845 HasRoundingMD = ROUND_MODE; \ 846 break; 847#include "llvm/IR/ConstrainedOps.def" 848 } 849 if (HasRoundingMD) { 850 Value *RoundingV = getConstrainedFPRounding(Rounding); 851 C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, RoundingV, ExceptV}, 852 nullptr, Name); 853 } else 854 C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, ExceptV}, nullptr, 855 Name); 856 857 setConstrainedFPCallAttr(C); 858 859 if (isa<FPMathOperator>(C)) 860 setFPAttrs(C, FPMathTag, UseFMF); 861 return C; 862} 863 864Value *IRBuilderBase::CreateFCmpHelper( 865 CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name, 866 MDNode *FPMathTag, bool IsSignaling) { 867 if (IsFPConstrained) { 868 auto ID = IsSignaling ? Intrinsic::experimental_constrained_fcmps 869 : Intrinsic::experimental_constrained_fcmp; 870 return CreateConstrainedFPCmp(ID, P, LHS, RHS, Name); 871 } 872 873 if (auto *LC = dyn_cast<Constant>(LHS)) 874 if (auto *RC = dyn_cast<Constant>(RHS)) 875 return Insert(Folder.CreateFCmp(P, LC, RC), Name); 876 return Insert(setFPAttrs(new FCmpInst(P, LHS, RHS), FPMathTag, FMF), Name); 877} 878 879CallInst *IRBuilderBase::CreateConstrainedFPCmp( 880 Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R, 881 const Twine &Name, Optional<fp::ExceptionBehavior> Except) { 882 Value *PredicateV = getConstrainedFPPredicate(P); 883 Value *ExceptV = getConstrainedFPExcept(Except); 884 885 CallInst *C = CreateIntrinsic(ID, {L->getType()}, 886 {L, R, PredicateV, ExceptV}, nullptr, Name); 887 setConstrainedFPCallAttr(C); 888 return C; 889} 890 891CallInst *IRBuilderBase::CreateConstrainedFPCall( 892 Function *Callee, ArrayRef<Value *> Args, const Twine &Name, 893 Optional<RoundingMode> Rounding, 894 Optional<fp::ExceptionBehavior> Except) { 895 llvm::SmallVector<Value *, 6> UseArgs; 896 897 for (auto *OneArg : Args) 898 UseArgs.push_back(OneArg); 899 bool HasRoundingMD = false; 900 switch (Callee->getIntrinsicID()) { 901 default: 902 break; 903#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \ 904 case Intrinsic::INTRINSIC: \ 905 HasRoundingMD = ROUND_MODE; \ 906 break; 907#include "llvm/IR/ConstrainedOps.def" 908 } 909 if (HasRoundingMD) 910 UseArgs.push_back(getConstrainedFPRounding(Rounding)); 911 UseArgs.push_back(getConstrainedFPExcept(Except)); 912 913 CallInst *C = CreateCall(Callee, UseArgs, Name); 914 setConstrainedFPCallAttr(C); 915 return C; 916} 917 918Value *IRBuilderBase::CreateSelect(Value *C, Value *True, Value *False, 919 const Twine &Name, Instruction *MDFrom) { 920 if (auto *CC = dyn_cast<Constant>(C)) 921 if (auto *TC = dyn_cast<Constant>(True)) 922 if (auto *FC = dyn_cast<Constant>(False)) 923 return Insert(Folder.CreateSelect(CC, TC, FC), Name); 924 925 SelectInst *Sel = SelectInst::Create(C, True, False); 926 if (MDFrom) { 927 MDNode *Prof = MDFrom->getMetadata(LLVMContext::MD_prof); 928 MDNode *Unpred = MDFrom->getMetadata(LLVMContext::MD_unpredictable); 929 Sel = addBranchMetadata(Sel, Prof, Unpred); 930 } 931 if (isa<FPMathOperator>(Sel)) 932 setFPAttrs(Sel, nullptr /* MDNode* */, FMF); 933 return Insert(Sel, Name); 934} 935 936Value *IRBuilderBase::CreatePtrDiff(Value *LHS, Value *RHS, 937 const Twine &Name) { 938 assert(LHS->getType() == RHS->getType() && 939 "Pointer subtraction operand types must match!"); 940 auto *ArgType = cast<PointerType>(LHS->getType()); 941 Value *LHS_int = CreatePtrToInt(LHS, Type::getInt64Ty(Context)); 942 Value *RHS_int = CreatePtrToInt(RHS, Type::getInt64Ty(Context)); 943 Value *Difference = CreateSub(LHS_int, RHS_int); 944 return CreateExactSDiv(Difference, 945 ConstantExpr::getSizeOf(ArgType->getElementType()), 946 Name); 947} 948 949Value *IRBuilderBase::CreateLaunderInvariantGroup(Value *Ptr) { 950 assert(isa<PointerType>(Ptr->getType()) && 951 "launder.invariant.group only applies to pointers."); 952 // FIXME: we could potentially avoid casts to/from i8*. 953 auto *PtrType = Ptr->getType(); 954 auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace()); 955 if (PtrType != Int8PtrTy) 956 Ptr = CreateBitCast(Ptr, Int8PtrTy); 957 Module *M = BB->getParent()->getParent(); 958 Function *FnLaunderInvariantGroup = Intrinsic::getDeclaration( 959 M, Intrinsic::launder_invariant_group, {Int8PtrTy}); 960 961 assert(FnLaunderInvariantGroup->getReturnType() == Int8PtrTy && 962 FnLaunderInvariantGroup->getFunctionType()->getParamType(0) == 963 Int8PtrTy && 964 "LaunderInvariantGroup should take and return the same type"); 965 966 CallInst *Fn = CreateCall(FnLaunderInvariantGroup, {Ptr}); 967 968 if (PtrType != Int8PtrTy) 969 return CreateBitCast(Fn, PtrType); 970 return Fn; 971} 972 973Value *IRBuilderBase::CreateStripInvariantGroup(Value *Ptr) { 974 assert(isa<PointerType>(Ptr->getType()) && 975 "strip.invariant.group only applies to pointers."); 976 977 // FIXME: we could potentially avoid casts to/from i8*. 978 auto *PtrType = Ptr->getType(); 979 auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace()); 980 if (PtrType != Int8PtrTy) 981 Ptr = CreateBitCast(Ptr, Int8PtrTy); 982 Module *M = BB->getParent()->getParent(); 983 Function *FnStripInvariantGroup = Intrinsic::getDeclaration( 984 M, Intrinsic::strip_invariant_group, {Int8PtrTy}); 985 986 assert(FnStripInvariantGroup->getReturnType() == Int8PtrTy && 987 FnStripInvariantGroup->getFunctionType()->getParamType(0) == 988 Int8PtrTy && 989 "StripInvariantGroup should take and return the same type"); 990 991 CallInst *Fn = CreateCall(FnStripInvariantGroup, {Ptr}); 992 993 if (PtrType != Int8PtrTy) 994 return CreateBitCast(Fn, PtrType); 995 return Fn; 996} 997 998Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V, 999 const Twine &Name) { 1000 assert(NumElts > 0 && "Cannot splat to an empty vector!"); 1001 1002 // First insert it into an undef vector so we can shuffle it. 1003 Type *I32Ty = getInt32Ty(); 1004 Value *Undef = UndefValue::get(FixedVectorType::get(V->getType(), NumElts)); 1005 V = CreateInsertElement(Undef, V, ConstantInt::get(I32Ty, 0), 1006 Name + ".splatinsert"); 1007 1008 // Shuffle the value across the desired number of elements. 1009 Value *Zeros = 1010 ConstantAggregateZero::get(FixedVectorType::get(I32Ty, NumElts)); 1011 return CreateShuffleVector(V, Undef, Zeros, Name + ".splat"); 1012} 1013 1014Value *IRBuilderBase::CreateExtractInteger( 1015 const DataLayout &DL, Value *From, IntegerType *ExtractedTy, 1016 uint64_t Offset, const Twine &Name) { 1017 auto *IntTy = cast<IntegerType>(From->getType()); 1018 assert(DL.getTypeStoreSize(ExtractedTy) + Offset <= 1019 DL.getTypeStoreSize(IntTy) && 1020 "Element extends past full value"); 1021 uint64_t ShAmt = 8 * Offset; 1022 Value *V = From; 1023 if (DL.isBigEndian()) 1024 ShAmt = 8 * (DL.getTypeStoreSize(IntTy) - 1025 DL.getTypeStoreSize(ExtractedTy) - Offset); 1026 if (ShAmt) { 1027 V = CreateLShr(V, ShAmt, Name + ".shift"); 1028 } 1029 assert(ExtractedTy->getBitWidth() <= IntTy->getBitWidth() && 1030 "Cannot extract to a larger integer!"); 1031 if (ExtractedTy != IntTy) { 1032 V = CreateTrunc(V, ExtractedTy, Name + ".trunc"); 1033 } 1034 return V; 1035} 1036 1037Value *IRBuilderBase::CreatePreserveArrayAccessIndex( 1038 Type *ElTy, Value *Base, unsigned Dimension, unsigned LastIndex, 1039 MDNode *DbgInfo) { 1040 assert(isa<PointerType>(Base->getType()) && 1041 "Invalid Base ptr type for preserve.array.access.index."); 1042 auto *BaseType = Base->getType(); 1043 1044 Value *LastIndexV = getInt32(LastIndex); 1045 Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0); 1046 SmallVector<Value *, 4> IdxList; 1047 for (unsigned I = 0; I < Dimension; ++I) 1048 IdxList.push_back(Zero); 1049 IdxList.push_back(LastIndexV); 1050 1051 Type *ResultType = 1052 GetElementPtrInst::getGEPReturnType(ElTy, Base, IdxList); 1053 1054 Module *M = BB->getParent()->getParent(); 1055 Function *FnPreserveArrayAccessIndex = Intrinsic::getDeclaration( 1056 M, Intrinsic::preserve_array_access_index, {ResultType, BaseType}); 1057 1058 Value *DimV = getInt32(Dimension); 1059 CallInst *Fn = 1060 CreateCall(FnPreserveArrayAccessIndex, {Base, DimV, LastIndexV}); 1061 if (DbgInfo) 1062 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); 1063 1064 return Fn; 1065} 1066 1067Value *IRBuilderBase::CreatePreserveUnionAccessIndex( 1068 Value *Base, unsigned FieldIndex, MDNode *DbgInfo) { 1069 assert(isa<PointerType>(Base->getType()) && 1070 "Invalid Base ptr type for preserve.union.access.index."); 1071 auto *BaseType = Base->getType(); 1072 1073 Module *M = BB->getParent()->getParent(); 1074 Function *FnPreserveUnionAccessIndex = Intrinsic::getDeclaration( 1075 M, Intrinsic::preserve_union_access_index, {BaseType, BaseType}); 1076 1077 Value *DIIndex = getInt32(FieldIndex); 1078 CallInst *Fn = 1079 CreateCall(FnPreserveUnionAccessIndex, {Base, DIIndex}); 1080 if (DbgInfo) 1081 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); 1082 1083 return Fn; 1084} 1085 1086Value *IRBuilderBase::CreatePreserveStructAccessIndex( 1087 Type *ElTy, Value *Base, unsigned Index, unsigned FieldIndex, 1088 MDNode *DbgInfo) { 1089 assert(isa<PointerType>(Base->getType()) && 1090 "Invalid Base ptr type for preserve.struct.access.index."); 1091 auto *BaseType = Base->getType(); 1092 1093 Value *GEPIndex = getInt32(Index); 1094 Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0); 1095 Type *ResultType = 1096 GetElementPtrInst::getGEPReturnType(ElTy, Base, {Zero, GEPIndex}); 1097 1098 Module *M = BB->getParent()->getParent(); 1099 Function *FnPreserveStructAccessIndex = Intrinsic::getDeclaration( 1100 M, Intrinsic::preserve_struct_access_index, {ResultType, BaseType}); 1101 1102 Value *DIIndex = getInt32(FieldIndex); 1103 CallInst *Fn = CreateCall(FnPreserveStructAccessIndex, 1104 {Base, GEPIndex, DIIndex}); 1105 if (DbgInfo) 1106 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); 1107 1108 return Fn; 1109} 1110 1111CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper( 1112 const DataLayout &DL, Value *PtrValue, Value *Mask, Type *IntPtrTy, 1113 Value *OffsetValue, Value **TheCheck) { 1114 Value *PtrIntValue = CreatePtrToInt(PtrValue, IntPtrTy, "ptrint"); 1115 1116 if (OffsetValue) { 1117 bool IsOffsetZero = false; 1118 if (const auto *CI = dyn_cast<ConstantInt>(OffsetValue)) 1119 IsOffsetZero = CI->isZero(); 1120 1121 if (!IsOffsetZero) { 1122 if (OffsetValue->getType() != IntPtrTy) 1123 OffsetValue = CreateIntCast(OffsetValue, IntPtrTy, /*isSigned*/ true, 1124 "offsetcast"); 1125 PtrIntValue = CreateSub(PtrIntValue, OffsetValue, "offsetptr"); 1126 } 1127 } 1128 1129 Value *Zero = ConstantInt::get(IntPtrTy, 0); 1130 Value *MaskedPtr = CreateAnd(PtrIntValue, Mask, "maskedptr"); 1131 Value *InvCond = CreateICmpEQ(MaskedPtr, Zero, "maskcond"); 1132 if (TheCheck) 1133 *TheCheck = InvCond; 1134 1135 return CreateAssumption(InvCond); 1136} 1137 1138CallInst *IRBuilderBase::CreateAlignmentAssumption( 1139 const DataLayout &DL, Value *PtrValue, unsigned Alignment, 1140 Value *OffsetValue, Value **TheCheck) { 1141 assert(isa<PointerType>(PtrValue->getType()) && 1142 "trying to create an alignment assumption on a non-pointer?"); 1143 assert(Alignment != 0 && "Invalid Alignment"); 1144 auto *PtrTy = cast<PointerType>(PtrValue->getType()); 1145 Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace()); 1146 1147 Value *Mask = ConstantInt::get(IntPtrTy, Alignment - 1); 1148 return CreateAlignmentAssumptionHelper(DL, PtrValue, Mask, IntPtrTy, 1149 OffsetValue, TheCheck); 1150} 1151 1152CallInst *IRBuilderBase::CreateAlignmentAssumption( 1153 const DataLayout &DL, Value *PtrValue, Value *Alignment, 1154 Value *OffsetValue, Value **TheCheck) { 1155 assert(isa<PointerType>(PtrValue->getType()) && 1156 "trying to create an alignment assumption on a non-pointer?"); 1157 auto *PtrTy = cast<PointerType>(PtrValue->getType()); 1158 Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace()); 1159 1160 if (Alignment->getType() != IntPtrTy) 1161 Alignment = CreateIntCast(Alignment, IntPtrTy, /*isSigned*/ false, 1162 "alignmentcast"); 1163 1164 Value *Mask = CreateSub(Alignment, ConstantInt::get(IntPtrTy, 1), "mask"); 1165 1166 return CreateAlignmentAssumptionHelper(DL, PtrValue, Mask, IntPtrTy, 1167 OffsetValue, TheCheck); 1168} 1169 1170IRBuilderDefaultInserter::~IRBuilderDefaultInserter() {} 1171IRBuilderCallbackInserter::~IRBuilderCallbackInserter() {} 1172IRBuilderFolder::~IRBuilderFolder() {} 1173void ConstantFolder::anchor() {} 1174void NoFolder::anchor() {} 1175