Lines Matching refs:Bld
643 CGBuilderTy &Bld = CGF.Builder;
644 return Bld.CreateAShr(getNVPTXThreadID(CGF), LaneIDBits, "nvptx_warp_id");
651 CGBuilderTy &Bld = CGF.Builder;
652 return Bld.CreateAnd(getNVPTXThreadID(CGF), Bld.getInt32(LaneIDMask),
671 CGBuilderTy &Bld = CGF.Builder;
674 : Bld.CreateNUWSub(getNVPTXNumThreads(CGF), getNVPTXWarpSize(CGF),
686 CGBuilderTy &Bld = CGF.Builder;
690 llvm::Value *Mask = Bld.CreateNUWSub(getNVPTXWarpSize(CGF), Bld.getInt32(1));
692 return Bld.CreateAnd(Bld.CreateNUWSub(NumThreads, Bld.getInt32(1)),
693 Bld.CreateNot(Mask), "master_tid");
1233 CGBuilderTy &Bld = CGF.Builder;
1241 Bld.CreateICmpULT(getNVPTXThreadID(CGF), getThreadLimit(CGF));
1242 Bld.CreateCondBr(IsWorker, WorkerBB, MasterCheckBB);
1250 Bld.CreateICmpEQ(getNVPTXThreadID(CGF), getMasterThreadID(CGF));
1251 Bld.CreateCondBr(IsMaster, MasterBB, EST.ExitBB);
1260 Bld.getInt16(/*RequiresOMPRuntime=*/1)};
1355 CGBuilderTy &Bld = CGF.Builder;
1363 Bld.getInt16(RequiresFullRuntime ? 1 : 0),
1364 /*RequiresDataSharing=*/Bld.getInt16(0)};
1443 CGBuilderTy &Bld = CGF.Builder;
1463 CGF.InitTempAlloca(ExecStatus, Bld.getInt8(/*C=*/0));
1470 Bld.CreateStore(Bld.CreateZExt(Ret, CGF.Int8Ty), ExecStatus);
1473 llvm::Value *WorkID = Bld.CreateLoad(WorkFn);
1474 llvm::Value *ShouldTerminate = Bld.CreateIsNull(WorkID, "should_terminate");
1475 Bld.CreateCondBr(ShouldTerminate, ExitBB, SelectWorkersBB);
1480 Bld.CreateIsNotNull(Bld.CreateLoad(ExecStatus), "is_active");
1481 Bld.CreateCondBr(IsActive, ExecuteBB, BarrierBB);
1491 llvm::Value *ID = Bld.CreatePointerBitCastOrAddrSpaceCast(W, CGM.Int8PtrTy);
1494 Bld.CreateICmpEQ(Bld.CreateLoad(WorkFn), ID, "work_match");
1498 Bld.CreateCondBr(WorkFnMatch, ExecuteFNBB, CheckNextBB);
1508 {Bld.getInt16(/*ParallelLevel=*/0), getThreadID(CGF, WST.Loc)});
1522 Bld.CreateBitCast(WorkID, ParallelFnTy->getPointerTo());
1528 {Bld.getInt16(/*ParallelLevel=*/0), getThreadID(CGF, WST.Loc)});
2141 CGBuilderTy &Bld = CGF.Builder;
2176 IsTTD = Bld.CreateIsNull(PL);
2178 llvm::Value *IsSPMD = Bld.CreateIsNotNull(CGF.EmitNounwindRuntimeCall(
2180 Bld.CreateCondBr(IsSPMD, SPMDBB, NonSPMDBB);
2205 Size = Bld.CreateSelect(
2216 GlobalRecCastAddr = Bld.CreatePointerBitCastOrAddrSpaceCast(
2219 auto *Phi = Bld.CreatePHI(GlobalRecPtrTy,
2269 llvm::Value *ResAddr = Bld.CreatePointerBitCastOrAddrSpaceCast(
2291 Bld.CreateConstInBoundsGEP(FrameAddr, Offset).getPointer();
2294 GlobalRecCastAddr = Bld.CreatePointerBitCastOrAddrSpaceCast(
2310 GlobalRecCastAddr = Bld.CreatePointerBitCastOrAddrSpaceCast(
2327 Bld.CreatePointerBitCastOrAddrSpaceCast(
2348 {Bld.getInt32(0), getNVPTXLaneID(CGF)});
2366 Address(Bld.CreateSelect(IsTTD, SecVarAddr.getPointer(CGF),
2374 Bld.CreateSelect(I->getSecond().IsInSPMDModeFlag,
2393 CGBuilderTy &Bld = CGF.Builder;
2396 Size = Bld.CreateNUWAdd(
2400 Size = Bld.CreateUDiv(Size, AlignVal);
2401 Size = Bld.CreateNUWMul(Size, AlignVal);
2410 llvm::Value *GlobalRecCastAddr = Bld.CreatePointerBitCastOrAddrSpaceCast(
2443 CGBuilderTy &Bld = CGF.Builder;
2446 Bld.CreateCondBr(I->getSecond().IsInSPMDModeFlag, ExitBB, NonSPMDBB);
2562 CGBuilderTy &Bld = CGF.Builder;
2565 llvm::Value *ID = Bld.CreateBitOrPointerCast(WFn, CGM.Int8PtrTy);
2598 Address Dst = Bld.CreateConstInBoundsGEP(SharedArgListAddress, Idx);
2601 PtrV = Bld.CreateIntToPtr(V, CGF.VoidPtrTy);
2603 PtrV = Bld.CreatePointerBitCastOrAddrSpaceCast(V, CGF.VoidPtrTy);
2643 CGBuilderTy &Bld = CGF.Builder;
2648 llvm::Value *IsSPMD = Bld.CreateIsNotNull(CGF.EmitNounwindRuntimeCall(
2650 Bld.CreateCondBr(IsSPMD, SeqBB, ParallelCheckBB);
2659 llvm::Value *Res = Bld.CreateIsNotNull(PL);
2660 Bld.CreateCondBr(Res, SeqBB, MasterBB);
2876 CGBuilderTy &Bld = CGF.Builder;
2893 Bld.CreateIntCast(getNVPTXWarpSize(CGF), CGM.Int16Ty, /*isSigned=*/true);
2904 CGBuilderTy &Bld = CGF.Builder;
2919 Address PtrEnd = Bld.CreatePointerBitCastOrAddrSpaceCast(
2920 Bld.CreateConstGEP(SrcAddr, 1), CGF.VoidPtrTy);
2928 Ptr = Bld.CreatePointerBitCastOrAddrSpaceCast(Ptr, IntTy->getPointerTo());
2930 Bld.CreatePointerBitCastOrAddrSpaceCast(ElemPtr, IntTy->getPointerTo());
2935 llvm::BasicBlock *CurrentBB = Bld.GetInsertBlock();
2938 Bld.CreatePHI(Ptr.getType(), /*NumReservedValues=*/2);
2941 Bld.CreatePHI(ElemPtr.getType(), /*NumReservedValues=*/2);
2945 llvm::Value *PtrDiff = Bld.CreatePtrDiff(
2946 PtrEnd.getPointer(), Bld.CreatePointerBitCastOrAddrSpaceCast(
2948 Bld.CreateCondBr(Bld.CreateICmpSGT(PtrDiff, Bld.getInt64(IntSize - 1)),
2960 Address LocalPtr = Bld.CreateConstGEP(Ptr, 1);
2961 Address LocalElemPtr = Bld.CreateConstGEP(ElemPtr, 1);
2976 Ptr = Bld.CreateConstGEP(Ptr, 1);
2977 ElemPtr = Bld.CreateConstGEP(ElemPtr, 1);
3013 CGBuilderTy &Bld = CGF.Builder;
3040 Address SrcElementPtrAddr = Bld.CreateConstArrayGEP(SrcBase, Idx);
3047 DestElementPtrAddr = Bld.CreateConstArrayGEP(DestBase, Idx);
3056 Address SrcElementPtrAddr = Bld.CreateConstArrayGEP(SrcBase, Idx);
3063 DestElementPtrAddr = Bld.CreateConstArrayGEP(DestBase, Idx);
3071 Address SrcElementPtrAddr = Bld.CreateConstArrayGEP(SrcBase, Idx);
3080 Bld.CreateNUWMul(ElementSizeInChars, ScratchpadIndex);
3082 Bld.CreateNUWAdd(DestBase.getPointer(), CurrentOffset);
3084 Bld.CreateIntToPtr(ScratchPadElemAbsolutePtrVal, CGF.VoidPtrTy);
3095 Bld.CreateNUWMul(ElementSizeInChars, ScratchpadIndex);
3097 Bld.CreateNUWAdd(SrcBase.getPointer(), CurrentOffset);
3099 Bld.CreateIntToPtr(ScratchPadElemAbsolutePtrVal, CGF.VoidPtrTy);
3106 DestElementPtrAddr = Bld.CreateConstArrayGEP(DestBase, Idx);
3116 SrcElementAddr = Bld.CreateElementBitCast(
3118 DestElementAddr = Bld.CreateElementBitCast(DestElementAddr,
3163 CGF.EmitStoreOfScalar(Bld.CreatePointerBitCastOrAddrSpaceCast(
3176 ScratchpadBasePtr = Bld.CreateNUWAdd(
3178 Bld.CreateNUWMul(ScratchpadWidth, ElementSizeInChars));
3181 ScratchpadBasePtr = Bld.CreateNUWSub(
3183 ScratchpadBasePtr = Bld.CreateUDiv(
3186 ScratchpadBasePtr = Bld.CreateNUWAdd(
3188 ScratchpadBasePtr = Bld.CreateNUWMul(
3245 CGBuilderTy &Bld = CGF.Builder;
3278 Bld.CreatePointerBitCastOrAddrSpaceCast(
3319 Bld.CreateICmpULT(Cnt, llvm::ConstantInt::get(CGM.IntTy, NumIters));
3320 Bld.CreateCondBr(Cmp, BodyBB, ExitBB);
3332 llvm::Value *IsWarpMaster = Bld.CreateIsNull(LaneID, "warp_master");
3333 Bld.CreateCondBr(IsWarpMaster, ThenBB, ElseBB);
3337 Address ElemPtrPtrAddr = Bld.CreateConstArrayGEP(LocalReduceList, Idx);
3342 ElemPtr = Bld.CreateElementBitCast(ElemPtr, CopyType);
3344 ElemPtr = Address(Bld.CreateGEP(ElemPtr.getPointer(), Cnt),
3350 llvm::Value *MediumPtrVal = Bld.CreateInBoundsGEP(
3355 MediumPtr = Bld.CreateElementBitCast(MediumPtr, CopyType);
3367 Bld.CreateBr(MergeBB);
3370 Bld.CreateBr(MergeBB);
3392 Bld.CreateICmpULT(ThreadID, NumWarpsVal, "is_active_thread");
3393 Bld.CreateCondBr(IsActiveThread, W0ThenBB, W0ElseBB);
3398 llvm::Value *SrcMediumPtrVal = Bld.CreateInBoundsGEP(
3403 SrcMediumPtr = Bld.CreateElementBitCast(SrcMediumPtr, CopyType);
3406 Address TargetElemPtrPtr = Bld.CreateConstArrayGEP(LocalReduceList, Idx);
3410 TargetElemPtr = Bld.CreateElementBitCast(TargetElemPtr, CopyType);
3412 TargetElemPtr = Address(Bld.CreateGEP(TargetElemPtr.getPointer(), Cnt),
3421 Bld.CreateBr(W0MergeBB);
3424 Bld.CreateBr(W0MergeBB);
3429 Cnt = Bld.CreateNSWAdd(Cnt, llvm::ConstantInt::get(CGM.IntTy, /*V=*/1));
3549 CGBuilderTy &Bld = CGF.Builder;
3553 Bld.CreatePointerBitCastOrAddrSpaceCast(
3606 llvm::Value *CondAlgo0 = Bld.CreateIsNull(AlgoVerArgVal);
3608 llvm::Value *Algo1 = Bld.CreateICmpEQ(AlgoVerArgVal, Bld.getInt16(1));
3609 llvm::Value *CondAlgo1 = Bld.CreateAnd(
3610 Algo1, Bld.CreateICmpULT(LaneIDArgVal, RemoteLaneOffsetArgVal));
3612 llvm::Value *Algo2 = Bld.CreateICmpEQ(AlgoVerArgVal, Bld.getInt16(2));
3613 llvm::Value *CondAlgo2 = Bld.CreateAnd(
3614 Algo2, Bld.CreateIsNull(Bld.CreateAnd(LaneIDArgVal, Bld.getInt16(1))));
3615 CondAlgo2 = Bld.CreateAnd(
3616 CondAlgo2, Bld.CreateICmpSGT(RemoteLaneOffsetArgVal, Bld.getInt16(0)));
3618 llvm::Value *CondReduce = Bld.CreateOr(CondAlgo0, CondAlgo1);
3619 CondReduce = Bld.CreateOr(CondReduce, CondAlgo2);
3624 Bld.CreateCondBr(CondReduce, ThenBB, ElseBB);
3628 llvm::Value *LocalReduceListPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
3630 llvm::Value *RemoteReduceListPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
3634 Bld.CreateBr(MergeBB);
3637 Bld.CreateBr(MergeBB);
3643 Algo1 = Bld.CreateICmpEQ(AlgoVerArgVal, Bld.getInt16(1));
3644 llvm::Value *CondCopy = Bld.CreateAnd(
3645 Algo1, Bld.CreateICmpUGE(LaneIDArgVal, RemoteLaneOffsetArgVal));
3650 Bld.CreateCondBr(CondCopy, CpyThenBB, CpyElseBB);
3655 Bld.CreateBr(CpyMergeBB);
3658 Bld.CreateBr(CpyMergeBB);
3704 CGBuilderTy &Bld = CGF.Builder;
3709 Bld.CreatePointerBitCastOrAddrSpaceCast(
3717 llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
3727 Address ElemPtrPtrAddr = Bld.CreateConstArrayGEP(LocalReduceList, Idx);
3731 ElemPtrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
3741 Bld.CreateInBoundsGEP(GlobLVal.getPointer(CGF), Idxs);
3812 CGBuilderTy &Bld = CGF.Builder;
3818 llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
3840 Bld.CreateInBoundsGEP(GlobLVal.getPointer(CGF), Idxs);
3907 CGBuilderTy &Bld = CGF.Builder;
3912 Bld.CreatePointerBitCastOrAddrSpaceCast(
3920 llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
3931 Address ElemPtrPtrAddr = Bld.CreateConstArrayGEP(LocalReduceList, Idx);
3935 ElemPtrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
3945 Bld.CreateInBoundsGEP(GlobLVal.getPointer(CGF), Idxs);
4016 CGBuilderTy &Bld = CGF.Builder;
4022 llvm::Value *BufferArrPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
4044 Bld.CreateInBoundsGEP(GlobLVal.getPointer(CGF), Idxs);
4642 CGBuilderTy &Bld = CGF.Builder;
4668 Address Src = Bld.CreateConstInBoundsGEP(SharedArgListAddress, Idx);
4669 Address TypedAddress = Bld.CreatePointerBitCastOrAddrSpaceCast(
4678 Src = Bld.CreateConstInBoundsGEP(SharedArgListAddress, Idx);
4679 TypedAddress = Bld.CreatePointerBitCastOrAddrSpaceCast(
4693 Address Src = Bld.CreateConstInBoundsGEP(SharedArgListAddress, I + Idx);
4694 Address TypedAddress = Bld.CreatePointerBitCastOrAddrSpaceCast(