Lines Matching refs:Builder
134 IRBuilder<> &Builder) {
142 Value *VecStart = Builder.CreateMul(VecIdx, Stride, "vec.start");
149 VecStart = Builder.CreateGEP(EltType, BasePtr, VecStart, "vec.gep");
155 return Builder.CreatePointerCast(VecStart, VecPtrType, "vec.cast");
290 Value *embedInVector(IRBuilder<> &Builder) const {
292 : concatenateVectors(Builder, Vectors);
330 IRBuilder<> &Builder) const {
333 return Builder.CreateShuffleVector(
421 IRBuilder<> &Builder) {
440 MatrixVal = M.embedInVector(Builder);
449 Value *V = Builder.CreateShuffleVector(
715 IRBuilder<> Builder(Inst);
725 Changed |= VisitLoad(cast<LoadInst>(Inst), Op1, Builder);
727 Changed |= VisitStore(cast<StoreInst>(Inst), Op1, Op2, Builder);
740 Value *createElementPtr(Value *BasePtr, Type *EltType, IRBuilder<> &Builder) {
743 return Builder.CreatePointerCast(BasePtr, EltPtrType);
793 bool IsVolatile, ShapeInfo Shape, IRBuilder<> &Builder) {
795 Value *EltPtr = createElementPtr(Ptr, VType->getElementType(), Builder);
798 Value *GEP = computeVectorAddr(EltPtr, Builder.getInt64(I), Stride,
800 Builder);
801 Value *Vector = Builder.CreateAlignedLoad(
816 IRBuilder<> &Builder) {
818 Value *Offset = Builder.CreateAdd(
819 Builder.CreateMul(J, Builder.getInt64(MatrixShape.getStride())), I);
823 Builder.CreatePointerCast(MatrixPtr, PointerType::get(EltTy, AS));
824 Value *TileStart = Builder.CreateGEP(EltTy, EltPtr, Offset);
829 Builder.CreatePointerCast(TileStart, TilePtrTy, "col.cast");
832 Builder.getInt64(MatrixShape.getStride()), IsVolatile,
833 ResultShape, Builder);
839 IRBuilder<> Builder(Inst);
842 Shape, Builder),
843 Builder);
863 Value *I, Value *J, Type *EltTy, IRBuilder<> &Builder) {
864 Value *Offset = Builder.CreateAdd(
865 Builder.CreateMul(J, Builder.getInt64(MatrixShape.getStride())), I);
869 Builder.CreatePointerCast(MatrixPtr, PointerType::get(EltTy, AS));
870 Value *TileStart = Builder.CreateGEP(EltTy, EltPtr, Offset);
875 Builder.CreatePointerCast(TileStart, TilePtrTy, "col.cast");
878 Builder.getInt64(MatrixShape.getStride()), IsVolatile, Builder);
885 IRBuilder<> &Builder) {
887 Value *EltPtr = createElementPtr(Ptr, VType->getElementType(), Builder);
889 Value *GEP = computeVectorAddr(EltPtr, Builder.getInt64(Vec.index()),
891 VType->getElementType(), Builder);
892 Builder.CreateAlignedStore(Vec.value(), GEP,
905 IRBuilder<> Builder(Inst);
906 auto StoreVal = getMatrix(Matrix, Shape, Builder);
909 IsVolatile, Builder),
910 Builder);
929 IRBuilder<> &Builder) {
938 Block = Builder.CreateShuffleVector(
957 return Builder.CreateShuffleVector(Col, Block, Mask);
961 IRBuilder<> &Builder, bool AllowContraction,
965 return UseFPOp ? Builder.CreateFMul(A, B) : Builder.CreateMul(A, B);
973 return Builder.CreateCall(FMulAdd, {A, B, Sum});
976 Value *Mul = Builder.CreateFMul(A, B);
977 return Builder.CreateFAdd(Sum, Mul);
981 Value *Mul = Builder.CreateMul(A, B);
982 return Builder.CreateAdd(Sum, Mul);
991 IRBuilder<> &Builder) {
1000 Flattened = Matrix.embedInVector(Builder);
1010 IRBuilder<> &Builder, bool isTiled) {
1038 Value *Sum = isTiled ? Result.extractVector(I, J, BlockSize, Builder)
1041 Value *L = A.extractVector(I, K, BlockSize, Builder);
1042 Value *RH = Builder.CreateExtractElement(B.getColumn(J), K);
1043 Value *Splat = Builder.CreateVectorSplat(BlockSize, RH, "splat");
1046 Builder, AllowContraction, NumComputeOps);
1049 insertVector(Result.getVector(J), I, Sum, Builder));
1066 Value *R = B.extractVector(K, J, BlockSize, Builder);
1067 Value *LH = Builder.CreateExtractElement(A.getVector(I), K);
1068 Value *Splat = Builder.CreateVectorSplat(BlockSize, LH, "splat");
1070 IsFP, Builder, AllowContraction, NumComputeOps);
1073 insertVector(Result.getVector(I), J, Sum, Builder));
1116 IRBuilder<> Builder(MatMul);
1118 Builder.SetInsertPoint(Check0);
1119 Type *IntPtrTy = Builder.getIntPtrTy(Load->getModule()->getDataLayout());
1120 Value *StoreBegin = Builder.CreatePtrToInt(
1122 Value *StoreEnd = Builder.CreateAdd(
1125 Value *LoadBegin = Builder.CreatePtrToInt(const_cast<Value *>(LoadLoc.Ptr),
1127 Builder.CreateCondBr(Builder.CreateICmpULT(LoadBegin, StoreEnd), Check1,
1134 Builder.SetInsertPoint(Check1, Check1->begin());
1135 Value *LoadEnd = Builder.CreateAdd(
1138 Builder.CreateCondBr(Builder.CreateICmpULT(StoreBegin, LoadEnd), Copy,
1142 Builder.SetInsertPoint(Copy, Copy->begin());
1144 Builder.CreateAlloca(Load->getType(), Load->getPointerAddressSpace());
1145 Builder.CreateMemCpy(NewLd, NewLd->getAlign(),
1148 Builder.SetInsertPoint(Fusion, Fusion->begin());
1149 PHINode *PHI = Builder.CreatePHI(Load->getPointerOperandType(), 3);
1227 IRBuilder<> Builder(Store);
1238 LShape, Builder.getInt64(I), Builder.getInt64(K),
1239 {TileR, TileM}, EltType, Builder);
1242 RShape, Builder.getInt64(K), Builder.getInt64(J),
1243 {TileM, TileC}, EltType, Builder);
1244 emitMatrixMultiply(Res, A, B, AllowContract, Builder, true);
1247 Builder.getInt64(I), Builder.getInt64(J), EltType, Builder);
1296 IRBuilder<> Builder(MatMul);
1301 const MatrixTy &Lhs = getMatrix(MatMul->getArgOperand(0), LShape, Builder);
1302 const MatrixTy &Rhs = getMatrix(MatMul->getArgOperand(1), RShape, Builder);
1313 emitMatrixMultiply(Result, Lhs, Rhs, AllowContract, Builder, false);
1314 finalizeLowering(MatMul, Result, Builder);
1320 IRBuilder<> Builder(Inst);
1324 MatrixTy InputMatrix = getMatrix(InputVal, ArgShape, Builder);
1337 Value *Elt = Builder.CreateExtractElement(J.value(), I);
1340 Builder.CreateInsertElement(ResultVector, Elt, J.index());
1351 Builder);
1355 bool VisitLoad(LoadInst *Inst, Value *Ptr, IRBuilder<> &Builder) {
1361 Builder.getInt64(I->second.getStride()), Inst->isVolatile(),
1367 IRBuilder<> &Builder) {
1373 Builder.getInt64(I->second.getStride()), Inst->isVolatile(),
1387 IRBuilder<> Builder(Inst);
1391 MatrixTy A = getMatrix(Lhs, Shape, Builder);
1392 MatrixTy B = getMatrix(Rhs, Shape, Builder);
1398 auto BuildVectorOp = [&Builder, Inst](Value *LHS, Value *RHS) {
1401 return Builder.CreateAdd(LHS, RHS);
1403 return Builder.CreateMul(LHS, RHS);
1405 return Builder.CreateSub(LHS, RHS);
1407 return Builder.CreateFAdd(LHS, RHS);
1409 return Builder.CreateFMul(LHS, RHS);
1411 return Builder.CreateFSub(LHS, RHS);
1423 Builder);