Lines Matching refs:Builder
901 static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder,
907 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
908 Op = Builder.CreateBitCast(Op, VecTy, "cast");
926 Res = Builder.CreateShuffleVector(Res, Op, makeArrayRef(Idxs, NumElts));
930 return Builder.CreateBitCast(Res, ResultTy, "cast");
935 static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
941 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
942 Op = Builder.CreateBitCast(Op, VecTy, "cast");
960 Res = Builder.CreateShuffleVector(Op, Res, makeArrayRef(Idxs, NumElts));
964 return Builder.CreateBitCast(Res, ResultTy, "cast");
967 static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
970 Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());
971 Mask = Builder.CreateBitCast(Mask, MaskTy);
979 Mask = Builder.CreateShuffleVector(Mask, Mask,
987 static Value *EmitX86Select(IRBuilder<> &Builder, Value *Mask,
994 Mask = getX86MaskVec(Builder, Mask,
996 return Builder.CreateSelect(Mask, Op0, Op1);
999 static Value *EmitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask,
1006 auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(),
1008 Mask = Builder.CreateBitCast(Mask, MaskTy);
1009 Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
1010 return Builder.CreateSelect(Mask, Op0, Op1);
1016 static Value *UpgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
1055 Value *Align = Builder.CreateShuffleVector(Op1, Op0,
1059 return EmitX86Select(Builder, Mask, Align, Passthru);
1062 static Value *UpgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallInst &CI,
1115 Value *V = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID),
1118 : Builder.CreateBitCast(CI.getArgOperand(1),
1120 return EmitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
1123 static Value *UpgradeX86AddSubSatIntrinsics(IRBuilder<> &Builder, CallInst &CI,
1133 Value *Res = Builder.CreateCall(Intrin, {Op0, Op1});
1138 Res = EmitX86Select(Builder, Mask, Res, VecSrc);
1143 static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallInst &CI,
1154 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
1155 Amt = Builder.CreateVectorSplat(NumElts, Amt);
1160 Value *Res = Builder.CreateCall(Intrin, {Src, Src, Amt});
1165 Res = EmitX86Select(Builder, Mask, Res, VecSrc);
1170 static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallInst &CI, unsigned Imm,
1204 Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);
1205 Value *Ext = Builder.CreateSExt(Cmp, Ty);
1209 static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallInst &CI,
1224 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
1225 Amt = Builder.CreateVectorSplat(NumElts, Amt);
1230 Value *Res = Builder.CreateCall(Intrin, {Op0, Op1, Amt});
1238 Res = EmitX86Select(Builder, Mask, Res, VecSrc);
1243 static Value *UpgradeMaskedStore(IRBuilder<> &Builder,
1247 Ptr = Builder.CreateBitCast(Ptr,
1257 return Builder.CreateAlignedStore(Data, Ptr, Alignment);
1261 Mask = getX86MaskVec(Builder, Mask, NumElts);
1262 return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask);
1265 static Value *UpgradeMaskedLoad(IRBuilder<> &Builder,
1270 Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(ValTy));
1280 return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
1284 Mask = getX86MaskVec(Builder, Mask, NumElts);
1285 return Builder.CreateMaskedLoad(Ptr, Alignment, Mask, Passthru);
1288 static Value *upgradeAbs(IRBuilder<> &Builder, CallInst &CI) {
1292 Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_SGT, Op0, Zero);
1293 Value *Neg = Builder.CreateNeg(Op0);
1294 Value *Res = Builder.CreateSelect(Cmp, Op0, Neg);
1297 Res = EmitX86Select(Builder,CI.getArgOperand(2), Res, CI.getArgOperand(1));
1302 static Value *upgradeIntMinMax(IRBuilder<> &Builder, CallInst &CI,
1306 Value *Cmp = Builder.CreateICmp(Pred, Op0, Op1);
1307 Value *Res = Builder.CreateSelect(Cmp, Op0, Op1);
1310 Res = EmitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
1315 static Value *upgradePMULDQ(IRBuilder<> &Builder, CallInst &CI, bool IsSigned) {
1319 Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty);
1320 Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty);
1325 LHS = Builder.CreateShl(LHS, ShiftAmt);
1326 LHS = Builder.CreateAShr(LHS, ShiftAmt);
1327 RHS = Builder.CreateShl(RHS, ShiftAmt);
1328 RHS = Builder.CreateAShr(RHS, ShiftAmt);
1332 LHS = Builder.CreateAnd(LHS, Mask);
1333 RHS = Builder.CreateAnd(RHS, Mask);
1336 Value *Res = Builder.CreateMul(LHS, RHS);
1339 Res = EmitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
1345 static Value *ApplyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
1351 Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));
1360 Vec = Builder.CreateShuffleVector(Vec,
1364 return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U)));
1367 static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallInst &CI,
1375 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
1378 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
1390 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
1395 return ApplyX86MaskOn1BitsVec(Builder, Cmp, Mask);
1399 static Value *UpgradeX86MaskedShift(IRBuilder<> &Builder, CallInst &CI,
1402 Value *Rep = Builder.CreateCall(Intrin,
1404 return EmitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
1407 static Value* upgradeMaskedMove(IRBuilder<> &Builder, CallInst &CI) {
1413 Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));
1414 Value* Cmp = Builder.CreateIsNotNull(AndNode);
1415 Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);
1416 Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);
1417 Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);
1418 return Builder.CreateInsertElement(A, Select, (uint64_t)0);
1422 static Value* UpgradeMaskToInt(IRBuilder<> &Builder, CallInst &CI) {
1426 Value *Mask = getX86MaskVec(Builder, Op, NumElts);
1427 return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
1431 static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
1662 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID),
1665 Rep = EmitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,
1687 IRBuilder<> Builder(C);
1688 Builder.SetInsertPoint(CI->getParent(), CI->getIterator());
1720 Value *Addr = Builder.CreateBitCast(Arg0, EltPtrTy, "cast");
1722 Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");
1724 StoreInst *SI = Builder.CreateAlignedStore(Extract, Addr, Align(1));
1744 Value *BC = Builder.CreateBitCast(Arg0,
1747 StoreInst *SI = Builder.CreateAlignedStore(
1762 Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
1763 Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);
1764 Value *BC = Builder.CreateBitCast(Arg0,
1767 Builder.CreateAlignedStore(Elt, BC, Align(1));
1780 Arg0 = Builder.CreateBitCast(Arg0,
1783 Builder.CreateAlignedStore(Arg1, Arg0, Align(1));
1791 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
1792 UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
1803 UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
1817 Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
1819 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
1826 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
1827 Rep = Builder.CreateVectorSplat(NumElts, Rep);
1831 Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);
1834 Elt0 = Builder.CreateCall(Intr, Elt0);
1835 Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);
1839 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
1851 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
1854 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
1859 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
1866 Rep = Builder.CreateAnd(Op0, Op1);
1871 Rep = Builder.CreateICmp(Pred, Rep, Zero);
1872 Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, Mask);
1876 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
1877 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
1881 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);
1882 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);
1889 LHS = Builder.CreateShuffleVector(LHS, LHS,
1891 RHS = Builder.CreateShuffleVector(RHS, RHS,
1895 Rep = Builder.CreateShuffleVector(RHS, LHS,
1897 Rep = Builder.CreateBitCast(Rep, CI->getType());
1899 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
1900 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
1901 Rep = Builder.CreateAnd(LHS, RHS);
1902 Rep = Builder.CreateBitCast(Rep, CI->getType());
1904 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
1905 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
1906 LHS = Builder.CreateNot(LHS);
1907 Rep = Builder.CreateAnd(LHS, RHS);
1908 Rep = Builder.CreateBitCast(Rep, CI->getType());
1910 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
1911 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
1912 Rep = Builder.CreateOr(LHS, RHS);
1913 Rep = Builder.CreateBitCast(Rep, CI->getType());
1915 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
1916 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
1917 Rep = Builder.CreateXor(LHS, RHS);
1918 Rep = Builder.CreateBitCast(Rep, CI->getType());
1920 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
1921 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
1922 LHS = Builder.CreateNot(LHS);
1923 Rep = Builder.CreateXor(LHS, RHS);
1924 Rep = Builder.CreateBitCast(Rep, CI->getType());
1926 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
1927 Rep = Builder.CreateNot(Rep);
1928 Rep = Builder.CreateBitCast(Rep, CI->getType());
1931 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
1932 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
1933 Rep = Builder.CreateOr(LHS, RHS);
1934 Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty());
1937 C = ConstantInt::getAllOnesValue(Builder.getInt16Ty());
1939 C = ConstantInt::getNullValue(Builder.getInt16Ty());
1940 Rep = Builder.CreateICmpEQ(Rep, C);
1941 Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());
1947 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
1949 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
1953 EltOp = Builder.CreateFAdd(Elt0, Elt1);
1955 EltOp = Builder.CreateFSub(Elt0, Elt1);
1957 EltOp = Builder.CreateFMul(Elt0, Elt1);
1959 EltOp = Builder.CreateFDiv(Elt0, Elt1);
1960 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
1965 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
1977 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
1979 Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
2000 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2002 Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
2030 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2032 Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(3));
2037 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
2040 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
2047 Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);
2048 Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, nullptr);
2054 Rep = upgradeAbs(Builder, *CI);
2060 Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SGT);
2066 Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_UGT);
2072 Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SLT);
2078 Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_ULT);
2083 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/false);
2088 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/true);
2093 Rep = Builder.CreateSIToFP(
2096 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
2098 Rep = Builder.CreateUIToFP(
2101 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
2103 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
2104 Rep = Builder.CreateFPExt(
2106 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
2132 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1});
2138 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
2146 Rep = Builder.CreateCall(F, { Rep, CI->getArgOperand(3) });
2148 Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
2149 : Builder.CreateSIToFP(Rep, DstTy, "cvt");
2153 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2163 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1, 2, 3});
2165 Rep = Builder.CreateBitCast(
2167 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps");
2169 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2172 Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),
2176 Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),
2184 Value *Ptr = Builder.CreateBitCast(CI->getOperand(0),
2187 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
2193 Rep = Builder.CreateCall(ELd, { Ptr, MaskVec, CI->getOperand(1) });
2199 Value *Ptr = Builder.CreateBitCast(CI->getOperand(0),
2202 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
2208 Rep = Builder.CreateCall(CSt, { CI->getArgOperand(1), Ptr, MaskVec });
2213 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
2220 Rep = Builder.CreateCall(Intr, { CI->getOperand(0), CI->getOperand(1),
2258 Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
2261 Value *NotSel = Builder.CreateNot(Sel);
2262 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
2263 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
2264 Rep = Builder.CreateOr(Sel0, Sel1);
2268 Rep = upgradeX86Rotate(Builder, *CI, false);
2271 Rep = upgradeX86Rotate(Builder, *CI, true);
2276 Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
2281 Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
2285 Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
2286 Rep = Builder.CreateCall(CRC32, {Trunc0, CI->getArgOperand(1)});
2287 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
2294 Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0),
2296 Value *Load = Builder.CreateLoad(EltTy, Cast);
2300 Rep = Builder.CreateInsertElement(Rep, Load,
2317 Value *SV = Builder.CreateShuffleVector(
2321 Rep = DoSext ? Builder.CreateSExt(SV, DstTy)
2322 : Builder.CreateZExt(SV, DstTy);
2325 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2332 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
2333 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2341 Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0),
2343 Value *Load = Builder.CreateAlignedLoad(VT, Op, Align(1));
2345 Rep = Builder.CreateShuffleVector(
2349 Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
2369 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
2371 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
2383 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
2386 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2396 Rep = Builder.CreateShuffleVector(Op, UndefValue::get(Op->getType()),
2400 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2411 Rep = UpgradeX86AddSubSatIntrinsics(Builder, *CI, true, IsAdd);
2419 Rep = UpgradeX86AddSubSatIntrinsics(Builder, *CI, false, IsAdd);
2421 Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
2428 Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
2438 Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
2444 Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
2451 Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
2457 Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
2473 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
2494 Rep = Builder.CreateShuffleVector(Op1, UndefV, Idxs);
2513 Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);
2517 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
2536 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2540 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2555 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2558 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2594 Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
2615 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2618 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2634 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2637 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2653 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2656 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2679 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
2681 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
2701 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2703 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2717 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
2719 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2733 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
2735 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2741 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
2742 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
2743 Rep = Builder.CreateBitCast(Rep, FTy);
2744 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2750 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
2751 Rep = Builder.CreateAnd(Rep,
2752 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
2753 Rep = Builder.CreateBitCast(Rep, FTy);
2754 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2760 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
2761 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
2762 Rep = Builder.CreateBitCast(Rep, FTy);
2763 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2769 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
2770 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
2771 Rep = Builder.CreateBitCast(Rep, FTy);
2772 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2775 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
2776 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2779 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
2780 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2783 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
2784 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2794 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2798 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
2800 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2810 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2814 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
2816 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2826 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2830 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
2832 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2842 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2846 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
2848 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2861 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2864 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2867 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
2870 { CI->getArgOperand(0), Builder.getInt1(false) });
2871 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2940 Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
3008 Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
3074 Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
3076 Rep = upgradeMaskedMove(Builder, *CI);
3078 Rep = UpgradeMaskToInt(Builder, *CI);
3087 Value *BC = Builder.CreateBitCast(
3089 LoadInst *LI = Builder.CreateAlignedLoad(
3106 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
3107 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
3108 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
3112 Ops[0] = Builder.CreateFNeg(Ops[0]);
3114 Ops[1] = Builder.CreateFNeg(Ops[1]);
3116 Ops[2] = Builder.CreateFNeg(Ops[2]);
3118 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
3124 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep,
3130 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
3131 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
3132 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
3134 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
3139 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
3158 A = Builder.CreateFNeg(A);
3160 B = Builder.CreateFNeg(B);
3162 C = Builder.CreateFNeg(C);
3164 A = Builder.CreateExtractElement(A, (uint64_t)0);
3165 B = Builder.CreateExtractElement(B, (uint64_t)0);
3166 C = Builder.CreateExtractElement(C, (uint64_t)0);
3178 Rep = Builder.CreateCall(FMA, Ops);
3183 Rep = Builder.CreateCall(FMA, { A, B, C });
3192 PassThru = Builder.CreateExtractElement(CI->getArgOperand(2),
3195 Rep = EmitX86ScalarSelect(Builder, CI->getArgOperand(3),
3197 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0),
3218 A = Builder.CreateFNeg(A);
3220 B = Builder.CreateFNeg(B);
3222 C = Builder.CreateFNeg(C);
3234 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3240 Rep = Builder.CreateCall(FMA, { A, B, C });
3247 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3265 Ops[2] = Builder.CreateFNeg(Ops[2]);
3266 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3288 Ops[2] = Builder.CreateFNeg(Ops[2]);
3290 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3300 Value *Odd = Builder.CreateCall(FMA, Ops);
3301 Ops[2] = Builder.CreateFNeg(Ops[2]);
3302 Value *Even = Builder.CreateCall(FMA, Ops);
3311 Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);
3318 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3342 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3346 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
3370 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3374 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3380 Rep = UpgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
3406 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3410 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3436 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3440 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3459 Value *NewCall = Builder.CreateCall(
3464 Value *Data = Builder.CreateExtractValue(NewCall, 1);
3466 Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(3),
3468 Builder.CreateAlignedStore(Data, Ptr, Align(1));
3470 Value *CF = Builder.CreateExtractValue(NewCall, 0);
3475 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
3479 Value *Neg = Builder.CreateNeg(Arg, "neg");
3480 Value *Cmp = Builder.CreateICmpSGE(
3482 Rep = Builder.CreateSelect(Cmp, Arg, Neg, "abs");
3487 Rep = Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, Ptr, Val,
3494 ? Builder.CreateICmpUGE(Arg0, Arg1, "max.cond")
3495 : Builder.CreateICmpSGE(Arg0, Arg1, "max.cond");
3496 Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "max");
3502 ? Builder.CreateICmpULE(Arg0, Arg1, "min.cond")
3503 : Builder.CreateICmpSLE(Arg0, Arg1, "min.cond");
3504 Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "min");
3508 Value *Ctlz = Builder.CreateCall(
3511 {Arg, Builder.getFalse()}, "ctlz");
3512 Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
3517 Value *Popc = Builder.CreateCall(
3521 Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
3523 Rep = Builder.CreateCall(Intrinsic::getDeclaration(
3525 {Builder.getFloatTy()}),
3557 NewCall = Builder.CreateCall(NewFn, Args);
3568 NewCall = Builder.CreateCall(NewFn, Args);
3588 NewCall = Builder.CreateCall(NewFn, Args);
3593 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
3601 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});
3606 ? Builder.getFalse()
3609 CI->getNumArgOperands() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
3610 NewCall = Builder.CreateCall(
3616 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
3620 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
3629 NewCall = Builder.CreateCall(
3639 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});
3650 Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);
3651 NewCall = Builder.CreateCall(NewFn, Args);
3671 Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");
3672 Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
3674 NewCall = Builder.CreateCall(NewFn, {BC0, BC1});
3684 NewCall = Builder.CreateCall(NewFn);
3686 Value *Data = Builder.CreateExtractValue(NewCall, 1);
3688 Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(0),
3690 Builder.CreateAlignedStore(Data, Ptr, Align(1));
3692 Value *TSC = Builder.CreateExtractValue(NewCall, 0);
3716 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
3717 NewCall = Builder.CreateCall(NewFn, Args);
3722 NewCall = Builder.CreateCall(NewFn, {});
3734 NewCall = Builder.CreateCall(NewFn, Args);
3756 NewCall = Builder.CreateCall(NewFn, Args);
3929 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
3955 Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I));
3964 CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);
3969 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());