Lines Matching refs:Intrinsic
40 static bool UpgradePTESTIntrinsic(Function* F, Intrinsic::ID IID,
50 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
56 static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
66 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
416 NewFn = Intrinsic::getDeclaration(F->getParent(),
417 Intrinsic::x86_rdtscp);
424 return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestc, NewFn);
426 return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestz, NewFn);
428 return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestnzc, NewFn);
433 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_insertps,
436 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dppd,
439 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dpps,
442 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_mpsadbw,
445 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256,
448 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw,
454 NewFn = Intrinsic::getDeclaration(F->getParent(),
455 Intrinsic::x86_xop_vfrcz_ss);
460 NewFn = Intrinsic::getDeclaration(F->getParent(),
461 Intrinsic::x86_xop_vfrcz_sd);
471 Intrinsic::ID Permil2ID;
473 Permil2ID = Intrinsic::x86_xop_vpermil2pd;
475 Permil2ID = Intrinsic::x86_xop_vpermil2ps;
477 Permil2ID = Intrinsic::x86_xop_vpermil2pd_256;
479 Permil2ID = Intrinsic::x86_xop_vpermil2ps_256;
480 NewFn = Intrinsic::getDeclaration(F->getParent(), Permil2ID);
486 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::eh_recoverfp);
506 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse,
515 // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to
524 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
532 // Can't use Intrinsic::getDeclaration here as the return types might
541 static const Intrinsic::ID StoreInts[] = {Intrinsic::arm_neon_vst1,
542 Intrinsic::arm_neon_vst2,
543 Intrinsic::arm_neon_vst3,
544 Intrinsic::arm_neon_vst4};
546 static const Intrinsic::ID StoreLaneInts[] = {
547 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
548 Intrinsic::arm_neon_vst4lane
554 NewFn = Intrinsic::getDeclaration(F->getParent(),
557 NewFn = Intrinsic::getDeclaration(F->getParent(),
562 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer);
566 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::sadd_sat,
571 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::uadd_sat,
576 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ssub_sat,
581 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::usub_sat,
590 NewFn = Intrinsic::getDeclaration(F->getParent(),
591 Intrinsic::aarch64_neon_faddp, Ty);
601 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
607 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz,
616 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value);
625 Intrinsic::ID ID = Intrinsic::not_intrinsic;
627 ID = Intrinsic::experimental_vector_reduce_v2_fadd;
629 ID = Intrinsic::experimental_vector_reduce_v2_fmul;
631 if (ID != Intrinsic::not_intrinsic) {
635 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
645 Intrinsic::ID ID = IsLifetimeStart ?
646 Intrinsic::lifetime_start : Intrinsic::invariant_start;
649 if (F->getName() != Intrinsic::getName(ID, ObjectPtr)) {
651 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr);
658 Intrinsic::ID ID = IsLifetimeEnd ?
659 Intrinsic::lifetime_end : Intrinsic::invariant_end;
663 if (F->getName() != Intrinsic::getName(ID, ObjectPtr)) {
665 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr);
674 NewFn = Intrinsic::getDeclaration(F->getParent(),
675 Intrinsic::launder_invariant_group, ObjectPtr);
685 if (F->getName() != Intrinsic::getName(Intrinsic::masked_load, Tys)) {
687 NewFn = Intrinsic::getDeclaration(F->getParent(),
688 Intrinsic::masked_load,
696 if (F->getName() != Intrinsic::getName(Intrinsic::masked_store, Tys)) {
698 NewFn = Intrinsic::getDeclaration(F->getParent(),
699 Intrinsic::masked_store,
708 if (F->getName() != Intrinsic::getName(Intrinsic::masked_gather, Tys)) {
710 NewFn = Intrinsic::getDeclaration(F->getParent(),
711 Intrinsic::masked_gather, Tys);
718 if (F->getName() != Intrinsic::getName(Intrinsic::masked_scatter, Tys)) {
720 NewFn = Intrinsic::getDeclaration(F->getParent(),
721 Intrinsic::masked_scatter, Tys);
732 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memcpy,
740 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memmove,
752 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memset,
763 Intrinsic::ID IID = StringSwitch<Intrinsic::ID>(Name)
764 .Cases("brev32", "brev64", Intrinsic::bitreverse)
765 .Case("clz.i", Intrinsic::ctlz)
766 .Case("popc.i", Intrinsic::ctpop)
767 .Default(Intrinsic::not_intrinsic);
768 if (IID != Intrinsic::not_intrinsic && F->arg_size() == 1) {
769 NewFn = Intrinsic::getDeclaration(F->getParent(), IID,
799 F->getName() != Intrinsic::getName(Intrinsic::objectsize, Tys)) {
801 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::objectsize,
812 if (F->getName() != Intrinsic::getName(Intrinsic::prefetch, Tys)) {
815 Intrinsic::getDeclaration(F->getParent(), Intrinsic::prefetch, Tys);
833 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
849 assert(F != NewFn && "Intrinsic function upgraded to the same function");
854 if (Intrinsic::ID id = F->getIntrinsicID())
855 F->setAttributes(Intrinsic::getAttributes(F->getContext(), id));
1059 Intrinsic::ID IID;
1061 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
1063 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
1065 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
1067 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
1069 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
1071 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
1073 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
1075 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
1077 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
1079 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
1081 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
1083 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
1085 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
1087 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
1089 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
1091 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
1093 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
1095 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
1106 Value *V = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID),
1120 Intrinsic::ID IID =
1121 IsSigned ? (IsAddition ? Intrinsic::sadd_sat : Intrinsic::ssub_sat)
1122 : (IsAddition ? Intrinsic::uadd_sat : Intrinsic::usub_sat);
1123 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
1149 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
1150 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
1219 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
1220 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
1384 Intrinsic::ID IID) {
1385 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID);
1421 Intrinsic::ID IID;
1424 IID = Intrinsic::x86_sse_max_ps;
1426 IID = Intrinsic::x86_sse2_max_pd;
1428 IID = Intrinsic::x86_avx_max_ps_256;
1430 IID = Intrinsic::x86_avx_max_pd_256;
1435 IID = Intrinsic::x86_sse_min_ps;
1437 IID = Intrinsic::x86_sse2_min_pd;
1439 IID = Intrinsic::x86_avx_min_ps_256;
1441 IID = Intrinsic::x86_avx_min_pd_256;
1446 IID = Intrinsic::x86_ssse3_pshuf_b_128;
1448 IID = Intrinsic::x86_avx2_pshuf_b;
1450 IID = Intrinsic::x86_avx512_pshuf_b_512;
1455 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
1457 IID = Intrinsic::x86_avx2_pmul_hr_sw;
1459 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
1464 IID = Intrinsic::x86_sse2_pmulh_w;
1466 IID = Intrinsic::x86_avx2_pmulh_w;
1468 IID = Intrinsic::x86_avx512_pmulh_w_512;
1473 IID = Intrinsic::x86_sse2_pmulhu_w;
1475 IID = Intrinsic::x86_avx2_pmulhu_w;
1477 IID = Intrinsic::x86_avx512_pmulhu_w_512;
1482 IID = Intrinsic::x86_sse2_pmadd_wd;
1484 IID = Intrinsic::x86_avx2_pmadd_wd;
1486 IID = Intrinsic::x86_avx512_pmaddw_d_512;
1491 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
1493 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
1495 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
1500 IID = Intrinsic::x86_sse2_packsswb_128;
1502 IID = Intrinsic::x86_avx2_packsswb;
1504 IID = Intrinsic::x86_avx512_packsswb_512;
1509 IID = Intrinsic::x86_sse2_packssdw_128;
1511 IID = Intrinsic::x86_avx2_packssdw;
1513 IID = Intrinsic::x86_avx512_packssdw_512;
1518 IID = Intrinsic::x86_sse2_packuswb_128;
1520 IID = Intrinsic::x86_avx2_packuswb;
1522 IID = Intrinsic::x86_avx512_packuswb_512;
1527 IID = Intrinsic::x86_sse41_packusdw;
1529 IID = Intrinsic::x86_avx2_packusdw;
1531 IID = Intrinsic::x86_avx512_packusdw_512;
1536 IID = Intrinsic::x86_avx_vpermilvar_ps;
1538 IID = Intrinsic::x86_avx_vpermilvar_pd;
1540 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
1542 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
1544 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
1546 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
1550 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
1552 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
1554 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
1556 IID = Intrinsic::x86_sse2_cvttps2dq;
1558 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
1562 IID = Intrinsic::x86_avx2_permps;
1564 IID = Intrinsic::x86_avx2_permd;
1566 IID = Intrinsic::x86_avx512_permvar_df_256;
1568 IID = Intrinsic::x86_avx512_permvar_di_256;
1570 IID = Intrinsic::x86_avx512_permvar_sf_512;
1572 IID = Intrinsic::x86_avx512_permvar_si_512;
1574 IID = Intrinsic::x86_avx512_permvar_df_512;
1576 IID = Intrinsic::x86_avx512_permvar_di_512;
1578 IID = Intrinsic::x86_avx512_permvar_hi_128;
1580 IID = Intrinsic::x86_avx512_permvar_hi_256;
1582 IID = Intrinsic::x86_avx512_permvar_hi_512;
1584 IID = Intrinsic::x86_avx512_permvar_qi_128;
1586 IID = Intrinsic::x86_avx512_permvar_qi_256;
1588 IID = Intrinsic::x86_avx512_permvar_qi_512;
1593 IID = Intrinsic::x86_avx512_dbpsadbw_128;
1595 IID = Intrinsic::x86_avx512_dbpsadbw_256;
1597 IID = Intrinsic::x86_avx512_dbpsadbw_512;
1602 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
1604 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
1606 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
1611 IID = Intrinsic::x86_avx512_conflict_d_128;
1613 IID = Intrinsic::x86_avx512_conflict_d_256;
1615 IID = Intrinsic::x86_avx512_conflict_d_512;
1617 IID = Intrinsic::x86_avx512_conflict_q_128;
1619 IID = Intrinsic::x86_avx512_conflict_q_256;
1621 IID = Intrinsic::x86_avx512_conflict_q_512;
1626 IID = Intrinsic::x86_sse2_pavg_b;
1628 IID = Intrinsic::x86_avx2_pavg_b;
1630 IID = Intrinsic::x86_avx512_pavg_b_512;
1632 IID = Intrinsic::x86_sse2_pavg_w;
1634 IID = Intrinsic::x86_avx2_pavg_w;
1636 IID = Intrinsic::x86_avx512_pavg_w_512;
1646 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID),
1674 assert(F && "Intrinsic call is not direct?");
1680 assert(Name.startswith("llvm.") && "Intrinsic doesn't start with 'llvm.'");
1816 Function *Intr = Intrinsic::getDeclaration(F->getParent(),
1817 Intrinsic::sqrt, Elt0->getType());
1823 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
1824 Intrinsic::sqrt,
1831 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
1832 : Intrinsic::x86_avx512_sqrt_pd_512;
1835 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
1838 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
1839 Intrinsic::sqrt,
1953 Intrinsic::ID IID;
1956 case 128: IID = Intrinsic::x86_avx512_vpshufbitqmb_128; break;
1957 case 256: IID = Intrinsic::x86_avx512_vpshufbitqmb_256; break;
1958 case 512: IID = Intrinsic::x86_avx512_vpshufbitqmb_512; break;
1961 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
1968 Intrinsic::ID IID;
1970 IID = Intrinsic::x86_avx512_fpclass_ps_128;
1972 IID = Intrinsic::x86_avx512_fpclass_ps_256;
1974 IID = Intrinsic::x86_avx512_fpclass_ps_512;
1976 IID = Intrinsic::x86_avx512_fpclass_pd_128;
1978 IID = Intrinsic::x86_avx512_fpclass_pd_256;
1980 IID = Intrinsic::x86_avx512_fpclass_pd_512;
1984 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
1991 Intrinsic::ID IID;
1993 IID = Intrinsic::x86_avx512_cmp_ps_128;
1995 IID = Intrinsic::x86_avx512_cmp_ps_256;
1997 IID = Intrinsic::x86_avx512_cmp_ps_512;
1999 IID = Intrinsic::x86_avx512_cmp_pd_128;
2001 IID = Intrinsic::x86_avx512_cmp_pd_256;
2003 IID = Intrinsic::x86_avx512_cmp_pd_512;
2014 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2124 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
2125 : Intrinsic::x86_avx512_sitofp_round;
2126 Function *F = Intrinsic::getDeclaration(CI->getModule(), IID,
2156 Function *ELd = Intrinsic::getDeclaration(F->getParent(),
2157 Intrinsic::masked_expandload,
2171 Function *CSt = Intrinsic::getDeclaration(F->getParent(),
2172 Intrinsic::masked_compressstore,
2183 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
2184 : Intrinsic::x86_avx512_mask_expand;
2185 Function *Intr = Intrinsic::getDeclaration(F->getParent(), IID, ResultTy);
2249 Function *CRC32 = Intrinsic::getDeclaration(F->getParent(),
2250 Intrinsic::x86_sse42_crc32_32_8);
2753 Intrinsic::ID IID;
2755 IID = Intrinsic::x86_avx512_add_ps_512;
2757 IID = Intrinsic::x86_avx512_add_pd_512;
2759 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2769 Intrinsic::ID IID;
2771 IID = Intrinsic::x86_avx512_div_ps_512;
2773 IID = Intrinsic::x86_avx512_div_pd_512;
2775 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2785 Intrinsic::ID IID;
2787 IID = Intrinsic::x86_avx512_mul_ps_512;
2789 IID = Intrinsic::x86_avx512_mul_pd_512;
2791 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2801 Intrinsic::ID IID;
2803 IID = Intrinsic::x86_avx512_sub_ps_512;
2805 IID = Intrinsic::x86_avx512_sub_pd_512;
2807 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2820 static const Intrinsic::ID MinMaxTbl[2][2] = {
2821 { Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512 },
2822 { Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512 }
2824 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
2826 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2832 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
2833 Intrinsic::ctlz,
2847 Intrinsic::ID IID;
2850 IID = Intrinsic::x86_avx2_psllv_q;
2852 IID = Intrinsic::x86_avx2_psllv_q_256;
2854 IID = Intrinsic::x86_avx2_psllv_d;
2856 IID = Intrinsic::x86_avx2_psllv_d_256;
2858 IID = Intrinsic::x86_avx512_psllv_w_128;
2860 IID = Intrinsic::x86_avx512_psllv_w_256;
2862 IID = Intrinsic::x86_avx512_psllv_w_512;
2867 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
2868 : Intrinsic::x86_sse2_psll_d;
2870 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
2871 : Intrinsic::x86_sse2_psll_q;
2873 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
2874 : Intrinsic::x86_sse2_psll_w;
2879 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
2880 : Intrinsic::x86_avx2_psll_d;
2882 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
2883 : Intrinsic::x86_avx2_psll_q;
2885 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
2886 : Intrinsic::x86_avx2_psll_w;
2891 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512 :
2892 IsVariable ? Intrinsic::x86_avx512_psllv_d_512 :
2893 Intrinsic::x86_avx512_psll_d_512;
2895 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512 :
2896 IsVariable ? Intrinsic::x86_avx512_psllv_q_512 :
2897 Intrinsic::x86_avx512_psll_q_512;
2899 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
2900 : Intrinsic::x86_avx512_psll_w_512;
2915 Intrinsic::ID IID;
2918 IID = Intrinsic::x86_avx2_psrlv_q;
2920 IID = Intrinsic::x86_avx2_psrlv_q_256;
2922 IID = Intrinsic::x86_avx2_psrlv_d;
2924 IID = Intrinsic::x86_avx2_psrlv_d_256;
2926 IID = Intrinsic::x86_avx512_psrlv_w_128;
2928 IID = Intrinsic::x86_avx512_psrlv_w_256;
2930 IID = Intrinsic::x86_avx512_psrlv_w_512;
2935 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
2936 : Intrinsic::x86_sse2_psrl_d;
2938 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
2939 : Intrinsic::x86_sse2_psrl_q;
2941 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
2942 : Intrinsic::x86_sse2_psrl_w;
2947 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
2948 : Intrinsic::x86_avx2_psrl_d;
2950 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
2951 : Intrinsic::x86_avx2_psrl_q;
2953 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
2954 : Intrinsic::x86_avx2_psrl_w;
2959 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512 :
2960 IsVariable ? Intrinsic::x86_avx512_psrlv_d_512 :
2961 Intrinsic::x86_avx512_psrl_d_512;
2963 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512 :
2964 IsVariable ? Intrinsic::x86_avx512_psrlv_q_512 :
2965 Intrinsic::x86_avx512_psrl_q_512;
2967 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
2968 : Intrinsic::x86_avx512_psrl_w_512;
2983 Intrinsic::ID IID;
2986 IID = Intrinsic::x86_avx2_psrav_d;
2988 IID = Intrinsic::x86_avx2_psrav_d_256;
2990 IID = Intrinsic::x86_avx512_psrav_w_128;
2992 IID = Intrinsic::x86_avx512_psrav_w_256;
2994 IID = Intrinsic::x86_avx512_psrav_w_512;
2999 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
3000 : Intrinsic::x86_sse2_psra_d;
3002 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128 :
3003 IsVariable ? Intrinsic::x86_avx512_psrav_q_128 :
3004 Intrinsic::x86_avx512_psra_q_128;
3006 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
3007 : Intrinsic::x86_sse2_psra_w;
3012 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
3013 : Intrinsic::x86_avx2_psra_d;
3015 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256 :
3016 IsVariable ? Intrinsic::x86_avx512_psrav_q_256 :
3017 Intrinsic::x86_avx512_psra_q_256;
3019 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
3020 : Intrinsic::x86_avx2_psra_w;
3025 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512 :
3026 IsVariable ? Intrinsic::x86_avx512_psrav_d_512 :
3027 Intrinsic::x86_avx512_psra_d_512;
3029 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512 :
3030 IsVariable ? Intrinsic::x86_avx512_psrav_q_512 :
3031 Intrinsic::x86_avx512_psra_q_512;
3033 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
3034 : Intrinsic::x86_avx512_psra_w_512;
3082 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
3083 Intrinsic::fma,
3098 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
3099 Intrinsic::fma,
3136 Intrinsic::ID IID;
3138 IID = Intrinsic::x86_avx512_vfmadd_f64;
3140 IID = Intrinsic::x86_avx512_vfmadd_f32;
3141 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), IID);
3144 Function *FMA = Intrinsic::getDeclaration(CI->getModule(),
3145 Intrinsic::fma,
3191 Intrinsic::ID IID;
3194 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
3196 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
3198 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3201 Function *FMA = Intrinsic::getDeclaration(CI->getModule(),
3202 Intrinsic::fma,
3220 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma,
3246 Intrinsic::ID IID;
3249 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
3251 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
3258 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3267 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma,
3293 Intrinsic::ID IID;
3295 IID = Intrinsic::x86_avx512_pternlog_d_128;
3297 IID = Intrinsic::x86_avx512_pternlog_d_256;
3299 IID = Intrinsic::x86_avx512_pternlog_d_512;
3301 IID = Intrinsic::x86_avx512_pternlog_q_128;
3303 IID = Intrinsic::x86_avx512_pternlog_q_256;
3305 IID = Intrinsic::x86_avx512_pternlog_q_512;
3311 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3321 Intrinsic::ID IID;
3323 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
3325 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
3327 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
3329 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
3331 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
3333 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
3339 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3357 Intrinsic::ID IID;
3359 IID = Intrinsic::x86_avx512_vpdpbusd_128;
3361 IID = Intrinsic::x86_avx512_vpdpbusd_256;
3363 IID = Intrinsic::x86_avx512_vpdpbusd_512;
3365 IID = Intrinsic::x86_avx512_vpdpbusds_128;
3367 IID = Intrinsic::x86_avx512_vpdpbusds_256;
3369 IID = Intrinsic::x86_avx512_vpdpbusds_512;
3375 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3387 Intrinsic::ID IID;
3389 IID = Intrinsic::x86_avx512_vpdpwssd_128;
3391 IID = Intrinsic::x86_avx512_vpdpwssd_256;
3393 IID = Intrinsic::x86_avx512_vpdpwssd_512;
3395 IID = Intrinsic::x86_avx512_vpdpwssds_128;
3397 IID = Intrinsic::x86_avx512_vpdpwssds_256;
3399 IID = Intrinsic::x86_avx512_vpdpwssds_512;
3405 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3413 Intrinsic::ID IID;
3415 IID = Intrinsic::x86_addcarry_32;
3417 IID = Intrinsic::x86_addcarry_64;
3419 IID = Intrinsic::x86_subborrow_32;
3421 IID = Intrinsic::x86_subborrow_64;
3429 Intrinsic::getDeclaration(CI->getModule(), IID),
3478 Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
3487 Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
3492 Rep = Builder.CreateCall(Intrinsic::getDeclaration(
3493 F->getParent(), Intrinsic::convert_from_fp16,
3519 case Intrinsic::experimental_vector_reduce_v2_fmul: {
3530 case Intrinsic::experimental_vector_reduce_v2_fadd: {
3541 case Intrinsic::arm_neon_vld1:
3542 case Intrinsic::arm_neon_vld2:
3543 case Intrinsic::arm_neon_vld3:
3544 case Intrinsic::arm_neon_vld4:
3545 case Intrinsic::arm_neon_vld2lane:
3546 case Intrinsic::arm_neon_vld3lane:
3547 case Intrinsic::arm_neon_vld4lane:
3548 case Intrinsic::arm_neon_vst1:
3549 case Intrinsic::arm_neon_vst2:
3550 case Intrinsic::arm_neon_vst3:
3551 case Intrinsic::arm_neon_vst4:
3552 case Intrinsic::arm_neon_vst2lane:
3553 case Intrinsic::arm_neon_vst3lane:
3554 case Intrinsic::arm_neon_vst4lane: {
3561 case Intrinsic::bitreverse:
3565 case Intrinsic::ctlz:
3566 case Intrinsic::cttz:
3573 case Intrinsic::objectsize: {
3584 case Intrinsic::ctpop:
3588 case Intrinsic::convert_from_fp16:
3592 case Intrinsic::dbg_value:
3606 case Intrinsic::x86_xop_vfrcz_ss:
3607 case Intrinsic::x86_xop_vfrcz_sd:
3611 case Intrinsic::x86_xop_vpermil2pd:
3612 case Intrinsic::x86_xop_vpermil2ps:
3613 case Intrinsic::x86_xop_vpermil2pd_256:
3614 case Intrinsic::x86_xop_vpermil2ps_256: {
3624 case Intrinsic::x86_sse41_ptestc:
3625 case Intrinsic::x86_sse41_ptestz:
3626 case Intrinsic::x86_sse41_ptestnzc: {
3647 case Intrinsic::x86_rdtscp: {
3673 case Intrinsic::x86_sse41_insertps:
3674 case Intrinsic::x86_sse41_dppd:
3675 case Intrinsic::x86_sse41_dpps:
3676 case Intrinsic::x86_sse41_mpsadbw:
3677 case Intrinsic::x86_avx_dp_ps_256:
3678 case Intrinsic::x86_avx2_mpsadbw: {
3690 case Intrinsic::thread_pointer: {
3695 case Intrinsic::invariant_start:
3696 case Intrinsic::invariant_end:
3697 case Intrinsic::masked_load:
3698 case Intrinsic::masked_store:
3699 case Intrinsic::masked_gather:
3700 case Intrinsic::masked_scatter: {
3707 case Intrinsic::memcpy:
3708 case Intrinsic::memmove:
3709 case Intrinsic::memset: {
3885 llvm::Intrinsic::ID IntrinsicFunc) {
3891 Function *NewFn = llvm::Intrinsic::getDeclaration(&M, IntrinsicFunc);
3951 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
3959 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
3960 {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
3961 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
3962 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
3964 llvm::Intrinsic::objc_autoreleaseReturnValue},
3965 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
3966 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
3967 {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
3968 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
3969 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
3970 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
3971 {"objc_release", llvm::Intrinsic::objc_release},
3972 {"objc_retain", llvm::Intrinsic::objc_retain},
3973 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
3975 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
3977 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
3978 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
3979 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
3980 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
3982 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
3983 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
3984 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
3985 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
3986 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
3987 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
3988 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
3990 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
3992 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
3994 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
3996 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};